From c2328ec0f4ad45d84b3d96c2b3f80017ef30af52 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 04:18:45 -0800 Subject: [PATCH 0001/1345] [backport to 3.6] Support "bpo-" in Misc/NEWS (#42) (cherry picked from commit 79ab8be05fb4ffb5c258d2ca49be5fc2d4880431) --- Doc/tools/extensions/pyspecific.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 273191bbd3c025..1141d6c5ac1891 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -34,7 +34,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/3.6/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -79,7 +79,7 @@ def new_depart_literal_block(self, node): def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) - text = 'issue ' + issue + text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] @@ -225,7 +225,7 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('([Ii])ssue #([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -253,7 +253,7 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', + content = issue_re.sub(r'`bpo-\1 `__', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading From 89b1824e693419b20b6a9113e5293f1c1a78065f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 12 Feb 2017 19:13:31 +0300 Subject: [PATCH 0002/1345] bpo-27122: Fix comment to point to correct issue number (#48) It took me quite a bit to figure out what this was referring to, since the given issue number is wrong, and the original commit message I found through git blame lists a different, also wrong issue number... see https://bugs.python.org/issue27122#msg279449 (cherry picked from commit af88e7eda4101f36e904771d3cf59a5f740b3b00) --- Lib/contextlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 7d94a579c872b7..8421968525947e 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -105,7 +105,7 @@ def __exit__(self, type, value, traceback): # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: - # Don't re-raise the passed in exception. (issue27112) + # Don't re-raise the passed in exception. (issue27122) if exc is value: return False # Likewise, avoid suppressing if a StopIteration exception From 308f789d00735e6707dbc5c8f60a1c8cf245fb4b Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 08:18:07 -0800 Subject: [PATCH 0003/1345] bpo-29474: Improve documentation for weakref.WeakValueDictionary (#22) There were some grammatical errors in weakref.WeakValueDictionary documentation. --- Doc/library/weakref.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index e289b971e7c269..b02a006d733b6e 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -166,8 +166,8 @@ Extension types can easily be made to support weak references; see performed by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakKeyDictionary` objects have the following additional methods. These -expose the internal references directly. The references are not guaranteed to +:class:`WeakKeyDictionary` objects have an additional method that +exposes the internal references directly. The references are not guaranteed to be "live" at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer @@ -192,9 +192,9 @@ than needed. by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakValueDictionary` objects have the following additional methods. -These method have the same issues as the and :meth:`keyrefs` method of -:class:`WeakKeyDictionary` objects. +:class:`WeakValueDictionary` objects have an additional method that has the +same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` +objects. .. method:: WeakValueDictionary.valuerefs() From f66c81ff499fb431e56bc68f5e39c2f7b9fcb6a7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 13:08:24 -0800 Subject: [PATCH 0004/1345] [backport to 3.6] bpo-28929: Link the documentation to its source file on GitHub (#37) * bpo-28929: Link the documentation to its source file on GitHub Change the documentation's `Show Source` link on the left menu to GitHub source file. (cherry picked from commit 23bafa294c75c20cb85ae5d97d7571a3a0ad8dd3) * remove if statement --- Doc/tools/templates/customsourcelink.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 243d8107779361..71d0bba683074e 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -3,8 +3,11 @@

{{ _('This Page') }}

{%- endif %} From 89ddffbe9dcb38b79f99563b0d4d594d1105a192 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 13 Feb 2017 09:19:05 +0900 Subject: [PATCH 0005/1345] bpo-29438: fixed use-after-free in key sharing dict (#39) --- Misc/NEWS | 2 ++ Objects/dictobject.c | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index d79ff2450defaa..ceea2226561fd6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29438: Fixed use-after-free problem in key sharing dict. + - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7b403bcecc5da..b63b78a337225a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4376,15 +4376,19 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - if (cached != ((PyDictObject *)dict)->ma_keys) { + // Since key sharing dict doesn't allow deletion, PyDict_DelItem() + // always converts dict to combined form. + if ((cached = CACHED_KEYS(tp)) != NULL) { CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } } else { - int was_shared = cached == ((PyDictObject *)dict)->ma_keys; + int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + if (was_shared && + (cached = CACHED_KEYS(tp)) != NULL && + cached != ((PyDictObject *)dict)->ma_keys) { /* PyDict_SetItem() may call dictresize and convert split table * into combined table. In such case, convert it to split * table again and update type's shared key only when this is From 2d0c2286f8e19e5d6a45cf16f766382115431c2e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 13 Feb 2017 15:48:40 -0800 Subject: [PATCH 0006/1345] A few README tweaks (#73) (#79) * Add a paragraph at the top for users, not builders, of Python. * Use nicer rst url syntax to avoid borking paragraphs in the plain text. Contributed by Ned Batchelder @nedbat (cherry picked from commit 3cdbd68ce8230cff1afb67472b96fbfa7f047e32) --- README | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README b/README index 4fc97e6013063e..03c4b03f68a1ff 100644 --- a/README +++ b/README @@ -11,6 +11,14 @@ especially how built-in objects like dictionaries and strings work, have changed considerably, and a lot of deprecated features have finally been removed. +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + Build Instructions ------------------ @@ -151,7 +159,7 @@ IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* include the output of "make test". It is useless. Run the failing test manually, as follows: - ./python -m test -v test_whatever + ./python -m test -v test_whatever (substituting the top of the source tree for '.' if you built in a different directory). This runs the test in verbose mode. @@ -198,11 +206,12 @@ Proposals for enhancement ------------------------- If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or python-ideas mailing lists for initial feedback. A Python +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. All current PEPs, as well as guidelines for submitting a new PEP, are listed at https://www.python.org/dev/peps/. +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ Release Schedule ---------------- @@ -231,4 +240,3 @@ so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. All trademarks referenced herein are property of their respective holders. - From 0230e64d2c976ab755c145e97bf86032e0fe3a53 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 06:11:12 -0800 Subject: [PATCH 0007/1345] bpo-28556: Various updates to typing (#28) (#77) various updates from upstream python/typing repo: - Added typing.Counter and typing.ChainMap generics - More flexible typing.NamedTuple - Improved generic ABC caching - More tests - Bugfixes - Other updates * Add Misc/NEWS entry (cherry picked from commit b692dc8475a032740576129d0990ddc3edccab2b) --- Lib/test/mod_generics_cache.py | 14 ++ Lib/test/test_typing.py | 237 +++++++++++++++++++++++++++++---- Lib/typing.py | 138 ++++++++++++++----- Misc/NEWS | 4 + 4 files changed, 335 insertions(+), 58 deletions(-) create mode 100644 Lib/test/mod_generics_cache.py diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py new file mode 100644 index 00000000000000..d9a60b4b28c325 --- /dev/null +++ b/Lib/test/mod_generics_cache.py @@ -0,0 +1,14 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic + +T = TypeVar('T') + + +class A(Generic[T]): + pass + + +class B(Generic[T]): + class A(Generic[T]): + pass diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fce6b5aaffdf93..64d8276658ee7f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -190,6 +190,10 @@ def test_bound_errors(self): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, contravariant=True) + class UnionTests(BaseTestCase): @@ -254,6 +258,11 @@ def test_repr(self): self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) u = Union[int, Employee] self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -304,6 +313,15 @@ def test_union_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Union[int, str]) + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 A = Union[str, Pattern] @@ -401,6 +419,8 @@ def test_callable_wrong_forms(self): Callable[[()], int] with self.assertRaises(TypeError): Callable[[int, 1], 2] + with self.assertRaises(TypeError): + Callable[int] def test_callable_instance_works(self): def f(): @@ -546,15 +566,27 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) def test_generic_errors(self): T = TypeVar('T') + S = TypeVar('S') with self.assertRaises(TypeError): Generic[T]() + with self.assertRaises(TypeError): + Generic[T][T] + with self.assertRaises(TypeError): + Generic[T][S] with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): issubclass(list, List[int]) + with self.assertRaises(TypeError): + class NewGeneric(Generic): ... + with self.assertRaises(TypeError): + class MyGeneric(Generic[T], Generic[S]): ... + with self.assertRaises(TypeError): + class MyGeneric(List[T], Generic[S]): ... def test_init(self): T = TypeVar('T') @@ -738,6 +770,53 @@ def test_subscript_meta(self): self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + def test_generic_hashes(self): + try: + from test import mod_generics_cache + except ImportError: # for Python 3.4 and previous versions + import mod_generics_cache + class A(Generic[T]): + ... + + class B(Generic[T]): + class A(Generic[T]): + ... + + self.assertEqual(A, A) + self.assertEqual(mod_generics_cache.A[str], mod_generics_cache.A[str]) + self.assertEqual(B.A, B.A) + self.assertEqual(mod_generics_cache.B.A[B.A[str]], + mod_generics_cache.B.A[B.A[str]]) + + self.assertNotEqual(A, B.A) + self.assertNotEqual(A, mod_generics_cache.A) + self.assertNotEqual(A, mod_generics_cache.B.A) + self.assertNotEqual(B.A, mod_generics_cache.A) + self.assertNotEqual(B.A, mod_generics_cache.B.A) + + self.assertNotEqual(A[str], B.A[str]) + self.assertNotEqual(A[List[Any]], B.A[List[Any]]) + self.assertNotEqual(A[str], mod_generics_cache.A[str]) + self.assertNotEqual(A[str], mod_generics_cache.B.A[str]) + self.assertNotEqual(B.A[int], mod_generics_cache.A[int]) + self.assertNotEqual(B.A[List[Any]], mod_generics_cache.B.A[List[Any]]) + + self.assertNotEqual(Tuple[A[str]], Tuple[B.A[str]]) + self.assertNotEqual(Tuple[A[List[Any]]], Tuple[B.A[List[Any]]]) + self.assertNotEqual(Union[str, A[str]], Union[str, mod_generics_cache.A[str]]) + self.assertNotEqual(Union[A[str], A[str]], + Union[A[str], mod_generics_cache.A[str]]) + self.assertNotEqual(typing.FrozenSet[A[str]], + typing.FrozenSet[mod_generics_cache.B.A[str]]) + + if sys.version_info[:2] > (3, 2): + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -835,6 +914,8 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] + with self.assertRaises(TypeError): + isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -853,6 +934,11 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + # ChainMap was added in 3.3 + if sys.version_info >= (3, 3): + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) @@ -1203,6 +1289,19 @@ def test_forwardref_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, fr) + def test_forwardref_subclass_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forward_equality(self): + fr = typing._ForwardRef('int') + self.assertEqual(fr, typing._ForwardRef('int')) + self.assertNotEqual(List['int'], List[int]) + + def test_forward_repr(self): + self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + def test_union_forward(self): def foo(a: Union['T']): @@ -1285,6 +1384,15 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) + def test_meta_no_type_check(self): @no_type_check_decorator @@ -1401,11 +1509,16 @@ class A: class B(A): x: ClassVar[Optional['B']] = None y: int + b: int class CSub(B): z: ClassVar['CSub'] = B() class G(Generic[T]): lst: ClassVar[List[T]] = [] +class NoneAndForward: + parent: 'NoneAndForward' + meaning: None + class CoolEmployee(NamedTuple): name: str cool: int @@ -1419,10 +1532,13 @@ class XMeth(NamedTuple): def double(self): return 2 * self.x -class XMethBad(NamedTuple): +class XRepr(NamedTuple): x: int - def _fields(self): - return 'no chance for this' + y: int = 1 + def __str__(self): + return f'{self.x} -> {self.y}' + def __add__(self, other): + return 0 """ if PY36: @@ -1431,7 +1547,7 @@ def _fields(self): # fake names for the sake of static analysis ann_module = ann_module2 = ann_module3 = None A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XMethBad = object + XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1466,6 +1582,8 @@ def test_get_type_hints_classes(self): {'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.S), {'x': str, 'y': str}) self.assertEqual(gth(ann_module.foo), {'x': int}) + self.assertEqual(gth(NoneAndForward, globals()), + {'parent': NoneAndForward, 'meaning': type(None)}) @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @@ -1482,17 +1600,22 @@ def meth(x: int): ... class Der(ABase): ... self.assertEqual(gth(ABase.meth), {'x': int}) - def test_get_type_hints_for_builins(self): + def test_get_type_hints_for_builtins(self): # Should not fail for built-in classes and functions. self.assertEqual(gth(int), {}) self.assertEqual(gth(type), {}) self.assertEqual(gth(dir), {}) self.assertEqual(gth(len), {}) + self.assertEqual(gth(object.__str__), {}) + self.assertEqual(gth(object().__str__), {}) + self.assertEqual(gth(str.join), {}) def test_previous_behavior(self): def testf(x, y): ... testf.__annotations__['x'] = 'int' self.assertEqual(gth(testf), {'x': int}) + def testg(x: None): ... + self.assertEqual(gth(testg), {'x': type(None)}) def test_get_type_hints_for_object_with_annotations(self): class A: ... @@ -1506,9 +1629,10 @@ def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) self.assertEqual(gth(B, globals()), - {'y': int, 'x': ClassVar[Optional[B]]}) + {'y': int, 'x': ClassVar[Optional[B]], 'b': int}) self.assertEqual(gth(CSub, globals()), - {'z': ClassVar[CSub], 'y': int, 'x': ClassVar[Optional[B]]}) + {'z': ClassVar[CSub], 'y': int, 'b': int, + 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) @@ -1628,6 +1752,11 @@ def test_list(self): def test_deque(self): self.assertIsSubclass(collections.deque, typing.Deque) + class MyDeque(typing.Deque[int]): ... + self.assertIsInstance(MyDeque(), collections.deque) + + def test_counter(self): + self.assertIsSubclass(collections.Counter, typing.Counter) def test_set(self): self.assertIsSubclass(set, typing.Set) @@ -1680,13 +1809,10 @@ class MyDict(typing.Dict[str, int]): self.assertIsSubclass(MyDict, dict) self.assertNotIsSubclass(dict, MyDict) - def test_no_defaultdict_instantiation(self): - with self.assertRaises(TypeError): - typing.DefaultDict() - with self.assertRaises(TypeError): - typing.DefaultDict[KT, VT]() - with self.assertRaises(TypeError): - typing.DefaultDict[str, int]() + def test_defaultdict_instantiation(self): + self.assertIs(type(typing.DefaultDict()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[KT, VT]()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[str, int]()), collections.defaultdict) def test_defaultdict_subclass(self): @@ -1699,13 +1825,49 @@ class MyDefDict(typing.DefaultDict[str, int]): self.assertIsSubclass(MyDefDict, collections.defaultdict) self.assertNotIsSubclass(collections.defaultdict, MyDefDict) - def test_no_deque_instantiation(self): - with self.assertRaises(TypeError): - typing.Deque() - with self.assertRaises(TypeError): - typing.Deque[T]() - with self.assertRaises(TypeError): - typing.Deque[int]() + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_instantiation(self): + self.assertIs(type(typing.ChainMap()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[str, int]()), collections.ChainMap) + class CM(typing.ChainMap[KT, VT]): ... + self.assertIs(type(CM[int, str]()), CM) + + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_subclass(self): + + class MyChainMap(typing.ChainMap[str, int]): + pass + + cm = MyChainMap() + self.assertIsInstance(cm, MyChainMap) + + self.assertIsSubclass(MyChainMap, collections.ChainMap) + self.assertNotIsSubclass(collections.ChainMap, MyChainMap) + + def test_deque_instantiation(self): + self.assertIs(type(typing.Deque()), collections.deque) + self.assertIs(type(typing.Deque[T]()), collections.deque) + self.assertIs(type(typing.Deque[int]()), collections.deque) + class D(typing.Deque[T]): ... + self.assertIs(type(D[int]()), D) + + def test_counter_instantiation(self): + self.assertIs(type(typing.Counter()), collections.Counter) + self.assertIs(type(typing.Counter[T]()), collections.Counter) + self.assertIs(type(typing.Counter[int]()), collections.Counter) + class C(typing.Counter[T]): ... + self.assertIs(type(C[int]()), C) + + def test_counter_subclass_instantiation(self): + + class MyCounter(typing.Counter[int]): + pass + + d = MyCounter() + self.assertIsInstance(d, MyCounter) + self.assertIsInstance(d, typing.Counter) + self.assertIsInstance(d, collections.Counter) def test_no_set_instantiation(self): with self.assertRaises(TypeError): @@ -2018,6 +2180,14 @@ def test_basics(self): collections.OrderedDict([('name', str), ('id', int)])) self.assertIs(Emp._field_types, Emp.__annotations__) + def test_namedtuple_pyversion(self): + if sys.version_info[:2] < (3, 6): + with self.assertRaises(TypeError): + NamedTuple('Name', one=int, other=str) + with self.assertRaises(TypeError): + class NotYet(NamedTuple): + whatever = 0 + @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) @@ -2055,10 +2225,18 @@ class NonDefaultAfterDefault(NamedTuple): @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): - self.assertEquals(XMeth(1).double(), 2) - self.assertEquals(XMeth(42).x, XMeth(42)[0]) - self.assertEquals(XMethBad(1)._fields, ('x',)) - self.assertEquals(XMethBad(1).__annotations__, {'x': int}) + self.assertEqual(XMeth(1).double(), 2) + self.assertEqual(XMeth(42).x, XMeth(42)[0]) + self.assertEqual(str(XRepr(42)), '42 -> 1') + self.assertEqual(XRepr(1, 2) + XRepr(3), 0) + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad(NamedTuple): + x: int + def _fields(self): + return 'no chance for this' +""") @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): @@ -2138,6 +2316,12 @@ def test_basics(self): Pattern[Union[str, bytes]] Match[Union[bytes, str]] + def test_alias_equality(self): + self.assertEqual(Pattern[str], Pattern[str]) + self.assertNotEqual(Pattern[str], Pattern[bytes]) + self.assertNotEqual(Pattern[str], Match[str]) + self.assertNotEqual(Pattern[str], str) + def test_errors(self): with self.assertRaises(TypeError): # Doesn't fit AnyStr. @@ -2152,6 +2336,9 @@ def test_errors(self): with self.assertRaises(TypeError): # We don't support isinstance(). isinstance(42, Pattern[str]) + with self.assertRaises(TypeError): + # We don't support issubclass(). + issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') diff --git a/Lib/typing.py b/Lib/typing.py index c9e341753769e0..efe358faf20988 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,12 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +try: + from types import SlotWrapperType, MethodWrapperType, MethodDescriptorType +except ImportError: + SlotWrapperType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) # Please keep __all__ alphabetized within each category. @@ -62,6 +68,7 @@ 'SupportsRound', # Concrete collection types. + 'Counter', 'Deque', 'Dict', 'DefaultDict', @@ -849,19 +856,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -872,7 +866,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +880,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -974,6 +966,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) self.__parameters__ = tvars @@ -982,8 +975,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -994,20 +985,56 @@ def __new__(cls, name, bases, namespace, # with issubclass() and isinstance() in the same way as their # collections.abc counterparts (e.g., isinstance([], Iterable)). if ( - # allow overriding '__subclasshook__' not in namespace and extra or - hasattr(self.__subclasshook__, '__name__') and - self.__subclasshook__.__name__ == '__extrahook__' + # allow overriding + getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' ): self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = hash(self._subs_tree()) if origin else hash((self.__name__,)) + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1095,8 +1122,10 @@ def __getitem__(self, params): _check_generic(self, params) tvars = _type_vars(params) args = params + + prepend = (self,) if self.__origin__ is None else () return self.__class__(self.__name__, - self.__bases__, + prepend + self.__bases__, _no_slots_copy(self.__dict__), tvars=tvars, args=args, @@ -1104,6 +1133,17 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the @@ -1398,6 +1438,11 @@ def _get_defaults(func): return res +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + SlotWrapperType, MethodWrapperType, MethodDescriptorType) + + def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. @@ -1452,12 +1497,7 @@ def get_type_hints(obj, globalns=None, localns=None): hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. - if ( - isinstance(obj, types.FunctionType) or - isinstance(obj, types.BuiltinFunctionType) or - isinstance(obj, types.MethodType) or - isinstance(obj, types.ModuleType) - ): + if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' @@ -1824,8 +1864,7 @@ class Deque(collections.deque, MutableSequence[T], extra=collections.deque): def __new__(cls, *args, **kwds): if _geqv(cls, Deque): - raise TypeError("Type Deque cannot be instantiated; " - "use deque() instead") + return collections.deque(*args, **kwds) return _generic_new(collections.deque, cls, *args, **kwds) @@ -1894,11 +1933,35 @@ class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], def __new__(cls, *args, **kwds): if _geqv(cls, DefaultDict): - raise TypeError("Type DefaultDict cannot be instantiated; " - "use collections.defaultdict() instead") + return collections.defaultdict(*args, **kwds) return _generic_new(collections.defaultdict, cls, *args, **kwds) +class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, Counter): + return collections.Counter(*args, **kwds) + return _generic_new(collections.Counter, cls, *args, **kwds) + + +if hasattr(collections, 'ChainMap'): + # ChainMap only exists in 3.3+ + __all__.append('ChainMap') + + class ChainMap(collections.ChainMap, MutableMapping[KT, VT], + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, ChainMap): + return collections.ChainMap(*args, **kwds) + return _generic_new(collections.ChainMap, cls, *args, **kwds) + + # Determine what base class to use for Generator. if hasattr(collections_abc, 'Generator'): # Sufficiently recent versions of 3.5 have a Generator ABC. @@ -1975,6 +2038,13 @@ def _make_nmtuple(name, types): _PY36 = sys.version_info[:2] >= (3, 6) +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + class NamedTupleMeta(type): @@ -2002,7 +2072,9 @@ def __new__(cls, typename, bases, ns): nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: - if not hasattr(nm_tpl, key): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl diff --git a/Misc/NEWS b/Misc/NEWS index ceea2226561fd6..0fe1f9b9d651bf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -59,6 +59,10 @@ Extension Modules Library ------- +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Łukasz Langa. + - Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python 3.6.0: check minimum and maximum years. From 9268855ba6e99bf595e75b75780cf29667371f45 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 06:11:48 -0800 Subject: [PATCH 0008/1345] Fix some sphinx warnings (#9) (#81) * Fix some deprecation warnings in Doc/conf.py * Fix an rst error in Misc/NEWS Contributed by Ryan Gonzalez @kirbyfan64 (cherry picked from commit e7ffb99f842ebff97cffa0fc90b18be4e5abecf2) --- Doc/Makefile | 2 +- Doc/conf.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 91f937f985831a..6e1c18304080d6 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -10,7 +10,7 @@ PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text changes linkcheck \ diff --git a/Doc/conf.py b/Doc/conf.py index b1bb6208bb4b8e..b3f26d5a692163 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -88,11 +88,24 @@ # Options for LaTeX output # ------------------------ +# Get LaTeX to handle Unicode correctly +latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} + +# Additional stuff for the LaTeX preamble. +latex_elements['preamble'] = r''' +\authoraddress{ + \strong{Python Software Foundation}\\ + Email: \email{docs@python.org} +} +\let\Verbatim=\OriginalVerbatim +\let\endVerbatim=\endOriginalVerbatim +''' + # The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +latex_elements['papersize'] = 'a4' # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_elements['font_size'] = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -125,22 +138,9 @@ for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst') -# Additional stuff for the LaTeX preamble. -latex_preamble = r''' -\authoraddress{ - \strong{Python Software Foundation}\\ - Email: \email{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] -# Get LaTeX to handle Unicode correctly -latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} - # Options for Epub output # ----------------------- From db3deb98e969d3f43b169cd320abd46b21a10c6d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 08:54:59 -0800 Subject: [PATCH 0009/1345] bpo-29521 Fix two minor documentation build warnings (#41) (#83) Much of bpo-29521 was fixed in parallel with commit e7ffb99 . This cleans up the rest. Apply parallel change to Doc/make.bat to read "set SPHINXOPTS=-D latex_elements.papersize=" I don't have a Windows system on which to observe the warning, but it should be necessary. The warning: .../workspace/cpython_github/Doc/faq/windows.rst:303: WARNING: unknown option: -t In the Windows FAQ, `How do I keep editors from inserting tabs into my Python source?`, contained a reference to a Python -t option. In Python 2.x, this caused Python to issue warnings about lines with mixed spaces and tabs, but as of Python 3.6 it does nothing. Per discussion at http://bugs.python.org/issue29387, take their wording. Python [3] raises an IndentationError or TabError. Tabnanny is now a module. (cherry picked from commit 3d707be950b387552585451071928e7b39cdfa53) --- Doc/faq/windows.rst | 7 ++++--- Doc/make.bat | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d7253436beaf61..b9b7b8d359bdf2 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -If you suspect mixed tabs and spaces are causing problems in leading whitespace, -run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to -check a directory tree in batch mode. +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +and spaces are causing problems in leading whitespace. +You may also run the :mod:`tabnanny` module to check a directory tree +in batch mode. How do I check for a keypress without blocking? diff --git a/Doc/make.bat b/Doc/make.bat index da1f8765a4f358..d0b59618261011 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -86,7 +86,7 @@ goto end :build if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* From 7d5587e687ba78563b0f026019e638800497e021 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 15 Feb 2017 18:59:47 +0900 Subject: [PATCH 0010/1345] Update URL of Mersenne Twister Home Page (#20) (#115) --- Modules/_randommodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0d3282db8f2810..e5dd2c96166384 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -2,7 +2,7 @@ /* ------------------------------------------------------------------ The code in this module was based on a download from: - http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html It was modified in 2002 by Raymond Hettinger as follows: @@ -60,8 +60,8 @@ Any feedback is very welcome. - http://www.math.keio.ac.jp/matumoto/emt.html - email: matumoto@math.keio.ac.jp + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html + email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ /* ---------------------------------------------------------------*/ From 1e8cc88a48a049b064f786bb4b97ea60b70dc504 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 15 Feb 2017 13:29:16 -0600 Subject: [PATCH 0011/1345] Backport master's README.rst to 3.6 (GH-86) Includes GH-2, GH-70, GH-73 (GH-79), and GH-21. --- README | 242 ----------------------------------------------------- README.rst | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 242 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/README b/README deleted file mode 100644 index 03c4b03f68a1ff..00000000000000 --- a/README +++ /dev/null @@ -1,242 +0,0 @@ -This is Python version 3.6.1 release candidate 1 -================================================ - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights -reserved. - -Python 3.x is a new version of the language, which is incompatible with the -2.x line of releases. The language is mostly the same, but many details, -especially how built-in objects like dictionaries and strings work, -have changed considerably, and a lot of deprecated features have finally -been removed. - -Using Python ------------- - -Installable Python kits, and information about using Python, are available at -`python.org`_. - -.. _python.org: https://www.python.org/ - - -Build Instructions ------------------- - -On Unix, Linux, BSD, OSX, and Cygwin: - - ./configure - make - make test - sudo make install - -This will install Python as python3. - -You can pass many options to the configure script; run "./configure --help" to -find out more. On OSX and Cygwin, the executable is called python.exe; -elsewhere it's just python. - -On Mac OS X, if you have configured Python with --enable-framework, you should -use "make frameworkinstall" to do the installation. Note that this installs the -Python executable in a place that is not normally on your PATH, you may want to -set up a symlink in /usr/local/bin. - -On Windows, see PCbuild/readme.txt. - -If you wish, you can create a subdirectory and invoke configure from there. -For example: - - mkdir debug - cd debug - ../configure --with-pydebug - make - make test - -(This will fail if you *also* built at the top-level directory. -You should do a "make clean" at the toplevel first.) - -To get an optimized build of Python, "configure --enable-optimizations" before -you run make. This sets the default make targets up to enable Profile Guided -Optimization (PGO) and may be used to auto-enable Link Time Optimization (LTO) -on some platforms. For more details, see the sections bellow. - - -Profile Guided Optimization ---------------------------- - -PGO takes advantage of recent versions of the GCC or Clang compilers. -If ran, "make profile-opt" will do several steps. - -First, the entire Python directory is cleaned of temporary files that -may have resulted in a previous compilation. - -Then, an instrumented version of the interpreter is built, using suitable -compiler flags for each flavour. Note that this is just an intermediary -step and the binary resulted after this step is not good for real life -workloads, as it has profiling instructions embedded inside. - -After this instrumented version of the interpreter is built, the Makefile -will automatically run a training workload. This is necessary in order to -profile the interpreter execution. Note also that any output, both stdout -and stderr, that may appear at this step is suppressed. - -Finally, the last step is to rebuild the interpreter, using the information -collected in the previous one. The end result will be a Python binary -that is optimized and suitable for distribution or production installation. - - -Link Time Optimization ----------------------- - -Enabled via configure's --with-lto flag. LTO takes advantages of recent -compiler toolchains ability to optimize across the otherwise arbitrary .o file -boundary when building final executables or shared libraries for additional -performance gains. - - -What's New ----------- - -We have a comprehensive overview of the changes in the "What's New in -Python 3.6" document, found at - - https://docs.python.org/3.6/whatsnew/3.6.html - -For a more detailed change log, read Misc/NEWS (though this file, too, -is incomplete, and also doesn't list anything merged in from the 2.7 -release under development). - -If you want to install multiple versions of Python see the section below -entitled "Installing multiple versions". - - -Documentation -------------- - -Documentation for Python 3.6 is online, updated daily: - - https://docs.python.org/3.6/ - -It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version -is primarily for documentation authors, translators, and people with special -formatting requirements. - -If you would like to contribute to the development of Python, relevant -documentation is available at: - - https://docs.python.org/devguide/ - -For information about building Python's documentation, refer to Doc/README.txt. - - -Converting From Python 2.x to 3.x ---------------------------------- - -Python starting with 2.6 contains features to help locating code that needs to -be changed, such as optional warnings when deprecated features are used, and -backported versions of certain key Python 3.x features. - -A source-to-source translation tool, "2to3", can take care of the mundane task -of converting large amounts of source code. It is not a complete solution but -is complemented by the deprecation warnings in 2.6. See -https://docs.python.org/3.6/library/2to3.html for more information. - - -Testing -------- - -To test the interpreter, type "make test" in the top-level directory. -The test set produces some output. You can generally ignore the messages -about skipped tests due to optional features which can't be imported. -If a message is printed about a failed test or a traceback or core dump -is produced, something is wrong. - -By default, tests are prevented from overusing resources like disk space and -memory. To enable these tests, run "make testall". - -IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* -include the output of "make test". It is useless. Run the failing test -manually, as follows: - - ./python -m test -v test_whatever - -(substituting the top of the source tree for '.' if you built in a different -directory). This runs the test in verbose mode. - - -Installing multiple versions ----------------------------- - -On Unix and Mac systems if you intend to install multiple versions of Python -using the same installation prefix (--prefix argument to the configure script) -you must take care that your primary python executable is not overwritten by the -installation of a different version. All files and directories installed using -"make altinstall" contain the major and minor version and can thus live -side-by-side. "make install" also creates ${prefix}/bin/python3 which refers to -${prefix}/bin/pythonX.Y. If you intend to install multiple versions using the -same prefix you must decide which version (if any) is your "primary" version. -Install that version using "make install". Install all other versions using -"make altinstall". - -For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the -primary version, you would execute "make install" in your 3.6 build directory -and "make altinstall" in the others. - - -Issue Tracker and Mailing List ------------------------------- - -We're soliciting bug reports about all aspects of the language. Fixes are also -welcome, preferably in unified diff format. Please use the issue tracker: - - https://bugs.python.org/ - -If you're not sure whether you're dealing with a bug or a feature, use the -mailing list: - - python-dev@python.org - -To subscribe to the list, use the mailman form: - - https://mail.python.org/mailman/listinfo/python-dev/ - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A Python -Enhancement Proposal (PEP) may be submitted if your idea gains ground. All -current PEPs, as well as guidelines for submitting a new PEP, are listed at -https://www.python.org/dev/peps/. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ - -Release Schedule ----------------- - -See PEP 494 for release details: https://www.python.org/dev/peps/pep-0494/ - - -Copyright and License Information ---------------------------------- - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. - -Copyright (c) 2000 BeOpen.com. All rights reserved. - -Copyright (c) 1995-2001 Corporation for National Research Initiatives. All -rights reserved. - -Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -See the file "LICENSE" for information on the history of this software, -terms & conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. - -This Python distribution contains *no* GNU General Public License (GPL) code, -so it may be used in proprietary projects. There are interfaces to some GNU -code but these are entirely optional. - -All trademarks referenced herein are property of their respective holders. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000000000..d02ae64b7fc660 --- /dev/null +++ b/README.rst @@ -0,0 +1,235 @@ +This is Python version 3.6.1 release candidate 1 +================================================ + +.. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 + :alt: CPython build status on Travis CI + :target: https://travis-ci.org/python/cpython + +.. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg + :alt: CPython code coverage on Codecov + :target: https://codecov.io/gh/python/cpython + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights +reserved. + +See the end of this file for further copyright and license information. + + +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + + +Build Instructions +------------------ + +On Unix, Linux, BSD, OSX, and Cygwin:: + + ./configure + make + make test + sudo make install + +This will install Python as python3. + +You can pass many options to the configure script; run ``./configure --help`` +to find out more. On OSX and Cygwin, the executable is called ``python.exe``; +elsewhere it's just ``python``. + +On Mac OS X, if you have configured Python with ``--enable-framework``, you +should use ``make frameworkinstall`` to do the installation. Note that this +installs the Python executable in a place that is not normally on your PATH, +you may want to set up a symlink in ``/usr/local/bin``. + +On Windows, see `PCbuild/readme.txt +`_. + +If you wish, you can create a subdirectory and invoke configure from there. +For example:: + + mkdir debug + cd debug + ../configure --with-pydebug + make + make test + +(This will fail if you *also* built at the top-level directory. You should do +a ``make clean`` at the toplevel first.) + +To get an optimized build of Python, ``configure --enable-optimizations`` +before you run ``make``. This sets the default make targets up to enable +Profile Guided Optimization (PGO) and may be used to auto-enable Link Time +Optimization (LTO) on some platforms. For more details, see the sections +below. + + +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. If ran, +``make profile-opt`` will do several steps. + +First, the entire Python directory is cleaned of temporary files that may have +resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary step +and the binary resulted after this step is not good for real life workloads, as +it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile will +automatically run a training workload. This is necessary in order to profile +the interpreter execution. Note also that any output, both stdout and stderr, +that may appear at this step is suppressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a Python binary that is +optimized and suitable for distribution or production installation. + + +Link Time Optimization +---------------------- + +Enabled via configure's ``--with-lto`` flag. LTO takes advantage of the +ability of recent compiler toolchains to optimize across the otherwise +arbitrary ``.o`` file boundary when building final executables or shared +libraries for additional performance gains. + + +What's New +---------- + +We have a comprehensive overview of the changes in the `What's New in Python +3.6 `_ document. For a more +detailed change log, read `Misc/NEWS +`_, but a full +accounting of changes can only be gleaned from the `commit history +`_. + +If you want to install multiple versions of Python see the section below +entitled "Installing multiple versions". + + +Documentation +------------- + +`Documentation for Python 3.6 `_ is online, +updated daily. + +It can also be downloaded in many formats for faster access. The documentation +is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is primarily for documentation authors, translators, and people with special +formatting requirements. + +If you would like to contribute to the development of Python, relevant +documentation is available in the `Python Developer's Guide +`_. + +For information about building Python's documentation, refer to `Doc/README.txt +`_. + + +Converting From Python 2.x to 3.x +--------------------------------- + +Significant backward incompatible changes were made for the release of Python +3.0, which may cause programs written for Python 2 to fail when run with Python +3. For more information about porting your code from Python 2 to Python 3, see +the `Porting HOWTO `_. + + +Testing +------- + +To test the interpreter, type ``make test`` in the top-level directory. The +test set produces some output. You can generally ignore the messages about +skipped tests due to optional features which can't be imported. If a message +is printed about a failed test or a traceback or core dump is produced, +something is wrong. + +By default, tests are prevented from overusing resources like disk space and +memory. To enable these tests, run ``make testall``. + +If any tests fail, you can re-run the failing test(s) in verbose mode:: + + make test TESTOPTS="-v test_that_failed" + +If the failure persists and appears to be a problem with Python rather than +your environment, you can `file a bug report `_ and +include relevant output from that command to show the issue. + + +Installing multiple versions +---------------------------- + +On Unix and Mac systems if you intend to install multiple versions of Python +using the same installation prefix (``--prefix`` argument to the configure +script) you must take care that your primary python executable is not +overwritten by the installation of a different version. All files and +directories installed using ``make altinstall`` contain the major and minor +version and can thus live side-by-side. ``make install`` also creates +``${prefix}/bin/python3`` which refers to ``${prefix}/bin/pythonX.Y``. If you +intend to install multiple versions using the same prefix you must decide which +version (if any) is your "primary" version. Install that version using ``make +install``. Install all other versions using ``make altinstall``. + +For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the +primary version, you would execute ``make install`` in your 3.6 build directory +and ``make altinstall`` in the others. + + +Issue Tracker and Mailing List +------------------------------ + +Bug reports are welcome! You can use the `issue tracker +`_ to report bugs, and/or submit pull requests `on +GitHub `_. + +You can also follow development discussion on the `python-dev mailing list +`_. + + +Proposals for enhancement +------------------------- + +If you have a proposal to change Python, you may want to send an email to the +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A +Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. +All current PEPs, as well as guidelines for submitting a new PEP, are listed at +`python.org/dev/peps/ `_. + +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ + + +Release Schedule +---------------- + +See :pep:`494` for Python 3.6 release details. + + +Copyright and License Information +--------------------------------- + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. + +Copyright (c) 2000 BeOpen.com. All rights reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. All +rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. + +See the file "LICENSE" for information on the history of this software, terms & +conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. + +This Python distribution contains *no* GNU General Public License (GPL) code, +so it may be used in proprietary projects. There are interfaces to some GNU +code but these are entirely optional. + +All trademarks referenced herein are property of their respective holders. From 7224a049b88ed37c510861528147e0db54911bb7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 15 Feb 2017 11:38:48 -0800 Subject: [PATCH 0012/1345] bpo-29481: add versionadded 3.6.1 to typing.Deque docs (#108) (cherry picked from commit 7e147f1ddb8233964ff0981e6b64fc12edac99aa) --- Doc/library/typing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6c8982ba743526..d130e1759d8f59 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -574,6 +574,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.6.1 + .. class:: List(list, MutableSequence[T]) Generic version of :class:`list`. From 02e3357e089f37c23d0f3d1ebee9aa3d7a1492a9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 15 Feb 2017 15:14:41 -0800 Subject: [PATCH 0013/1345] bpo-29521 update Misc/ACKS (#110) (cherry picked from commit 6420088b924a23e5de40be6623d2a80b12f71d97) --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index a6b3df12ff230c..1d6e773b553c52 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -347,6 +347,7 @@ A. Jesse Jiryu Davis Merlijn van Deen John DeGood Ned Deily +Jim DeLaHunt Vincent Delft Arnaud Delobelle Konrad Delong From b300c660d34d2027d443098ea605a8e0eb51d383 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Feb 2017 00:56:54 +0100 Subject: [PATCH 0014/1345] Backport36 doc fixes: PR#68 and PR#124 (#125) * Travis CI: run rstlint.py in the docs job (#68) Currently, http://buildbot.python.org/all/buildslaves/ware-docs buildbot is only run as post-commit. For example, bpo-29521 (PR#41) introduced two warnings, unnotified by the Travis CI docs job. Modify the docs job to run toosl/rstlint.py. Fix also the two minor warnings which causes the buildbot slave to fail. (cherry picked from commit 2b501866ed493758e4c4b29f0ce9b24023d910a1) * Doc/Makefile: set PYTHON to python3 (#124) rstlint.py run by "make check" doesn't support Python 2. "make venv" runs "$(PYTHON) -m venv", whereas Python 2 doens't provide the venv module: it's a module of Python 3 standard library. (cherry picked from commit 91b0e7d0ca7c59df28f6a6fc1e8eb86a3925b76c) --- Doc/Makefile | 2 +- Doc/faq/windows.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 6e1c18304080d6..eb62df5b87cebe 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -4,7 +4,7 @@ # # You can set these variables from the command line. -PYTHON = python +PYTHON = python3 SPHINXBUILD = sphinx-build PAPER = SOURCES = diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index b9b7b8d359bdf2..6ac83e45d2e815 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,9 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs and spaces are causing problems in leading whitespace. -You may also run the :mod:`tabnanny` module to check a directory tree +You may also run the :mod:`tabnanny` module to check a directory tree in batch mode. From d372cda5cd46712c4e59262ec1ab981773b20bff Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 18 Feb 2017 16:12:06 -0800 Subject: [PATCH 0015/1345] bpo-29579: Removes readme.txt from the installer. (#161) --- Misc/NEWS | 2 ++ Tools/msi/exe/exe_files.wxs | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 0fe1f9b9d651bf..5f682e5d8fb4a4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -160,6 +160,8 @@ Library Windows ------- +- bpo-29579: Removes readme.txt from the installer + - Issue #29326: Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) - Issue #28164: Correctly handle special console filenames (patch by Eryk Sun) diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs index 01385874fa90b7..e675c21c8975ef 100644 --- a/Tools/msi/exe/exe_files.wxs +++ b/Tools/msi/exe/exe_files.wxs @@ -8,9 +8,6 @@ - - - From 0683d6889bd4430599d22e12e201b8e9c45be5a2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 19 Feb 2017 10:03:35 +0530 Subject: [PATCH 0016/1345] [3.6] bpo-29571: Use correct locale encoding in test_re (#149) (#153) ``local.getlocale(locale.LC_CTYPE)`` and ``locale.getpreferredencoding(False)`` may give different answers in some cases (such as the ``en_IN`` locale). ``re.LOCALE`` uses the latter, so update the test case to match. --- Lib/test/test_re.py | 2 +- Misc/NEWS | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b945cf094e99c1..a506b98be4aeb2 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1402,7 +1402,7 @@ def test_ascii_and_unicode_flag(self): def test_locale_flag(self): import locale - _, enc = locale.getlocale(locale.LC_CTYPE) + enc = locale.getpreferredencoding(False) # Search non-ASCII letter for i in range(128, 256): try: diff --git a/Misc/NEWS b/Misc/NEWS index 5f682e5d8fb4a4..b631fde8e52e0d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -202,6 +202,11 @@ Documentation Tests ----- +- Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, + test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to + determine the candidate encoding for the test regex (allowing it to correctly + skip the test when the default locale encoding is a multi-byte encoding) + - Issue #28950: Disallow -j0 to be combined with -T/-l in regrtest command line arguments. From 3337d33a4518f7ab8a7ab6c9a75b8b92ba348b27 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Feb 2017 14:11:59 +0900 Subject: [PATCH 0017/1345] [backport to 3.6] bpo-29529: Add .travis.yml to 3.6 branch (#25) * Add .travis.yml to 3.6 branch * Travis CI: run rstlint.py in the docs job (#68) Currently, http://buildbot.python.org/all/buildslaves/ware-docs buildbot is only run as post-commit. For example, bpo-29521 (PR#41) introduced two warnings, unnotified by the Travis CI docs job. Modify the docs job to run toosl/rstlint.py. Fix also the two minor warnings which causes the buildbot slave to fail. * Only run CI checks when appropriate files have changed (#74) Closes python/core-workflow#14 * Use 'make check' instead of 'python3 tools/rstlint.py' (#96) --- .travis.yml | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..dc3a00d1f4b5a3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,104 @@ +language: c +dist: trusty +sudo: false +group: beta + +# To cache doc-building dependencies. +cache: pip + +os: + - linux + # macOS builds are disabled as the machines are under-provisioned on Travis, + # adding up to an extra hour completing a full CI run. + #- osx + +compiler: + - clang + - gcc + +env: + - TESTING=cpython + +matrix: + allow_failures: + - env: + - TESTING=coverage + include: + - os: linux + language: python + python: 3.5 + env: + - TESTING=docs + before_script: + - | + if git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '^Doc/' + then + echo "Docs weren't updated, stopping build process." + exit + fi + cd Doc + make venv PYTHON=python3 + script: + - make html SPHINXBUILD="./venv/bin/python3 -m sphinx" SPHINXOPTS="-q" + - make check + - os: linux + language: c + compiler: clang + env: + - TESTING=coverage + before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure + make -s -j4 + # Need a venv that can parse covered code. + ./python -m venv venv + ./venv/bin/python -m pip install -U coverage + script: + # Skip tests that re-run the entire test suite. + - ./venv/bin/python -m coverage run --pylib -m test -uall -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - source ./venv/bin/activate + - bash <(curl -s https://codecov.io/bash) + - os: linux + language: cpp + compiler: clang + env: + - TESTING="C++ header compatibility" + before_script: + - ./configure + script: + - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . + +# Travis provides only 2 cores, so don't overdue the parallelism and waste memory. +before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure --with-pydebug + make -j4 + +script: + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4" + +notifications: + email: false + irc: + channels: + # This is set to a secure variable to prevent forks from notifying the + # IRC channel whenever they fail a build. This can be removed when travis + # implements https://github.com/travis-ci/travis-ci/issues/1094. + # The actual value here is: irc.freenode.net#python-dev + - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" + on_success: change + on_failure: always + skip_join: true From b121d737fe60e56dac5a44535e02089c6c02613d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 19 Feb 2017 14:40:55 +0300 Subject: [PATCH 0018/1345] Tweak PEP 519 documentation in stdlib (#164) * Drop duplicate word 'object' in lzma docs * Fix typo in os docs: fpr -> for (cherry picked from commit d4d48743ac20854de104e08dd66972471684f676) --- Doc/library/lzma.rst | 4 ++-- Doc/library/os.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 5edb23de83bcff..cce6c23e611e31 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,7 +39,7 @@ Reading and writing compressed files object`. The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like object` object), in + :class:`str`, :class:`bytes` or :term:`path-like ` object), in which case the named file is opened, or it can be an existing file object to read from or write to. @@ -76,7 +76,7 @@ Reading and writing compressed files An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate directly on a named file. The *filename* argument specifies either the file object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like object` object). When wrapping an + :class:`bytes` or :term:`path-like ` object). When wrapping an existing file object, the wrapped file will not be closed when the :class:`LZMAFile` is closed. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 37fa2a2868dac9..974ab2d481e210 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2859,7 +2859,7 @@ These functions are all available on Linux only. :ref:`not following symlinks `. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` fpr *path* and *attribute*. + Accepts a :term:`path-like object` for *path* and *attribute*. .. function:: listxattr(path=None, *, follow_symlinks=True) From 7970cd483346dfd7723da214fb27399ecc574095 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 20 Feb 2017 09:46:46 +0900 Subject: [PATCH 0019/1345] bpo-29520: doc: fix deprecation warning from 'defindex' template (GH-178) --- Doc/tools/templates/indexcontent.html | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1076c1f51b7d19..d795c0a5586bc8 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -1,5 +1,12 @@ -{% extends "defindex.html" %} -{% block tables %} +{% extends "layout.html" %} +{%- block htmltitle -%} +{{ shorttitle }} +{%- endblock -%} +{% block body %} +

{{ docstitle|e }}

+

+ {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} +

{% trans %}Parts of the documentation:{% endtrans %}

From a4afdfcf27c2fa53c60b65867765244361abb1fc Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 19 Feb 2017 18:58:33 -0800 Subject: [PATCH 0020/1345] Change some mercurial/ hg.python.org references. (#8) (#185) (cherry picked from commit b2ee40ed9c9041dcff9c898aa19aacf9ec60308a) --- Doc/faq/general.rst | 6 +++--- Lib/idlelib/help.html | 2 +- Tools/README | 2 +- Tools/importbench/README | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index f1e33afdabf8a5..8f6a907a8a2fda 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -159,7 +159,7 @@ How do I obtain a copy of the Python source? The latest Python source distribution is always available from python.org, at https://www.python.org/downloads/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +at https://github.com/python/cpython/. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and @@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce newsgroups and on the Python home page at https://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Git. See +`The Python Developer's Guide `_ for details. How do I submit bug reports and patches for Python? diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ffc03c4112f073..f10cd345e886c8 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -90,7 +90,7 @@

Navigation

25.5. IDLE

-

Source code: Lib/idlelib/

+

Source code: Lib/idlelib/

IDLE is Python’s Integrated Development and Learning Environment.

IDLE has the following features:

diff --git a/Tools/README b/Tools/README index 0d961de23d6cb8..edbf4fb83321ba 100644 --- a/Tools/README +++ b/Tools/README @@ -45,4 +45,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test discovery. -(*) A generic benchmark suite is maintained separately at http://hg.python.org/benchmarks/ +(*) A generic benchmark suite is maintained separately at https://github.com/python/performance diff --git a/Tools/importbench/README b/Tools/importbench/README index 81a5544a383b4c..6ba386c2b608d5 100644 --- a/Tools/importbench/README +++ b/Tools/importbench/README @@ -3,4 +3,4 @@ Importbench is a set of micro-benchmarks for various import scenarios. It should not be used as an overall benchmark of import performance, but rather an easy way to measure impact of possible code changes. For a real-world benchmark of import, use the normal_startup benchmark from -hg.python.org/benchmarks. +https://github.com/python/performance From 9a4577a4bb23888fed2cf192cf1a4c95ce5c26f8 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 20 Feb 2017 14:33:02 +0800 Subject: [PATCH 0021/1345] bpo-29347: Fix possibly dereferencing undefined pointers when creating weakref objects (#128) (#186) --- Misc/NEWS | 3 +++ Objects/weakrefobject.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index b631fde8e52e0d..0fed99b5e7808f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + - bpo-29438: Fixed use-after-free problem in key sharing dict. - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b23525552b8..9ca386da2563cb 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -24,6 +24,8 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; self->wr_object = ob; + self->wr_prev = NULL; + self->wr_next = NULL; Py_XINCREF(callback); self->wr_callback = callback; } From ce552e2d5c4ff90218fb41847e8ffb1fd3ba3b2d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 20 Feb 2017 22:58:11 +0900 Subject: [PATCH 0022/1345] bpo-24274: fix erroneous comment in dictobject.c (GH-200) lookdict_unicode() and lookdict_unicode_nodummy() may raise exception when key is not unicode. --- Objects/dictobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b63b78a337225a..a0c1977037b9e6 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -676,7 +676,8 @@ Christian Tismer. lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR. +never raise an exception; that function can never return DKIX_ERROR when key +is string. Otherwise, it falls back to lookdict(). lookdict_unicode_nodummy is further specialized for string keys that cannot be the value. For both, when the key isn't found a DKIX_EMPTY is returned. hashpos returns From d9b3cdd137239a5913de2252c3ce269e35ac63d2 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 20 Feb 2017 21:59:30 +0000 Subject: [PATCH 0023/1345] bpo-29602: fix signed zero handling in complex constructor. (#203) (#206) * Fix incorrect handling of signed zeros for complex-related classes. * Add Misc/NEWS entry. (cherry picked from commit 112ec38c15b388fe025ccb85369a584d218b1160) --- Lib/test/test_complex.py | 23 +++++++++++++++++++++++ Misc/NEWS | 4 ++++ Objects/complexobject.c | 6 +++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c249ca724bf0ce..cee49343e268a2 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -387,6 +387,29 @@ def __complex__(self): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + @support.requires_IEEE_754 + def test_constructor_special_numbers(self): + class complex2(complex): + pass + for x in 0.0, -0.0, INF, -INF, NAN: + for y in 0.0, -0.0, INF, -INF, NAN: + with self.subTest(x=x, y=y): + z = complex(x, y) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(x, y) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex(complex2(x, y)) + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(complex(x, y)) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: diff --git a/Misc/NEWS b/Misc/NEWS index 0fed99b5e7808f..d5bbd835aa21a6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31e12784cc34f9..cfaba688c684f6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1025,11 +1025,11 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } cr.real = PyFloat_AsDouble(tmp); - cr.imag = 0.0; /* Shut up compiler warning */ + cr.imag = 0.0; Py_DECREF(tmp); } if (i == NULL) { - ci.real = 0.0; + ci.real = cr.imag; } else if (PyComplex_Check(i)) { ci = ((PyComplexObject*)i)->cval; @@ -1051,7 +1051,7 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ci_is_complex) { cr.real -= ci.imag; } - if (cr_is_complex) { + if (cr_is_complex && i != NULL) { ci.real += cr.imag; } return complex_subtype_from_doubles(type, cr.real, ci.real); From 51a477c0d53b09d5e876c23288ad006ad64c1e97 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 21 Feb 2017 23:56:26 +0900 Subject: [PATCH 0024/1345] doc: fix compile error on "shoddy" example extension (GH-216) (cherry picked from commit fb8fe72fc593438f6a0b934c6ff2d9c4aa28673d) --- Doc/includes/setup.py | 1 + Doc/includes/shoddy.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py index b853d23b170985..a38a39de3e7c86 100644 --- a/Doc/includes/setup.py +++ b/Doc/includes/setup.py @@ -5,4 +5,5 @@ Extension("noddy2", ["noddy2.c"]), Extension("noddy3", ["noddy3.c"]), Extension("noddy4", ["noddy4.c"]), + Extension("shoddy", ["shoddy.c"]), ]) diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 07a272124ceaba..0c6d412b3c4cda 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -31,7 +31,7 @@ Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) static PyTypeObject ShoddyType = { - PyObject_HEAD_INIT(NULL) + PyVarObject_HEAD_INIT(NULL, 0) "shoddy.Shoddy", /* tp_name */ sizeof(Shoddy), /* tp_basicsize */ 0, /* tp_itemsize */ From e48fd93bbb36c6d80aa4eb6af09f58c69d8cf965 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 21 Feb 2017 18:18:27 +0200 Subject: [PATCH 0025/1345] bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. (#209) --- Lib/test/test_functools.py | 9 +++++++++ Misc/NEWS | 13 +++++++++++++ Modules/_functoolsmodule.c | 5 ++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 824549b80342ed..b7d648d0b15a55 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -89,6 +89,15 @@ def func(a=10, b=20): p(b=7) self.assertEqual(d, {'a':3}) + def test_kwargs_copy(self): + # Issue #29532: Altering a kwarg dictionary passed to a constructor + # should not affect a partial object after creation + d = {'a': 3} + p = self.partial(capture, **d) + self.assertEqual(p(), ((), {'a': 3})) + d['a'] = 5 + self.assertEqual(p(), ((), {'a': 3})) + def test_arg_combinations(self): # exercise special code paths for zero args in either partial # object or the caller diff --git a/Misc/NEWS b/Misc/NEWS index d5bbd835aa21a6..83b316c644d72b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,19 @@ Extension Modules Library ------- +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel Krebber, and Łukasz Langa. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f785a7260e6a2e..7abc9f464027f7 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -88,10 +88,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else { + else if (Py_REFCNT(kw) == 1) { Py_INCREF(kw); pto->kw = kw; } + else { + pto->kw = PyDict_Copy(kw); + } } else { pto->kw = PyDict_Copy(pkw); From 3ab24bdd47fdd9d45719ad49f93d3878d4442d7e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Feb 2017 02:33:24 +0900 Subject: [PATCH 0026/1345] bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX (GH-219) (cherry picked from commit 3a9ac827c7c87dffc60c4200323948551bcb6662) --- Misc/NEWS | 3 +++ Python/compile.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 83b316c644d72b..5688883375824e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + - bpo-29602: Fix incorrect handling of signed zeros in complex constructor for complex subclasses and for inputs having a __complex__ method. Patch by Serhiy Storchaka. diff --git a/Python/compile.c b/Python/compile.c index 0e1607585245b9..6255ec7d47f5a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1043,7 +1043,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case CALL_FUNCTION_KW: return -oparg-1; case CALL_FUNCTION_EX: - return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); + return -1 - ((oparg & 0x01) != 0); case MAKE_FUNCTION: return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); From 9b49133082ec23b67e84d2589e66d7810018e424 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 21 Feb 2017 10:30:07 -0800 Subject: [PATCH 0027/1345] bpo-29453: Remove reference to undefined dictionary ordering in Tutorial (GH-140) (#208) As of Python 3.6 **kwargs are ordered, thus, remove the paragraph stating that ordering is undefined and change snippet to remove the unecessary sorted call. * Add sentence mentioning guaranteed output order of kwargs (cherry picked from commit 32e8f9bdfd4324f1aa4fbbdf1ed8536f2b00cabb) --- Doc/tutorial/controlflow.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index d43461886e7933..6a9bb4889ff857 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -492,8 +492,7 @@ function like this:: for arg in arguments: print(arg) print("-" * 40) - keys = sorted(keywords.keys()) - for kw in keys: + for kw in keywords: print(kw, ":", keywords[kw]) It could be called like this:: @@ -513,13 +512,13 @@ and of course it would print: It's very runny, sir. It's really very, VERY runny, sir. ---------------------------------------- - client : John Cleese shopkeeper : Michael Palin + client : John Cleese sketch : Cheese Shop Sketch -Note that the list of keyword argument names is created by sorting the result -of the keywords dictionary's ``keys()`` method before printing its contents; -if this is not done, the order in which the arguments are printed is undefined. +Note that the order in which the keyword arguments are printed is guaranteed +to match the order in which they were provided in the function call. + .. _tut-arbitraryargs: From e9c0e5559bbadb164d7c57b5a47b5544746dcb89 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Tue, 21 Feb 2017 22:56:07 +0300 Subject: [PATCH 0028/1345] [3.6] Add .codecov.yml (#210) --- .codecov.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000000000..fcf9df6a7a698e --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,35 @@ +codecov: + notify: + require_ci_to_pass: true +comment: off +ignore: + - "Doc/**/*" + - "Misc/*" + - "Mac/**/*" + - "PC/**/*" + - "PCbuild/**/*" + - "Tools/**/*" + - "Grammar/*" +coverage: + precision: 2 + range: + - 70.0 + - 100.0 + round: down + status: + changes: off + project: off + patch: + default: + target: 100% + only_pulls: true + threshold: 0.05 +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false From b067a5eef7fdf69264d3578654996fc3755df4ea Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 22 Feb 2017 04:55:33 +0300 Subject: [PATCH 0029/1345] bpo-29554: Improve docs for pstat module and profile. (#88) (#227) Clarify that methods take a string which is interpreted as a regex, not a regex object. Also clarify what the old `-1`, `0`, `1` and `2` options were. (cherry picked from commit 8fb1f6e039cbdeb333d83b7a62f0f37af4ce6e02) --- Doc/library/profile.rst | 7 ++++--- Lib/pstats.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 959d9b98a84691..bd67fe486abf77 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -444,9 +444,10 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: + select a percentage of lines), or a string that will interpreted as a + regular expression (to pattern match the standard name that is printed). + If several restrictions are provided, then they are applied sequentially. + For example:: print_stats(.1, 'foo:') diff --git a/Lib/pstats.py b/Lib/pstats.py index d861413d4195f7..2c5bf981b85cf8 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -48,11 +48,14 @@ class Stats: printed. The sort_stats() method now processes some additional options (i.e., in - addition to the old -1, 0, 1, or 2). It takes an arbitrary number of - quoted strings to select the sort order. For example sort_stats('time', - 'name') sorts on the major key of 'internal function time', and on the - minor key of 'the name of the function'. Look at the two tables in - sort_stats() and get_sort_arg_defs(self) for more examples. + addition to the old -1, 0, 1, or 2 that are respectively interpreted as + 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number + of quoted strings to select the sort order. + + For example sort_stats('time', 'name') sorts on the major key of 'internal + function time', and on the minor key of 'the name of the function'. Look at + the two tables in sort_stats() and get_sort_arg_defs(self) for more + examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ From 3cc5817cfaf5663645f4ee447eaed603d2ad290a Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 22 Feb 2017 06:21:17 +0000 Subject: [PATCH 0030/1345] Fixed bpo-29565: Corrected ctypes passing of large structs by value on Windows AMD64. (#168) (#220) Fixed bpo-29565: Corrected ctypes passing of large structs by value. (cherry picked from commit a86339b83fbd0932e0529a3c91935e997a234582) --- Lib/ctypes/test/test_callbacks.py | 11 +++++++++++ Lib/ctypes/test/test_structures.py | 23 +++++++++++++++++++++++ Modules/_ctypes/_ctypes_test.c | 13 +++++++++++++ Modules/_ctypes/libffi_msvc/ffi.c | 10 ++++++++++ 4 files changed, 57 insertions(+) diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index 8eac58f0262caf..f622093df61da5 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -244,6 +244,7 @@ def callback(a, b, c, d, e): def test_callback_large_struct(self): class Check: pass + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c class X(Structure): _fields_ = [ ('first', c_ulong), @@ -255,6 +256,11 @@ def callback(check, s): check.first = s.first check.second = s.second check.third = s.third + # See issue #29565. + # The structure should be passed by value, so + # any changes to it should not be reflected in + # the value passed + s.first = s.second = s.third = 0x0badf00d check = Check() s = X() @@ -275,6 +281,11 @@ def callback(check, s): self.assertEqual(check.first, 0xdeadbeef) self.assertEqual(check.second, 0xcafebabe) self.assertEqual(check.third, 0x0bad1dea) + # See issue #29565. + # Ensure that the original struct is unchanged. + self.assertEqual(s.first, check.first) + self.assertEqual(s.second, check.second) + self.assertEqual(s.third, check.third) ################################################################ diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 8f6fe5f25429e5..3eded7749ed95e 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -3,6 +3,7 @@ from ctypes.test import need_symbol from struct import calcsize import _testcapi +import _ctypes_test class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -391,6 +392,28 @@ class Z(Y): (1, 0, 0, 0, 0, 0)) self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7)) + def test_pass_by_value(self): + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c + class X(Structure): + _fields_ = [ + ('first', c_ulong), + ('second', c_ulong), + ('third', c_ulong), + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + s.third = 0x0bad1dea + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_large_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + self.assertEqual(s.third, 0x0bad1dea) + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 629ddf66bc4212..59d56d0d218732 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -44,6 +44,19 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * See issue 29565. Update a structure passed by value; + * the caller should not see any change. + */ + +EXPORT(void) +_testfunc_large_struct_update_value(Test in) +{ + in.first = 0x0badf00d; + in.second = 0x0badf00d; + in.third = 0x0badf00d; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/libffi_msvc/ffi.c b/Modules/_ctypes/libffi_msvc/ffi.c index 1d82929f530220..91a27dce3f2575 100644 --- a/Modules/_ctypes/libffi_msvc/ffi.c +++ b/Modules/_ctypes/libffi_msvc/ffi.c @@ -239,6 +239,16 @@ ffi_call(/*@dependent@*/ ffi_cif *cif, break; #else case FFI_SYSV: + /* If a single argument takes more than 8 bytes, + then a copy is passed by reference. */ + for (unsigned i = 0; i < cif->nargs; i++) { + size_t z = cif->arg_types[i]->size; + if (z > 8) { + void *temp = alloca(z); + memcpy(temp, avalue[i], z); + avalue[i] = temp; + } + } /*@-usedef@*/ return ffi_call_AMD64(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, fn); From 55b82e10dc6b75b30a72fa56beb60eaf54a008d4 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 23 Feb 2017 16:17:11 +0100 Subject: [PATCH 0031/1345] bpo-28911: Clarify the behaviour of assert_called_once_with. (#252) (cherry picked from commit 9d56b34af2efc4e266bf3ae62da5cd2e422a42be) --- Doc/library/unittest.mock.rst | 11 ++++++----- Lib/unittest/mock.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c6d0ec92b6e9ab..a552cbfc70ad2e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -303,14 +303,14 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and with the specified - arguments. + Assert that the mock was called exactly once and that that call was + with the specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') + >>> mock('other', bar='values') + >>> mock.assert_called_once_with('other', bar='values') Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. @@ -322,7 +322,8 @@ the *new_callable* argument to :func:`patch`. The assert passes if the mock has *ever* been called, unlike :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one. + only pass if the call is the most recent one, and in the case of + :meth:`assert_called_once_with` it must also be the only call. >>> mock = Mock(return_value=None) >>> mock(1, 2, arg='thing') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6b38362341f16..5f97728de28c59 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -815,8 +815,8 @@ def _error_message(): def assert_called_once_with(_mock_self, *args, **kwargs): - """assert that the mock was called exactly once and with the specified - arguments.""" + """assert that the mock was called exactly once and that that call was + with the specified arguments.""" self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times." % From 2197eac6104311472f200645bc844adb46444b10 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 23 Feb 2017 18:42:30 +0300 Subject: [PATCH 0032/1345] bpo-28814: Undeprecate inadvertently deprecated inspect functions. (#122) (#243) Nick Coghlan said on bpo-28814: > inspect.getargvalues() and inspect.formatargvalues() were deprecated > in Python 3.5 as part of implementing bpo-20438 > This is incorrect, as these are *frame* introspection related functions, > not callable introspection ones. The documentation and implementation > layout is confusing though, as they're interleaved with the callable > introspection operation This commit undeprecates these functions and adds a note to ignore previous deprecation notices. (cherry picked from commit 0899b9809547ec2894dcf88cf4bba732c5d47d0d) --- Doc/library/inspect.rst | 12 ++++-------- Doc/whatsnew/3.5.rst | 11 ++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 41a784d982d6c2..3fa44a0c99891c 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -905,10 +905,8 @@ Classes and functions are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the locals dictionary of the given frame. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) @@ -944,10 +942,8 @@ Classes and functions :func:`getargvalues`. The format\* arguments are the corresponding optional formatting functions that are called to turn names and values into strings. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: getmro(cls) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index edb74f043e0b64..a6ba5bbb720ca2 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2327,11 +2327,12 @@ The :func:`inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getargvalues`, :func:`~inspect.getcallargs`, -:func:`~inspect.getargvalues`, :func:`~inspect.formatargspec`, and -:func:`~inspect.formatargvalues` functions are deprecated in favor of -the :func:`inspect.signature` API. -(Contributed by Yury Selivanov in :issue:`20438`.) +:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury +Selivanov in :issue:`20438`.) + +:func:`~inspect.getargvalues` and :func:`~inspect.formatargvalues` functions +were inadvertently marked as deprecated with the release of Python 3.5.0. Use of :const:`re.LOCALE` flag with str patterns or :const:`re.ASCII` is now deprecated. (Contributed by Serhiy Storchaka in :issue:`22407`.) From f2beceb7e54fadc507137f86cbe9d74f7e5b8f7c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 23 Feb 2017 15:03:20 -0800 Subject: [PATCH 0033/1345] bpo-29624: Adds purge step and layout test after uploading files. (#258) (#264) --- Tools/msi/uploadrelease.bat | 31 ++++++++++++++++++++++--------- Tools/msi/uploadrelease.proj | 32 +++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 4e319ce4c98ccb..670836be8c1c8b 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -9,6 +9,8 @@ set USER= set TARGET= set DRYRUN=false set NOGPG= +set PURGE_OPTION=/p:Purge=true +set NOTEST= :CheckOpts if "%1" EQU "-h" goto Help @@ -19,7 +21,11 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--no-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts +if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts +if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts +if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" @@ -35,7 +41,7 @@ echo Found pscp.exe at %PSCP% if defined NOGPG ( set GPG= - echo Skipping GPG signature generation because of --no-gpg + echo Skipping GPG signature generation because of --skip-gpg ) else ( if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" @@ -45,8 +51,12 @@ if defined NOGPG ( call "%PCBUILD%env.bat" > nul 2> nul pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOTEST ( + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 +) msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd @@ -55,9 +65,12 @@ exit /B 0 :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo -h Display this help information +echo --host (-o) Specify the upload host (required) +echo --user (-u) Specify the user on the host (required) +echo --target (-t) Specify the target directory on the host +echo --dry-run Display commands and filenames without executing them +echo --skip-gpg Does not generate GPG signatures before uploading +echo --skip-purge Does not perform CDN purge after uploading +echo --skip-test (-T) Does not perform post-upload tests +echo -h Display this help information echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 0d472dea5428a0..305e84fc2d4b28 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -9,6 +9,7 @@ /srv/www.python.org/ftp/python true false + false @@ -64,7 +65,36 @@ echo. echo." /> - + + + + + + + + + + $(TEMP)\%(Filename)_source + $(TEMP)\%(Filename)_source\%(Filename)%(Extension) + $(TEMP)\%(Filename)_layout + $(OutputPath)\%(Filename)_layoutlog + $(OutputPath)\%(Filename)_layoutlog\%(Filename).log + + + + + + + + + + + + + + + From 0a1b656d8ce3da14f8acf947477b8e998e68ef3b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 24 Feb 2017 15:44:34 -0500 Subject: [PATCH 0034/1345] bpo-25008: Deprecate smtpd and point to aiosmtpd (#274) (#278) --- Doc/library/smtpd.rst | 6 ++++++ Misc/NEWS | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index e383201aab1461..85ee8a75cf77a9 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -13,6 +13,12 @@ This module offers several classes to implement SMTP (email) servers. +.. seealso:: + + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. :mod:`smtpd` should be considered deprecated. + Several server implementations are present; one is a generic do-nothing implementation, which can be overridden, while the other two offer specific mail-sending strategies. diff --git a/Misc/NEWS b/Misc/NEWS index 5688883375824e..d885ca900217ea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,7 +10,7 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. - bpo-29602: Fix incorrect handling of signed zeros in complex constructor for @@ -217,6 +217,9 @@ C API Documentation ------------- +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier. From f28db60179e59633a99fab27646d22d0bd5976a0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Feb 2017 16:39:30 -0800 Subject: [PATCH 0035/1345] [3.6] Fix small typos in expressions.rst (GH-276) (GH-281) (cherry picked from commit 132ac381fe7eb593e3d7c63926f90285bba13cdc) --- Doc/reference/expressions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3a4b80557caf67..f4a82699b0be12 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -190,7 +190,7 @@ Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading -expression, may contan additonal :keyword:`for` or :keyword:`async for` +expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. If a comprehension contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -1317,7 +1317,7 @@ built-in types. * Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can be compared only within each of their types, with the restriction that ranges do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises + results in inequality, and ordering comparison across these types raises :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding From bea9d2f6488f01794098d9fd4710b98df1bd9472 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Feb 2017 16:40:50 -0800 Subject: [PATCH 0036/1345] [3.6] bpo-28556: Update to typing: treat subscripted generics as proxies (GH-265) (GH-268) (cherry picked from commit abb3b8ad94d699c8560d94ee9bac9c917b382abe) (cherry picked from commit 365cb5bb9069273e6970c9d5d17ee2fe5003e7ac) --- Lib/test/test_typing.py | 35 +++++++++++++++++++++++++++++++++++ Lib/typing.py | 10 ++++++++++ 2 files changed, 45 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 64d8276658ee7f..f0070ec975791a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -674,6 +674,41 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_subscripted_generics_as_proxies(self): + T = TypeVar('T') + class C(Generic[T]): + x = 'def' + self.assertEqual(C[int].x, 'def') + self.assertEqual(C[C[int]].x, 'def') + C[C[int]].x = 'changed' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[str].x, 'changed') + C[List[str]].z = 'new' + self.assertEqual(C.z, 'new') + self.assertEqual(C[Tuple[int]].z, 'new') + + self.assertEqual(C().x, 'changed') + self.assertEqual(C[Tuple[str]]().z, 'new') + + class D(C[T]): + pass + self.assertEqual(D[int].x, 'changed') + self.assertEqual(D.z, 'new') + D.z = 'from derived z' + D[int].x = 'from derived x' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[int].z, 'new') + self.assertEqual(D.x, 'from derived x') + self.assertEqual(D[str].z, 'from derived z') + + def test_abc_registry_kept(self): + T = TypeVar('T') + class C(Generic[T]): ... + C.register(int) + self.assertIsInstance(1, C) + C[int] + self.assertIsInstance(1, C) + def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass self.assertNotIsInstance({}, MyMapping) diff --git a/Lib/typing.py b/Lib/typing.py index efe358faf20988..9a0f49099a3114 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1158,6 +1158,16 @@ def __copy__(self): self.__parameters__, self.__args__, self.__origin__, self.__extra__, self.__orig_bases__) + def __setattr__(self, attr, value): + # We consider all the subscripted genrics as proxies for original class + if ( + attr.startswith('__') and attr.endswith('__') or + attr.startswith('_abc_') + ): + super(GenericMeta, self).__setattr__(attr, value) + else: + super(GenericMeta, _gorg(self)).__setattr__(attr, value) + # Prevent checks for Generic to crash when defining Generic. Generic = None From 814213bdf1b95a5d863311b61c671010291f1197 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 11:56:48 -0800 Subject: [PATCH 0037/1345] [3.6] Backport README.rst changes (GH-294) (cherry picked from commit b3f1f59cf451d4a25b204e7a24f7be4c95e40be8) (cherry picked from commit 1b3d88eb33085e90af729c4c2f78b5ba1b942b1e) (cherry picked from commit c0866855d54bbbe7bcac299c4af45389efacce0b) (cherry picked from commit e32ec9334b35f897ace8a05128838f92c5e0b2fb) (cherry picked from commit 6329aea78e3627d250009aa5aaaf47cd11587f4d) --- Doc/Makefile | 2 +- Doc/{README.txt => README.rst} | 21 +++++++++++---------- Doc/conf.py | 2 +- Doc/make.bat | 2 +- README.rst | 30 +++++++++++++++++++++--------- 5 files changed, 35 insertions(+), 22 deletions(-) rename Doc/{README.txt => README.rst} (83%) diff --git a/Doc/Makefile b/Doc/Makefile index eb62df5b87cebe..94697f95671894 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -153,7 +153,7 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools -i venv + $(PYTHON) tools/rstlint.py -i tools -i venv -i README.rst serve: ../Tools/scripts/serve.py build/html diff --git a/Doc/README.txt b/Doc/README.rst similarity index 83% rename from Doc/README.txt rename to Doc/README.rst index 4f8e9f8f1417fb..dcd3d6e80ff3c4 100644 --- a/Doc/README.txt +++ b/Doc/README.rst @@ -2,20 +2,21 @@ Python Documentation README ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, prebuilt versions are -available at . +documentation. You don't need to build them yourself, `prebuilt versions are +available `_. Documentation on authoring Python documentation, including information about -both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +both style and markup, is available in the "`Documenting Python +`_" chapter of the +developers guide. Building the docs ================= -You need to have Sphinx installed; it is the toolset +You need to have `Sphinx `_ installed; it is the toolset used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +separately and `available from PyPI `_. Using make @@ -108,11 +109,11 @@ see the make targets above). Contributing ============ -Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +Bugs in the content should be reported to the +`Python bug tracker `_. -Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +Bugs in the toolset should be reported in the +`Sphinx bug tracker `_. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. diff --git a/Doc/conf.py b/Doc/conf.py index b3f26d5a692163..18aebb68a8d8df 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,7 @@ needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] +exclude_patterns = ['venv/*', 'README.rst'] # Options for HTML output diff --git a/Doc/make.bat b/Doc/make.bat index d0b59618261011..b1a3738f309d3b 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -74,7 +74,7 @@ echo. Provided by this script: echo. clean, check, serve, htmlview echo. echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or +echo.filenames to build or are ignored. See README.rst in this directory or echo.the documentation for your version of Sphinx for more exhaustive lists echo.of available targets and descriptions of each. echo. diff --git a/README.rst b/README.rst index d02ae64b7fc660..242572c5e361db 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,22 @@ reserved. See the end of this file for further copyright and license information. +General Information +------------------- + +- Website: https://www.python.org +- Source code: https://github.com/python/cpython +- Issue tracker: https://bugs.python.org +- Documentation: https://docs.python.org +- Developer's Guide: https://docs.python.org/devguide/ + +Contributing to CPython +----------------------- + +For more complete instructions on contributing to CPython development, +see the `Developer Guide`_. + +.. _Developer Guide: https://docs.python.org/devguide/ Using Python ------------ @@ -28,7 +44,7 @@ Installable Python kits, and information about using Python, are available at Build Instructions ------------------ -On Unix, Linux, BSD, OSX, and Cygwin:: +On Unix, Linux, BSD, macOS, and Cygwin:: ./configure make @@ -38,10 +54,10 @@ On Unix, Linux, BSD, OSX, and Cygwin:: This will install Python as python3. You can pass many options to the configure script; run ``./configure --help`` -to find out more. On OSX and Cygwin, the executable is called ``python.exe``; +to find out more. On macOS and Cygwin, the executable is called ``python.exe``; elsewhere it's just ``python``. -On Mac OS X, if you have configured Python with ``--enable-framework``, you +On macOS, if you have configured Python with ``--enable-framework``, you should use ``make frameworkinstall`` to do the installation. Note that this installs the Python executable in a place that is not normally on your PATH, you may want to set up a symlink in ``/usr/local/bin``. @@ -126,12 +142,8 @@ is downloadable in HTML, PDF, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. -If you would like to contribute to the development of Python, relevant -documentation is available in the `Python Developer's Guide -`_. - -For information about building Python's documentation, refer to `Doc/README.txt -`_. +For information about building Python's documentation, refer to `Doc/README.rst +`_. Converting From Python 2.x to 3.x From 1a8dd944d4016593e78a7a6c3c635adee783abfd Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 22:34:06 -0800 Subject: [PATCH 0038/1345] [3.6] bpo-28929: Add to Misc/NEWS (GH-284) mention bpo-28929 in the Documentation section of What's New in Python 3.6.1 release candidate 1 --- Misc/NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index d885ca900217ea..0f5c27902ba423 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -217,6 +217,8 @@ C API Documentation ------------- +- bpo-28929: Link the documentation to its source file on GitHub. + - bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to aiosmtpd, a third-party asyncio-based replacement. From b00232842c5dd0d41054a02a7800fc5f82a92beb Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 22:34:51 -0800 Subject: [PATCH 0039/1345] Fix small typos in introduction and datastructures of tutorial (GH-272) (GH-297) (cherry picked from commit 53c1892dc3de1de612b1cf95dc7bf09f82c1babf) --- Doc/tutorial/datastructures.rst | 8 ++++---- Doc/tutorial/introduction.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 953a68b44af7bb..6140ece046b975 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -22,11 +22,11 @@ objects: Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. -.. method:: list.extend(L) +.. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items in the given list. Equivalent to - ``a[len(a):] = L``. + Extend the list by appending all the items from the iterable. Equivalent to + ``a[len(a):] = iterable``. .. method:: list.insert(i, x) @@ -68,7 +68,7 @@ objects: The optional arguments *start* and *end* are interpreted as in the slice notation and are used to limit the search to a particular subsequence of - *x*. The returned index is computed relative to the beginning of the full + the list. The returned index is computed relative to the beginning of the full sequence rather than the *start* argument. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 7e8ee3e5ea19bc..52120a0b16e75f 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -359,7 +359,7 @@ The built-in function :func:`len` returns the length of a string:: Information about string formatting with :meth:`str.format`. :ref:`old-string-formatting` - The old formatting operations invoked when strings and Unicode strings are + The old formatting operations invoked when strings are the left operand of the ``%`` operator are described in more detail here. From 91c499f8f94b89c08686ac9e7c81ea1d89221cbc Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 26 Feb 2017 00:00:09 -0800 Subject: [PATCH 0040/1345] mark various test data binary (#233) (#304) --- .gitattributes | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000000..55a3f49922bf19 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +Misc/NEWS merge=union + +*.pck binary +Lib/test/cjkencodings/* binary +Lib/test/decimaltestdata/*.decTest binary +Lib/test/sndhdrdata/sndhdr.* binary +Lib/test/test_email/data/msg_26.txt binary +Lib/test/xmltestdata/* binary +Lib/venv/scripts/nt/* binary +Lib/test/coding20731.py binary From fae59e1aa87ee9598d032e0bd697969a5784025f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 16:05:30 +0300 Subject: [PATCH 0041/1345] bpo-28961: Address my comments from earlier code review (#306) (cherry picked from commit dea1536fd3a8424d537794cd53715df0989cbbe1) --- Lib/unittest/test/testmock/testhelpers.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d5f9e7c1d69204..d2202a7b4132e9 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -307,18 +307,9 @@ def test_two_args_call(self): self.assertEqual(args, other_args) def test_call_with_name(self): - self.assertEqual( - 'foo', - _Call((), 'foo')[0], - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), ), )[0] - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), {'hello': 'world'}), )[0] - ) + self.assertEqual(_Call((), 'foo')[0], 'foo') + self.assertEqual(_Call((('bar', 'barz'),),)[0], '') + self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '') class SpecSignatureTest(unittest.TestCase): From 5c7b8f423ab1de6221351036ac4e139861a2f4a6 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 16:07:12 +0300 Subject: [PATCH 0042/1345] Backport recent .travis.yml changes (#308) Backported changes from master: * b52260d8bf392aa04c48b8c2467a4c034184de86 * 984eef7d6d78e1213d6ea99897343a5059a07c59 * 532519770dea5d353f0b0d718c8881a15c7542df * 91b0e7d0ca7c59df28f6a6fc1e8eb86a3925b76c --- .travis.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc3a00d1f4b5a3..27b63c6c08b376 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,11 @@ group: beta # To cache doc-building dependencies. cache: pip +branches: + only: + - master + - /^\d\.\d$/ + os: - linux # macOS builds are disabled as the machines are under-provisioned on Travis, @@ -20,6 +25,7 @@ env: - TESTING=cpython matrix: + fast_finish: true allow_failures: - env: - TESTING=coverage @@ -30,17 +36,10 @@ matrix: env: - TESTING=docs before_script: - - | - if git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '^Doc/' - then - echo "Docs weren't updated, stopping build process." - exit - fi - cd Doc - make venv PYTHON=python3 + - cd Doc + - make venv script: - - make html SPHINXBUILD="./venv/bin/python3 -m sphinx" SPHINXOPTS="-q" - - make check + - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q" - os: linux language: c compiler: clang From 7333d1760e12cf27bcf63265f72521892285c10a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 07:33:54 -0800 Subject: [PATCH 0043/1345] bpo-29648: import.rst: Add reference to create_module() (GH-290) (GH-314) (cherry picked from commit 46ce7599af82a929506baeaaee5c149970440c4c) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 5e2c1c8b0758d8..b603d1f978a7a6 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -431,7 +431,7 @@ on the module object. If the method returns ``None``, the import machinery will create the new module itself. .. versionadded:: 3.4 - The create_module() method of loaders. + The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 The :meth:`~importlib.abc.Loader.load_module` method was replaced by From 1457984c003b1e461f8aab93bfc37fd8e42312d5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 07:44:16 -0800 Subject: [PATCH 0044/1345] bpo-22594: Add a link to the regex module in re documentation (GH-241) (GH-316) (cherry picked from commit ed6795e46f7653e23b862efad240a93453e7df97) --- Doc/library/re.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe5ebcc67c840b..9cced512706554 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -42,6 +42,12 @@ module-level functions and methods on that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + The third-party `regex `_ module, + which has an API compatible with the standard library :mod:`re` module, + but offers additional functionality and a more thorough Unicode support. + .. _re-syntax: From 893e86e9d3c0caeb878ccb1120c7259e022f3b68 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 19:04:00 +0300 Subject: [PATCH 0045/1345] bpo-29121: Remove outdated documentation about transactions (#313) (#319) Patch by Aviv Palivoda. (cherry picked from commit fe70d924bb6106d4c21eb414f4a1ba1324e8f46a) --- Doc/library/sqlite3.rst | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5635577da0f946..d1f7a6f120620b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -927,14 +927,6 @@ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. ``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -So if you are within a transaction and issue a command like ``CREATE TABLE -...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly -before executing that command. There are two reasons for doing that. The first -is that some of these commands don't work within transactions. The other reason -is that sqlite3 needs to keep track of the transaction state (if a transaction -is active or not). The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. - You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes (or none at all) via the *isolation_level* parameter to the :func:`connect` call, or via the :attr:`isolation_level` property of connections. @@ -945,6 +937,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +The current transaction state is exposed through the +:attr:`Connection.in_transaction` attribute of the connection object. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. From 76995cab69d5ef83d31d8a5754cbad11be4038cb Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 19:09:10 +0300 Subject: [PATCH 0046/1345] bpo-28518: Start a transaction implicitly before a DML statement (#245) (#318) Patch by Aviv Palivoda. (cherry picked from commit 4a926caf8e5fd8af771b2c34bfb6e91c732331fe) --- Lib/sqlite3/test/transactions.py | 9 +++++++++ Misc/NEWS | 3 +++ Modules/_sqlite/cursor.c | 9 ++++----- Modules/_sqlite/statement.c | 12 +++++++----- Modules/_sqlite/statement.h | 2 +- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py index 45f1b04c69648f..b8a13de55bc720 100644 --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -179,6 +179,15 @@ def CheckDdlDoesNotAutostartTransaction(self): result = self.con.execute("select * from test").fetchall() self.assertEqual(result, []) + def CheckImmediateTransactionalDDL(self): + # You can achieve transactional DDL by issuing a BEGIN + # statement manually. + self.con.execute("begin immediate") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + def CheckTransactionalDDL(self): # You can achieve transactional DDL by issuing a BEGIN # statement manually. diff --git a/Misc/NEWS b/Misc/NEWS index 0f5c27902ba423..a48debdc20ecff 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,9 @@ Extension Modules Library ------- +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + - bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 39f7a6508c2aef..8341fb8480172c 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -511,10 +511,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - /* For backwards compatibility reasons, do not start a transaction if a - DDL statement is encountered. If anybody wants transactional DDL, - they can issue a BEGIN statement manually. */ - if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + /* We start a transaction implicitly before a DML statement. + SELECT is the only exception. See #9924. */ + if (self->connection->begin_statement && self->statement->is_dml) { if (sqlite3_get_autocommit(self->connection->db)) { result = _pysqlite_connection_begin(self->connection); if (!result) { @@ -609,7 +608,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } } - if (!sqlite3_stmt_readonly(self->statement->st)) { + if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } else { self->rowcount= -1L; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 0df661b9c7ad31..087375be9b63d8 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -73,8 +73,9 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con Py_INCREF(sql); self->sql = sql; - /* determine if the statement is a DDL statement */ - self->is_ddl = 0; + /* Determine if the statement is a DML statement. + SELECT is the only exception. See #9924. */ + self->is_dml = 0; for (p = sql_cstr; *p != 0; p++) { switch (*p) { case ' ': @@ -84,9 +85,10 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con continue; } - self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0) - || (PyOS_strnicmp(p, "drop ", 5) == 0) - || (PyOS_strnicmp(p, "reindex ", 8) == 0); + self->is_dml = (PyOS_strnicmp(p, "insert ", 7) == 0) + || (PyOS_strnicmp(p, "update ", 7) == 0) + || (PyOS_strnicmp(p, "delete ", 7) == 0) + || (PyOS_strnicmp(p, "replace ", 8) == 0); break; } diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 6eef16857f7b85..8db10f6649ce80 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -38,7 +38,7 @@ typedef struct sqlite3_stmt* st; PyObject* sql; int in_use; - int is_ddl; + int is_dml; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; From deea29e61e61f0e216bff3f0ca008f5ee231793f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 08:55:23 -0800 Subject: [PATCH 0047/1345] bpo-26184: import.rst: Improve versionchanged note (GH-277) (#320) (cherry picked from commit 6b4a5f45e2df524174a97832571c82c76a3d424a) --- Doc/reference/import.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index b603d1f978a7a6..484457a8a5bb5c 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -464,8 +464,11 @@ import machinery will create the new module itself. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. + ``create_module()`` is not. + +.. versionchanged:: 3.6 + An :exc:`ImportError` is raised when ``exec_module()`` is defined but + ``create_module`` is not. Submodules ---------- From 6b81003bdbd9375886bae54f876650bcdccfe6c7 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 20:38:31 +0300 Subject: [PATCH 0048/1345] bpo-28624: Add a test that checks that cwd parameter of Popen() accepts PathLike objects (#157) (#323) (cherry picked from commit d5c11f7ace48701bb950c6345deee88c35c66e26) --- Doc/library/subprocess.rst | 10 +++++++--- Lib/test/test_subprocess.py | 10 ++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad2abe82453622..548e4a68cbf1d1 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -466,9 +466,13 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + *cwd* before executing the child. *cwd* can be a :class:`str` and + :term:`path-like ` object. In particular, the function + looks for *executable* (or for the first item in *args*) relative to *cwd* + if the executable path is a relative path. + + .. versionchanged:: 3.6 + *cwd* parameter accepts a :term:`path-like object`. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e63f9f254cd2b2..3c871dde5c532f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -347,6 +347,16 @@ def test_cwd(self): temp_dir = self._normalize_cwd(temp_dir) self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): # Check that Popen looks for args[0] relative to cwd if args[0] diff --git a/Misc/ACKS b/Misc/ACKS index 1d6e773b553c52..c3b29a42a34f75 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -269,6 +269,7 @@ Albert Chin-A-Young Adal Chiriliuc Matt Chisholm Lita Cho +Sayan Chowdhury Anders Chrigström Tom Christiansen Renee Chu diff --git a/Misc/NEWS b/Misc/NEWS index a48debdc20ecff..dfafe7b081adae 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,9 @@ Extension Modules Library ------- +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + - bpo-28518: Start a transaction implicitly before a DML statement. Patch by Aviv Palivoda. From c9131b61fa060a51ec181053cade9f0a7ee91e4f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 27 Feb 2017 04:09:32 +0900 Subject: [PATCH 0049/1345] [3.6] bpo-29110: Fix file object leak in `aifc.open` (#310) (cherry picked from commit 03f68b60e17b57f6f13729ff73245dbb37b30a4c) (GH-162) (cherry picked from commit 5dc33eea538361f8a218255f83db2e9298dd8c53) (GH-293) --- Lib/aifc.py | 37 +++++++++++++++++++++++++------------ Lib/test/test_aifc.py | 18 +++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 692d0bfd272bf0..13ad7dc5ca3d62 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -303,6 +303,8 @@ class Aifc_read: # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file + _file = None # Set here since __del__ checks it + def initfp(self, file): self._version = 0 self._convert = None @@ -344,9 +346,15 @@ def initfp(self, file): def __init__(self, f): if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) + file_object = builtins.open(f, 'rb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + else: + # assume it is an open file object already + self.initfp(f) def __enter__(self): return self @@ -541,18 +549,23 @@ class Aifc_write: # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written + _file = None # Set here since __del__ checks it + def __init__(self, f): if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') - else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 + file_object = builtins.open(f, 'wb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + + # treat .aiff file extensions as non-compressed audio + if f.endswith('.aiff'): + self._aifc = 0 else: - self._aifc = 1 + # assume it is an open file object already + self.initfp(f) def initfp(self, file): self._file = file diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 1bd1f89c8aa609..a731a5136ba5fd 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -1,5 +1,6 @@ -from test.support import findfile, TESTFN, unlink +from test.support import check_no_resource_warning, findfile, TESTFN, unlink import unittest +from unittest import mock from test import audiotests from audioop import byteswap import io @@ -149,6 +150,21 @@ def test_skipunknown(self): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_close_opened_files_on_error(self): + non_aifc_file = findfile('pluck-pcm8.wav', subdir='audiodata') + with check_no_resource_warning(self): + with self.assertRaises(aifc.Error): + # Try opening a non-AIFC file, with the expectation that + # `aifc.open` will fail (without raising a ResourceWarning) + self.f = aifc.open(non_aifc_file, 'rb') + + # Aifc_write.initfp() won't raise in normal case. But some errors + # (e.g. MemoryError, KeyboardInterrupt, etc..) can happen. + with mock.patch.object(aifc.Aifc_write, 'initfp', + side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.fout = aifc.open(TESTFN, 'wb') + def test_params_added(self): f = self.f = aifc.open(TESTFN, 'wb') f.aiff() diff --git a/Misc/NEWS b/Misc/NEWS index dfafe7b081adae..5fad3c82c1a3a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -78,6 +78,9 @@ Library - bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + - bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from the platform about whether generated UUIDs are generated with a multiprocessing safe method. From 7e4897a2fb91e49f131a42ed6de88b5185f7dea8 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 13:30:44 -0800 Subject: [PATCH 0050/1345] bpo-26184: import.rst: Improve versionchanged note (GH-325) (GH-326) (cherry picked from commit 1f5639c77f736c18fb5a85b4a1850121e25c788e) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 484457a8a5bb5c..4dbd9d88c301a0 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -468,7 +468,7 @@ import machinery will create the new module itself. .. versionchanged:: 3.6 An :exc:`ImportError` is raised when ``exec_module()`` is defined but - ``create_module`` is not. + ``create_module()`` is not. Submodules ---------- From 4b6c41768a15fc85e3069603ef89344bd97f79af Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 27 Feb 2017 11:45:42 +0800 Subject: [PATCH 0051/1345] bpo-29376: Fix assertion error in threading._DummyThread.is_alive() (GH-330) --- Lib/test/test_threading.py | 3 +++ Lib/threading.py | 4 ++++ Misc/NEWS | 2 ++ 3 files changed, 9 insertions(+) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2c2914fd6d8969..6b6c4d220a3bd0 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -170,6 +170,9 @@ def f(mutex): mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) diff --git a/Lib/threading.py b/Lib/threading.py index 4829ff426e0bfd..95978d310a2f3a 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1217,6 +1217,10 @@ def __init__(self): def _stop(self): pass + def is_alive(self): + assert not self._is_stopped and self._started.is_set() + return True + def join(self, timeout=None): assert False, "cannot join a dummy thread" diff --git a/Misc/NEWS b/Misc/NEWS index 5fad3c82c1a3a0..c47657fb28357f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,8 @@ Extension Modules Library ------- +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + - bpo-28624: Add a test that checks that cwd parameter of Popen() accepts PathLike objects. Patch by Sayan Chowdhury. From 8aa1ad892ff3cf4d8881cdae0e95be1a5bad175e Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 27 Feb 2017 13:55:07 +0800 Subject: [PATCH 0052/1345] bpo-29662: fix wrong indentation in timeit.Timer's doc (GH-333) --- Doc/library/timeit.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 3b772765aca260..5793c54ae8d1a3 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -134,21 +134,21 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None) - Automatically determine how many times to call :meth:`.timeit`. + Automatically determine how many times to call :meth:`.timeit`. - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. + This is a convenience function that calls :meth:`.timeit` repeatedly + so that the total time >= 0.2 second, returning the eventual + (number of loops, time taken for that number of loops). It calls + :meth:`.timeit` with *number* set to successive powers of ten (10, + 100, 1000, ...) up to a maximum of one billion, until the time taken + is at least 0.2 second, or the maximum is reached. - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. + If *callback* is given and is not ``None``, it will be called after + each trial with two arguments: ``callback(number, time_taken)``. - .. versionadded:: 3.6 + .. versionadded:: 3.6 .. method:: Timer.repeat(repeat=3, number=1000000) From bc10e6bd7b6b4a738ce327e31550048d3dd8d8bd Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 27 Feb 2017 06:02:21 -0800 Subject: [PATCH 0053/1345] Asyncio documentation: remove `self` from method signatures (GH-334) (GH-337) (cherry picked from commit 091b84f23a2ff57e8320ebf6fdf889af39096ab9) --- Doc/library/asyncio-protocol.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3fbf51058673d9..482ffbbed35b1e 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -36,7 +36,7 @@ BaseTransport Base class for transports. - .. method:: close(self) + .. method:: close() Close the transport. If the transport has a buffer for outgoing data, buffered data will be flushed asynchronously. No more data @@ -44,7 +44,7 @@ BaseTransport protocol's :meth:`connection_lost` method will be called with :const:`None` as its argument. - .. method:: is_closing(self) + .. method:: is_closing() Return ``True`` if the transport is closing or is closed. @@ -251,7 +251,7 @@ BaseSubprocessTransport if it hasn't returned, similarly to the :attr:`subprocess.Popen.returncode` attribute. - .. method:: kill(self) + .. method:: kill() Kill the subprocess, as in :meth:`subprocess.Popen.kill`. From 046041e23b3c7e0b4a08705e2792ef3e28d03384 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 17:26:10 +0300 Subject: [PATCH 0054/1345] Delete accidentally merged entries from Misc/NEWS (#338) --- Misc/NEWS | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c47657fb28357f..7ec6b1291a3d2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -83,16 +83,6 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel Krebber, and Łukasz Langa. From 53039ad3814a8918c5311f37bd654428b9843fcc Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 27 Feb 2017 16:08:01 +0000 Subject: [PATCH 0055/1345] bpo-28598: Support __rmod__ for RHS subclasses of str in % string formatting operations (#95) --- Lib/test/test_unicode.py | 9 +++++++++ Misc/NEWS | 3 +++ Python/ceval.c | 12 +++++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 86ebd45e7b031d..2844bc5540c38b 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1448,6 +1448,15 @@ def test_formatting_huge_precision(self): with self.assertRaises(ValueError): result = format_string % 2.34 + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + @support.cpython_only def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX diff --git a/Misc/NEWS b/Misc/NEWS index 7ec6b1291a3d2d..fe420b62f231dd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- Issue #28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. diff --git a/Python/ceval.c b/Python/ceval.c index d5172b9631f43d..c9e9c327f31401 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1409,9 +1409,15 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(BINARY_MODULO) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *res = PyUnicode_CheckExact(dividend) ? - PyUnicode_Format(dividend, divisor) : - PyNumber_Remainder(dividend, divisor); + PyObject *res; + if (PyUnicode_CheckExact(dividend) && ( + !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { + // fast path; string formatting, but not if the RHS is a str subclass + // (see issue28598) + res = PyUnicode_Format(dividend, divisor); + } else { + res = PyNumber_Remainder(dividend, divisor); + } Py_DECREF(divisor); Py_DECREF(dividend); SET_TOP(res); From bb59d89ceeb1abfb4d73c7fc60b534e4464adf35 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 19:14:11 +0300 Subject: [PATCH 0056/1345] bpo-27788 : synchronise platform.py version number (#246) (#341) Was bumped in the docstring by b9f4feab1b9c9ffa8ea29af3d82bc536f9f3005a but not in `__version__` (cherry picked from commit 6059ce45aa96f52fa0150e68ea655fbfdc25609a) --- Lib/platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/platform.py b/Lib/platform.py index e48ad0b6e7bdea..062347537531ba 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -110,7 +110,7 @@ """ -__version__ = '1.0.7' +__version__ = '1.0.8' import collections import sys, os, re, subprocess From 7accf2033d03025cc5324f8b9d22582bca3623e9 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 20:41:21 +0300 Subject: [PATCH 0057/1345] bpo-29655: Fixed possible reference leaks in `import *`. (#301) (#348) Patch by Matthias Bussonnier. (cherry picked from commit 160edb43571311a3785785c1dfa784afc52d87be) --- Python/ceval.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index c9e9c327f31401..83667353af232c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2857,13 +2857,16 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) + if (PyFrame_FastToLocalsWithError(f) < 0) { + Py_DECREF(from); goto error; + } locals = f->f_locals; if (locals == NULL) { PyErr_SetString(PyExc_SystemError, "no locals found during 'import *'"); + Py_DECREF(from); goto error; } err = import_all_from(locals, from); From c4a786b8ad2a625594551ab3bc991a6831ba7688 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 28 Feb 2017 11:28:44 +0800 Subject: [PATCH 0058/1345] bpo-29661: fix contradiction in timeit.Timer.autorange's docstring (GH-331) (GH-353) --- Lib/timeit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 Lib/timeit.py diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100644 new mode 100755 index 2770efa35a0160..8eea766b4c344b --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -208,7 +208,7 @@ def repeat(self, repeat=default_repeat, number=default_number): return r def autorange(self, callback=None): - """Return the number of loops so that total time >= 0.2. + """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with *number* set to successive powers of ten (10, 100, 1000, ...) up to a maximum of one billion, until From d413aa7893fd8ac64693c22deb2c1eee1840dbd7 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 28 Feb 2017 23:19:58 -0500 Subject: [PATCH 0059/1345] Disable mention-bot for maintenance branches (GH-365) --- .mention-bot | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .mention-bot diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 00000000000000..cb53b993fb1e8a --- /dev/null +++ b/.mention-bot @@ -0,0 +1,3 @@ +{ + "findPotentialReviewers": false +} From 8458b2b4ef06805d65c9e253433727e60820992e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Feb 2017 21:07:32 -0800 Subject: [PATCH 0060/1345] correct documentation for enum.html (GH-358) (GH-367) (cherry picked from commit 626584284e74a68fff8157f9afe77b3088ff7be9) --- Doc/library/enum.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5cd6472f3e2f45..6548adf789da17 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -24,8 +24,8 @@ Module Contents --------------- This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, and -:class:`IntFlags`. It also defines one decorator, :func:`unique`, and one +sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and +:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one helper, :class:`auto`. .. class:: Enum From 1b93ed4f3ea5a4ac9786f15870e9b2812ba87cb1 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Feb 2017 21:08:13 -0800 Subject: [PATCH 0061/1345] Fixed a handful of typos (GH-343) (GH-364) (cherry picked from commit 1cf2a809b1eb4f9fc8ac3ccc97424586892d1e1a) --- Doc/library/email.compat32-message.rst | 4 ++-- Doc/library/email.message.rst | 2 +- Doc/library/idle.rst | 2 +- Doc/library/ssl.rst | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 2c65079ebdc3f1..afb5df5d428929 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -37,7 +37,7 @@ generating a serialized version of the mssage, and for recursively walking over the object tree. Note that duplicate headers are supported but special methods must be used to access them. -The :class:`Message` psuedo-dictionary is indexed by the header names, which +The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are supposed to contain only ASCII characters; there is some special handling for non-ASCII input, but it doesn't always produce the correct results. Headers @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: This is a legacy method. On the :class:`~email.emailmessage.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the - realted ``make`` and ``add`` methods. + related ``make`` and ``add`` methods. .. method:: get_payload(i=None, decode=False) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 32852e706986b4..d36e769f763cd8 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -364,7 +364,7 @@ message objects. *header* specifies an alternative header to :mailheader:`Content-Type`. If the value contains non-ASCII characters, the charset and language may - be explicity specified using the optional *charset* and *language* + be explicitly specified using the optional *charset* and *language* parameters. Optional *language* specifies the :rfc:`2231` language, defaulting to the empty string. Both *charset* and *language* should be strings. The default is to use the ``utf8`` *charset* and ``None`` for diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a629bc50dbc749..07c2a25d6a012c 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -446,7 +446,7 @@ longer or disable the extension. Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *acccessible* +A calltip is shown when one types :kbd:`(` after the name of an *accessible* function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or :kbd:`)` is typed. When the cursor is in the argument part of a definition, diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 07b97715a985b5..7b76b2a0064bf8 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -616,7 +616,7 @@ Constants .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -948,7 +948,7 @@ SSL Sockets :ref:`notes on non-blocking sockets `. Usually, :class:`SSLSocket` are not created directly, but using the - the :meth:`SSLContext.wrap_socket` method. + :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 The :meth:`sendfile` method was added. From 023532e558bb0c5bf60195aebbafe63a0bebd85e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 1 Mar 2017 21:14:43 +0900 Subject: [PATCH 0062/1345] bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. (GH-378) --- Misc/NEWS | 4 ++++ Python/ceval.c | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index fe420b62f231dd..33559a7abb8ce1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. diff --git a/Python/ceval.c b/Python/ceval.c index 83667353af232c..02bc67ef71090a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4699,11 +4699,7 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) assert(!PyErr_Occurred()); #endif - if (args == NULL) { - return _PyObject_FastCallDict(func, NULL, 0, kwargs); - } - - if (!PyTuple_Check(args)) { + if (args != NULL && !PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; @@ -4715,7 +4711,12 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) return NULL; } - return PyObject_Call(func, args, kwargs); + if (args == NULL) { + return _PyObject_FastCallDict(func, NULL, 0, kwargs); + } + else { + return PyObject_Call(func, args, kwargs); + } } const char * From b05d48e021bc9c0700939978ab1ba300e0407a0e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 1 Mar 2017 07:35:23 -0800 Subject: [PATCH 0063/1345] email.compat32-message.rst: Fix typo in the word `message` (GH-379) (GH-380) (cherry picked from commit da62373b0d32c14a4137512ef6f13c24fbcaa2c1) --- Doc/library/email.compat32-message.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index afb5df5d428929..b0707646e64ac6 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -33,9 +33,9 @@ having a MIME type such as :mimetype:`multipart/\*` or The conceptual model provided by a :class:`Message` object is that of an ordered dictionary of headers with additional methods for accessing both specialized information from the headers, for accessing the payload, for -generating a serialized version of the mssage, and for recursively walking over -the object tree. Note that duplicate headers are supported but special methods -must be used to access them. +generating a serialized version of the message, and for recursively walking +over the object tree. Note that duplicate headers are supported but special +methods must be used to access them. The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are From 784ba7c8ad53638c94270011d55d2536ff0cd2dd Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 2 Mar 2017 12:32:13 -0500 Subject: [PATCH 0064/1345] bpo-29697: Don't use OpenSSL <1.0.2 fallback on 1.1+ (#397) --- Modules/_ssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b1988570604db1..c0a7b8e1052c33 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2729,12 +2729,12 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif -#ifndef OPENSSL_NO_ECDH +#if !defined(OPENSSL_NO_ECDH) && !defined(OPENSSL_VERSION_1_1) /* Allow automatic ECDH curve selection (on OpenSSL 1.0.2+), or use prime256v1 by default. This is Apache mod_ssl's initialization policy, so we should be safe. OpenSSL 1.1 has it enabled by default. */ -#if defined(SSL_CTX_set_ecdh_auto) && !defined(OPENSSL_VERSION_1_1) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(self->ctx, 1); #else { From 65bd0bdf3d285e3917d66c600c95cb0842e3b3be Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 2 Mar 2017 12:37:50 -0500 Subject: [PATCH 0065/1345] In SSL module version examples, don't use a legacy version. (#381) (#400) --- Doc/library/ssl.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 7b76b2a0064bf8..bbb13745b203e6 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -846,7 +846,7 @@ Constants The version string of the OpenSSL library loaded by the interpreter:: >>> ssl.OPENSSL_VERSION - 'OpenSSL 0.9.8k 25 Mar 2009' + 'OpenSSL 1.0.2k 26 Jan 2017' .. versionadded:: 3.2 @@ -856,7 +856,7 @@ Constants OpenSSL library:: >>> ssl.OPENSSL_VERSION_INFO - (0, 9, 8, 11, 15) + (1, 0, 2, 11, 15) .. versionadded:: 3.2 @@ -865,9 +865,9 @@ Constants The raw version number of the OpenSSL library, as a single integer:: >>> ssl.OPENSSL_VERSION_NUMBER - 9470143 + 268443839 >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x9080bf' + '0x100020bf' .. versionadded:: 3.2 From a6e84933d204f807e0e81b6a2237193b2e8ab89a Mon Sep 17 00:00:00 2001 From: Brian Coleman Date: Thu, 2 Mar 2017 22:21:53 +0000 Subject: [PATCH 0066/1345] bpo-29683 - Fixes to _PyCode_SetExtra when co_extra->ce->extras is (#402) allocated. On PyMem_Realloc failure, _PyCode_SetExtra should free co_extra if co_extra->ce_extras could not be allocated. On PyMem_Realloc success, _PyCode_SetExtra should set all unused slots in co_extra->ce_extras to NULL. --- Misc/NEWS | 3 +++ Objects/codeobject.c | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 33559a7abb8ce1..a57c476a0b5be1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + - bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. It should raise TypeError when kwargs is not a dict. But it might cause segv when args=NULL and kwargs is not a dict. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d8a675f9fbfc6..df8b9538fe9ed5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -856,16 +856,15 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL) { - o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc( - sizeof(_PyCodeObjectExtra)); - if (o->co_extra == NULL) { + co_extra = PyMem_Malloc(sizeof(_PyCodeObjectExtra)); + if (co_extra == NULL) { return -1; } - co_extra = (_PyCodeObjectExtra *) o->co_extra; co_extra->ce_extras = PyMem_Malloc( tstate->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { + PyMem_Free(co_extra); return -1; } @@ -874,20 +873,25 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; } + + o->co_extra = co_extra; } else if (co_extra->ce_size <= index) { - co_extra->ce_extras = PyMem_Realloc( + void** ce_extras = PyMem_Realloc( co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); - if (co_extra->ce_extras == NULL) { + if (ce_extras == NULL) { return -1; } - co_extra->ce_size = tstate->co_extra_user_count; - - for (Py_ssize_t i = co_extra->ce_size; i < co_extra->ce_size; i++) { - co_extra->ce_extras[i] = NULL; + for (Py_ssize_t i = co_extra->ce_size; + i < tstate->co_extra_user_count; + i++) { + ce_extras[i] = NULL; } + + co_extra->ce_extras = ce_extras; + co_extra->ce_size = tstate->co_extra_user_count; } co_extra->ce_extras[index] = extra; From 01e5230ef0b28658cf7311be199363eda98808bd Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:06:15 -0500 Subject: [PATCH 0067/1345] bpo-29703: asyncio: Fix creating new event loops in child processes. (#404) (#410) --- Lib/asyncio/events.py | 8 +++++++- Lib/asyncio/test_utils.py | 5 ++++- Lib/test/test_asyncio/test_events.py | 22 ++++++++++++++++++++++ Misc/NEWS | 3 +++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 28a45fc3cc5aee..7b30b4c84ad9a7 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,6 +11,7 @@ import functools import inspect +import os import reprlib import socket import subprocess @@ -611,6 +612,9 @@ def new_event_loop(self): # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): _loop = None + _pid = None + + _running_loop = _RunningLoop() @@ -620,7 +624,8 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + if _running_loop._pid == os.getpid(): + return _running_loop._loop def _set_running_loop(loop): @@ -629,6 +634,7 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ + _running_loop._pid = os.getpid() _running_loop._loop = loop diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index 99e3839f456858..b12d5db2a9755d 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -449,12 +449,15 @@ def new_test_loop(self, gen=None): self.set_event_loop(loop) return loop + def unpatch_get_running_loop(self): + events._get_running_loop = self._get_running_loop + def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None def tearDown(self): - events._get_running_loop = self._get_running_loop + self.unpatch_get_running_loop() events.set_event_loop(None) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 28d92a9f4e3eac..802763bd11ff61 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1,6 +1,7 @@ """Tests for events.py.""" import collections.abc +import concurrent.futures import functools import gc import io @@ -57,6 +58,15 @@ def osx_tiger(): return version < (10, 5) +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + ONLYCERT = data_file('ssl_cert.pem') ONLYKEY = data_file('ssl_key.pem') SIGNED_CERTFILE = data_file('keycert3.pem') @@ -2181,6 +2191,18 @@ def tearDown(self): asyncio.set_child_watcher(None) super().tearDown() + def test_get_event_loop_new_process(self): + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + return await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + + self.unpatch_get_running_loop() + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, SubprocessTestsMixin, diff --git a/Misc/NEWS b/Misc/NEWS index a57c476a0b5be1..8391c6a795a2c7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -79,6 +79,9 @@ Extension Modules Library ------- +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + - bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - bpo-28624: Add a test that checks that cwd parameter of Popen() accepts From dea5101ae101aefed14de98e6bb1658f4cae8712 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:20:00 -0500 Subject: [PATCH 0068/1345] bpo-28893: Set __cause__ for errors in async iteration protocol (#407) --- Lib/test/test_coroutines.py | 38 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Python/ceval.c | 6 +++--- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 78439a2acae2d8..b4c7b5be6e208c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1680,6 +1680,44 @@ async def foo(): warnings.simplefilter("error") run_async(foo()) + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_12(self): + class F: + def __aiter__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __aiter__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + def test_for_tuple(self): class Done(Exception): pass diff --git a/Misc/NEWS b/Misc/NEWS index 8391c6a795a2c7..7e355e1b91c041 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + - bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by Brian Coleman. diff --git a/Python/ceval.c b/Python/ceval.c index 02bc67ef71090a..9cac771abdf65b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1904,13 +1904,13 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { - SET_TOP(NULL); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __aiter__: %.100s", Py_TYPE(iter)->tp_name); + SET_TOP(NULL); Py_DECREF(iter); goto error; } else { @@ -1969,7 +1969,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", From 13802a3b11eb5202b16e464cbfb85c144f8581ce Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:16:33 -0500 Subject: [PATCH 0069/1345] bpo-29271: Fix Task.current_task and Task.all_tasks to accept None. (#406) --- Lib/test/test_asyncio/test_tasks.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 16 ++++++++-------- Modules/clinic/_asynciomodule.c.h | 6 +++--- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a18d49ae374112..4f05319457985a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1462,6 +1462,14 @@ def test_current_task(self): def coro(loop): self.assertTrue(Task.current_task(loop=loop) is task) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(loop) + try: + self.assertIs(Task.current_task(None), task) + self.assertIs(Task.current_task(), task) + finally: + asyncio.set_event_loop(None) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) self.assertIsNone(Task.current_task(loop=self.loop)) @@ -1806,8 +1814,17 @@ def kill_me(loop): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(self.loop) + try: + self.assertEqual(Task.all_tasks(), {task}) + self.assertEqual(Task.all_tasks(None), {task}) + finally: + asyncio.set_event_loop(None) + # execute the task so it waits for future self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) diff --git a/Misc/NEWS b/Misc/NEWS index 7e355e1b91c041..ab5c602c454e93 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -82,6 +82,9 @@ Extension Modules Library ------- +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + - bpo-29703: Fix asyncio to support instantiation of new event loops in child processes. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fff90468a5927c..d0e43ae47d9ec8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1413,7 +1413,7 @@ TaskObj_get_fut_waiter(TaskObj *task) @classmethod _asyncio.Task.current_task - loop: 'O' = NULL + loop: 'O' = None Return the currently running task in an event loop or None. @@ -1424,12 +1424,12 @@ None is returned when called not in the context of a Task. static PyObject * _asyncio_Task_current_task_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=99fbe7332c516e03 input=cd784537f02cf833]*/ +/*[clinic end generated code: output=99fbe7332c516e03 input=a0d6cdf2e3b243e1]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1500,7 +1500,7 @@ task_all_tasks(PyObject *loop) @classmethod _asyncio.Task.all_tasks - loop: 'O' = NULL + loop: 'O' = None Return a set of all tasks for an event loop. @@ -1509,12 +1509,12 @@ By default all tasks for the current event loop are returned. static PyObject * _asyncio_Task_all_tasks_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=11f9b20749ccca5d input=cd64aa5f88bd5c49]*/ +/*[clinic end generated code: output=11f9b20749ccca5d input=c6f5b53bd487488f]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 052d252331f6a7..41f11f4cac5494 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -278,7 +278,7 @@ _asyncio_Task_current_task(PyTypeObject *type, PyObject **args, Py_ssize_t nargs PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:current_task", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -310,7 +310,7 @@ _asyncio_Task_all_tasks(PyTypeObject *type, PyObject **args, Py_ssize_t nargs, P PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:all_tasks", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -517,4 +517,4 @@ _asyncio_Task__wakeup(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=8f036321bb083066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=40ca6c9da517da73 input=a9049054013a1b77]*/ From 604faba1db724951ee440337099d111e3ecade49 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 22:21:18 -0600 Subject: [PATCH 0070/1345] bpo-29704: Fix asyncio.SubprocessStreamProtocol closing (#405) --- Lib/asyncio/subprocess.py | 17 +- Lib/test/test_asyncio/test_subprocess.py | 24 + Misc/NEWS | 785 +++++++++++++++++++++++ 3 files changed, 824 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772121..4c85466859f8f0 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -24,6 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -86,9 +90,18 @@ def pipe_connection_lost(self, fd, exc): else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7..2e14a8a9735535 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -459,6 +459,30 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS b/Misc/NEWS index ab5c602c454e93..ef45957e47e76c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,6 +4,791 @@ Python News What's New in Python 3.6.1 release candidate 1? =============================================== +What's New in Python 3.7.0 alpha 1? +=================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- bpo-28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, + and AsyncFunctionDef ast nodes. docstring is not first stmt in their body + anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object + for module and class. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + +- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. + +- bpo-29546: Improve from-import error message with location + +- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. + +- Issue #29337: Fixed possible BytesWarning when compare the code objects. + Warnings could be emitted at compile time. + +- Issue #29327: Fixed a crash when pass the iterable keyword argument to + sorted(). + +- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). + +- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. + +- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. + Calling function is up to 5% faster. + +- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII + whitespace, not only spaces. Patch by Robert Xiao. + +- Issue #28932: Do not include if it does not exist. + +- Issue #25677: Correct the positioning of the syntax error caret for + indented blocks. Based on patch by Michael Layzell. + +- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate + form. + +- Issue #18896: Python function can now have more than 255 parameters. + collections.namedtuple() now supports tuples with more than 255 elements. + +- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by + Chi Hsuan Yen. + +- Issue #26919: On Android, operating system data is now always encoded/decoded + to/from UTF-8, instead of the locale encoding to avoid inconsistencies with + os.fsencode() and os.fsdecode() which are already using UTF-8. + +- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy + bug triggerable by a monkey-patched len() function. + +- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() + must not convert combined table into split table. Patch written by INADA + Naoki. + +- Issue #28739: f-string expressions are no longer accepted as docstrings and + by ast.literal_eval() even if they do not include expressions. + +- Issue #28512: Fixed setting the offset attribute of SyntaxError by + PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +- Issue #28918: Fix the cross compilation of xxlimited when Python has been + built with Py_DEBUG defined. + +- Issue #23722: Rather than silently producing a class that doesn't support + zero-argument ``super()`` in methods, failing to pass the new + ``__classcell__`` namespace entry up to ``type.__new__`` now results in a + ``DeprecationWarning`` and a class that supports zero-argument ``super()``. + +- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of + a descriptor that is used inside that class no longer prevents calling the + __set_name__ method of other descriptors. + +- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate + the untested and undocumented ``sys.callstats()`` function. Remove the + ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, + :mod:`cProfile` or :mod:`profile` to profile function calls. + +- Issue #12844: More than 255 arguments can now be passed to a function. + +- Issue #28782: Fix a bug in the implementation ``yield from`` when checking + if the next instruction is YIELD_FROM. Regression introduced by WORDCODE + (issue #26647). + +- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 + encoders when a string returned by the error handler contains multiple + non-encodable characters (non-ASCII for the ASCII codec, characters out + of the U+0000-U+00FF range for Latin1). + +- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. + Improve speed of dict literal with constant keys up to 30%. + +- Issue #28532: Show sys.version when -V option is supplied twice. + +- Issue #27100: The with-statement now checks for __enter__ before it + checks for __exit__. This gives less confusing error messages when + both methods are missing. Patch by Jonathan Ellington. + +- Issue #28746: Fix the set_inheritable() file descriptor method on platforms + that do not have the ioctl FIOCLEX and FIONCLEX commands. + +- Issue #26920: Fix not getting the locale's charset upon initializing the + interpreter, on platforms that do not have langinfo. + +- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X + when decode astral characters. Patch by Xiang Zhang. + +- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. + +- Issue #19398: Extra slash no longer added to sys.path components in case of + empty compile-time PYTHONPATH components. + +- Issue #28621: Sped up converting int to float by reusing faster bits counting + implementation. Patch by Adrian Wielgosik. + +- Issue #28580: Optimize iterating split table values. + Patch by Xiang Zhang. + +- Issue #28583: PyDict_SetDefault didn't combine split table when needed. + Patch by Xiang Zhang. + +- Issue #28128: Deprecation warning for invalid str and byte escape + sequences now prints better information about where the error + occurs. Patch by Serhiy Storchaka and Eric Smith. + +- Issue #28509: dict.update() no longer allocate unnecessary large memory. + +- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug + build. + +- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused + keeping unreachable code. + +- Issue #28214: Improved exception reporting for problematic __set_name__ + attributes. + +- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception + loss in PyTraceBack_Here(). + +- Issue #28183: Optimize and cleanup dict iteration. + +- Issue #26081: Added C implementation of asyncio.Future. + Original patch by Yury Selivanov. + +- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). + Patch by Xiang Zhang. + +- Issue #28376: The type of long range iterator is now registered as Iterator. + Patch by Oren Milman. + +- Issue #28376: Creating instances of range_iterator by calling range_iterator + type now is disallowed. Calling iter() on range instance is the only way. + Patch by Oren Milman. + +- Issue #26906: Resolving special methods of uninitialized type now causes + implicit initialization of the type instead of a fail. + +- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. + Original patch by Niklas Koep. + +- Issue #24098: Fixed possible crash when AST is changed in process of + compiling it. + +- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when + hashes have same lower bits. + +- Issue #28350: String constants with null character no longer interned. + +- Issue #26617: Fix crash when GC runs during weakref callbacks. + +- Issue #27942: String constants now interned recursively in tuples and frozensets. + +- Issue #28289: ImportError.__init__ now resets not specified attributes. + +- Issue #21578: Fixed misleading error message when ImportError called with + invalid keyword args. + +- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. + Patch by Soumya Sharma. + +- Issue #28086: Single var-positional argument of tuple subtype was passed + unscathed to the C-defined function. Now it is converted to exact tuple. + +- Issue #28214: Now __set_name__ is looked up on the class instead of the + instance. + +- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() + syscall fails with EPERM, for example when blocked by SECCOMP. + +- Issue #28192: Don't import readline in isolated mode. + +- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. + Thanks Oren Milman. + +- Issue #27222: Clean up redundant code in long_rshift function. Thanks + Oren Milman. + +- Upgrade internal unicode databases to Unicode version 9.0.0. + +- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport + should use the same optimization level as the interpreter. + +- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly + optimize memcpy(). + +- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a + "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. + +- Issue #26182: Raise DeprecationWarning when async and await keywords are + used as variable/attribute/class/function name. + +- Issue #26182: Fix a refleak in code that raises DeprecationWarning. + +- Issue #28721: Fix asynchronous generators aclose() and athrow() to + handle StopAsyncIteration propagation properly. + +- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD + opcodes. + +Extension Modules +----------------- + +- Issue #29169: Update zlib to 1.2.11. + +Library +------- + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher + and its subclasses can now be used as a decorator. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes + '~' in the set of characters that is not quoted by default. Patch by + Christian Theune and Ratnadeep Debnath. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Łukasz Langa. + +- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python + 3.6.0: check minimum and maximum years. + +- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir + +- Issue #29444: Fixed out-of-bounds buffer access in the group() method of + the match object. Based on patch by WGH. + +- Issue #29377: Add SlotWrapperType, MethodWrapperType, and + MethodDescriptorType built-in types to types module. + Original patch by Manuel Krebber. + +- Issue #29218: Unused install_misc command is now removed. It has been + documented as unused since 2000. Patch by Eric N. Vander Weele. + +- Issue #29368: The extend() method is now called instead of the append() + method when unpickle collections.deque and other list-like objects. + This can speed up unpickling to 2 times. + +- Issue #29338: The help of a builtin or extension class now includes the + constructor signature if __text_signature__ is provided for the class. + +- Issue #29335: Fix subprocess.Popen.wait() when the child process has + exited to a stopped instead of terminated state (ex: when under ptrace). + +- Issue #29290: Fix a regression in argparse that help messages would wrap at + non-breaking spaces. + +- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. + +- Issue #29197: Removed deprecated function ntpath.splitunc(). + +- Issue #29210: Removed support of deprecated argument "exclude" in + tarfile.TarFile.add(). + +- Issue #29219: Fixed infinite recursion in the repr of uninitialized + ctypes.CDLL instances. + +- Issue #29192: Removed deprecated features in the http.cookies module. + +- Issue #29193: A format string argument for string.Formatter.format() + is now positional-only. + +- Issue #29195: Removed support of deprecated undocumented keyword arguments + in methods of regular expression objects. + +- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. + KeyError could be raised when cached function with full cache was + simultaneously called from differen threads with the same uncached arguments. + +- Issue #20804: The unittest.mock.sentinel attributes now preserve their + identity when they are copied or pickled. + +- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with + leading dots could match related hostnames again (e.g. .b.c matches a.b.c). + Patch by Milan Oberkirch. + +- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter + anymore. Patch written by Jiajun Huang. + +- Issue #15812: inspect.getframeinfo() now correctly shows the first line of + a context. Patch by Sam Breese. + +- Issue #28985: Update authorizer constants in sqlite3 module. + Patch by Dingyuan Wang. + +- Issue #29094: Offsets in a ZIP file created with extern file object and modes + "w" and "x" now are relative to the start of the file. + +- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows + +- Issue #13051: Fixed recursion errors in large or resized + curses.textpad.Textbox. Based on patch by Tycho Andersen. + +- Issue #9770: curses.ascii predicates now work correctly with negative + integers. + +- Issue #28427: old keys should not remove new values from + WeakValueDictionary when collecting from another thread. + +- Issue 28923: Remove editor artifacts from Tix.py. + +- Issue #28871: Fixed a crash when deallocate deep ElementTree. + +- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and + WeakValueDictionary.pop() when a GC collection happens in another + thread. + +- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that + doesn't own its elements as limits. + +- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, + instead of /bin/sh. + +- Issue #28779: multiprocessing.set_forkserver_preload() would crash the + forkserver process if a preloaded module instantiated some + multiprocessing objects such as locks. + +- Issue #26937: The chown() method of the tarfile.TarFile class does not fail + now when the grp module cannot be imported, as for example on Android + platforms. + +- Issue #28847: dbm.dumb now supports reading read-only files and no longer + writes the index file when it is not changed. A deprecation warning is now + emitted if the index file is missed and recreated in the 'r' and 'w' modes + (will be an error in future Python releases). + +- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in + re.sub() replacement templates regular expressions now are errors. + +- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): + call warnings.showwarning() if it was overriden inside the context manager. + +- Issue #27172: To assist with upgrades from 2.7, the previously documented + deprecation of ``inspect.getfullargspec()`` has been reversed. This decision + may be revisited again after the Python 2.7 branch is no longer officially + supported. + +- Issue #28740: Add sys.getandroidapilevel(): return the build time API version + of Android as an integer. Function only available on Android. + +- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and + :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by + Omar Sandoval. + +- Issue #28752: Restored the __reduce__() methods of datetime objects. + +- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created + by re.compile(), become comparable (only x==y and x!=y operators). This + change should fix the issue #18383: don't duplicate warning filters when the + warnings module is reloaded (thing usually only done in unit tests). + +- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was + deprecated in 3.4 and undocumented prior to that. + +- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and + from_buffer_copy() methods on abstract classes like Array. + +- Issue #28548: In the "http.server" module, parse the protocol version if + possible, to avoid using HTTP 0.9 in some error responses. + +- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. + Patch by Vajrasky Kok + +- Issue #28563: Fixed possible DoS and arbitrary code execution when handle + plural form selections in the gettext module. The expression parser now + supports exact syntax supported by GNU gettext. + +- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when + the garbage collector is invoked in other thread. Based on patch by + Sebastian Cufre. + +- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if + given empty data twice. Patch by Benjamin Fogle. + +- Issue #28549: Fixed segfault in curses's addch() with ncurses6. + +- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar + file with compression before trying to open it without compression. Otherwise + it had 50% chance failed with ignore_zeros=True. + +- Issue #23262: The webbrowser module now supports Firefox 36+ and derived + browsers. Based on patch by Oleg Broytman. + +- Issue #24241: The webbrowser in an X environment now prefers using the + default browser directly. Also, the webbrowser register() function now has + a documented 'preferred' argument, to specify browsers to be returned by + get() with no arguments. Patch by David Steele + +- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused + by representing the scale as float value internally in Tk. tkinter.IntVar + now works if float value is set to underlying Tk variable. + +- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after + a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant + newline after a year's calendar. Based on patch by Xiang Zhang. + +- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space + at the start of new line after printing a month's calendar. Patch by + Xiang Zhang. + +- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. + Based on patch by Kaarle Ritvanen. + +- Issue #28353: os.fwalk() no longer fails on broken links. + +- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept + non-None value is passed to it.send(val). + +- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix + for readability (was "`"). + +- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin + a workaround to Tix library bug. + +- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. + +- Issue #25953: re.sub() now raises an error for invalid numerical group + reference in replacement template even if the pattern is not found in + the string. Error message for invalid group reference now includes the + group index and the position of the reference. + Based on patch by SilentGhost. + +- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead + of 1, 10, 100,... for autoranging. + +- Issue #28115: Command-line interface of the zipfile module now uses argparse. + Added support of long options. + +- Issue #18219: Optimize csv.DictWriter for large number of columns. + Patch by Mariatta Wijaya. + +- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. + +- Issue #23214: In the "io" module, the argument to BufferedReader and + BytesIO's read1() methods is now optional and can be -1, matching the + BufferedIOBase specification. + +- Issue #28480: Fix error building socket module when multithreading is + disabled. + +- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line + options which were deprecated since Python 3.3. + +- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 + to make benchmarks more reliable. + +- Issue #28240: timeit autorange now uses a single loop iteration if the + benchmark takes less than 10 seconds, instead of 10 iterations. + "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds + instead of 40 seconds. + +- Distutils.sdist now looks for README and setup.py files with case + sensitivity. This behavior matches that found in Setuptools 6.0 and + later. See `setuptools 100 + `_ for rationale. + +- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by + Ned Batchelder. + +- Issue #20766: Fix references leaked by pdb in the handling of SIGINT + handlers. + +- Issue #27998: Fixed bytes path support in os.scandir() on Windows. + Patch by Eryk Sun. + +- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. + +- Issue #26293: Fixed writing ZIP files that starts not from the start of the + file. Offsets in ZIP file now are relative to the start of the archive in + conforming to the specification. + +- Issue #28380: unittest.mock Mock autospec functions now properly support + assert_called, assert_not_called, and assert_called_once. + +- Issue #28229: lzma module now supports pathlib. + +- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. + +- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. + +- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. + +- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. + Original patch by Oren Milman. + +- Issue #27358: Optimized merging var-keyword arguments and improved error + message when passing a non-mapping as a var-keyword argument. + +- Issue #28257: Improved error message when passing a non-iterable as + a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. + +- Issue #28322: Fixed possible crashes when unpickle itertools objects from + incorrect pickle data. Based on patch by John Leitch. + +- Issue #28228: imghdr now supports pathlib. + +- Issue #28226: compileall now supports pathlib. + +- Issue #28314: Fix function declaration (C flags) for the getiterator() method + of xml.etree.ElementTree.Element. + +- Issue #28148: Stop using localtime() and gmtime() in the time + module. + + Introduced platform independent _PyTime_localtime API that is + similar to POSIX localtime_r, but available on all platforms. Patch + by Ed Schouten. + +- Issue #28253: Fixed calendar functions for extreme months: 0001-01 + and 9999-12. + + Methods itermonthdays() and itermonthdays2() are reimplemented so + that they don't call itermonthdates() which can cause datetime.date + under/overflow. + +- Issue #28275: Fixed possible use after free in the decompress() + methods of the LZMADecompressor and BZ2Decompressor classes. + Original patch by John Leitch. + +- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() + if pass invalid string-like object as a name. Patch by Xiang Zhang. + +- Issue #18844: random.choices() now has k as a keyword-only argument + to improve the readability of common cases and come into line + with the signature used in other languages. + +- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. + Patch by Madison May. + +- Issue #27611: Fixed support of default root window in the tkinter.tix module. + Added the master parameter in the DisplayStyle constructor. + +- Issue #27348: In the traceback module, restore the formatting of exception + messages like "Exception: None". This fixes a regression introduced in + 3.5a2. + +- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). + +- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is + interrupted by a signal and a signal handler raises a Python exception. + +- Issue #28200: Fix memory leak on Windows in the os module (fix + path_converter() function). + +- Issue #25400: RobotFileParser now correctly returns default values for + crawl_delay and request_rate. Initial patch by Peter Wirtz. + +- Issue #27932: Prevent memory leak in win32_ver(). + +- Fix UnboundLocalError in socket._sendfile_use_sendfile. + +- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of + os.stat(). Patch by Eryk Sun. + +- Issue #22493: Warning message emitted by using inline flags in the middle of + regular expression now contains a (truncated) regex pattern. + Patch by Tim Graham. + +- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when + an empty bytestring is passed. + +- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. + Patch by Gergely Imreh and Markus Holtermann. + +- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. + Patch by Eryk Sun. + +- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +- Issue #27906: Fix socket accept exhaustion during high TCP traffic. + Patch by Kevin Conway. + +- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. + Patch by Seth Michael Larson. + +- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. + Patch by iceboy. + +- Issue #26909: Fix slow pipes IO in asyncio. + Patch by INADA Naoki. + +- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. + Patch by Mark Williams. + +- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. + +- Issue #28368: Refuse monitoring processes if the child watcher has + no loop attached. + Patch by Vincent Michel. + +- Issue #28369: Raise RuntimeError when transport's FD is used with + add_reader, add_writer, etc. + +- Issue #28370: Speedup asyncio.StreamReader.readexactly. + Patch by Коренберг Марк. + +- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. + +- Issue #28372: Fix asyncio to support formatting of non-python coroutines. + +- Issue #28399: Remove UNIX socket from FS before binding. + Patch by Коренберг Марк. + +- Issue #27972: Prohibit Tasks to await on themselves. + +- Issue #24142: Reading a corrupt config file left configparser in an + invalid state. Original patch by Florian Höch. + +Windows +------- + +- bpo-29579: Removes readme.txt from the installer. + +- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) + +- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default + +- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c + +- Issue #28402: Adds signed catalog files for stdlib on Windows. + +- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by + Eryk Sun) + +- Issue #28251: Improvements to help manuals on Windows. + +- Issue #28110: launcher.msi has different product codes between 32-bit and + 64-bit + +- Issue #28161: Opening CON for write access fails + +- Issue #28162: WindowsConsoleIO readall() fails if first line starts with + Ctrl+Z + +- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to + _open_osfhandle + +- Issue #28164: _PyIO_get_console_type fails for various paths + +- Issue #28137: Renames Windows path file to ._pth + +- Issue #28138: Windows ._pth file should allow import site + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with + a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 + and 0x03060000 (not including) or 0x03060100 or higher. Added functions + PySlice_Unpack() and PySlice_AdjustIndices(). + +- Issue #29083: Fixed the declaration of some public API functions. + PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in + limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and + Py_BuildValue() were not available in limited API of version < 3.3 when + PY_SSIZE_T_CLEAN is defined. + +- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() + is now of type ``const char *`` rather of ``char *``. + +- Issue #29058: All stable API extensions added after Python 3.2 are now + available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of + the minimum Python version supporting this API. + +- Issue #28822: The index parameters *start* and *end* of PyUnicode_FindChar() + are now adjusted to behave like ``str[start:end]``. + +- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. + +- Issue #28761: The fields name and doc of structures PyMemberDef, PyGetSetDef, + PyStructSequence_Field, PyStructSequence_Desc, and wrapperbase are now of + type ``const char *`` rather of ``char *``. + +- Issue #28748: Private variable _Py_PackageContext is now of type ``const char *`` + rather of ``char *``. + +- Issue #19569: Compiler warnings are now emitted if use most of deprecated + functions. + +- Issue #28426: Deprecated undocumented functions PyUnicode_AsEncodedObject(), + PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and + PyUnicode_AsEncodedUnicode(). + +Documentation +------------- + +- bpo-28929: Link the documentation to its source file on GitHub. + +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + +- Issue #26355: Add canonical header link on each page to corresponding major + version of the documentation. Patch by Matthias Bussonnier. + +- Issue #29349: Fix Python 2 syntax in code for building the documentation. *Release date: XXXX-XX-XX* From 2ef08d3be780457c444741b67e6181675b044be9 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 23:57:33 -0500 Subject: [PATCH 0071/1345] asyncio: Optimize _get_running_loop() to call getpid() only when there's a loop --- Lib/asyncio/events.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 7b30b4c84ad9a7..e85634e588f5a5 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -624,8 +624,9 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - if _running_loop._pid == os.getpid(): - return _running_loop._loop + running_loop = _running_loop._loop + if running_loop is not None and _running_loop._pid == os.getpid(): + return running_loop def _set_running_loop(loop): From d8b72e4a0673c414120b029065dbe77055f12e82 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 23:46:56 -0500 Subject: [PATCH 0072/1345] bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback/C (#408) --- Lib/test/test_asyncio/test_futures.py | 29 +++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 89afdcaff28665..99336f86ab824e 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -569,6 +569,35 @@ def test_remove_done_callback(self): self.assertEqual(bag, [2]) self.assertEqual(f.result(), 'foo') + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS b/Misc/NEWS index ef45957e47e76c..a13d8c74f23228 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -261,6 +261,9 @@ Extension Modules Library ------- +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + - bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before all pipes are closed. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d0e43ae47d9ec8..a77ff9689e4705 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -521,7 +521,7 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < len; i++) { + for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); From b8f5d07158f171a18be4d42f9769d3c9c073e748 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 2 Mar 2017 21:50:10 -0800 Subject: [PATCH 0073/1345] bpo-29026: Clarify documentation of time.time (GH-34) (GH-417) (cherry picked from commit 23557d59b819f57800ddef0b1373acef8e024670) --- Doc/library/time.rst | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ae17f6f4f0d236..d2e2ac4410aba3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -17,11 +17,23 @@ semantics of these functions varies among platforms. An explanation of some terminology and conventions is in order. +.. _epoch: + .. index:: single: epoch -* The :dfn:`epoch` is the point where the time starts. On January 1st of that - year, at 0 hours, the "time since the epoch" is zero. For Unix, the epoch is - 1970. To find out what the epoch is, look at ``gmtime(0)``. +* The :dfn:`epoch` is the point where the time starts, and is platform + dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). + To find out what the epoch is on a given platform, look at + ``time.gmtime(0)``. + +.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second + +.. index:: seconds since the epoch + +* The term :dfn:`seconds since the epoch` refers to the total number + of elapsed seconds since the epoch, typically excluding + `leap seconds`_. Leap seconds are excluded from this total on all + POSIX-compliant platforms. .. index:: single: Year 2038 @@ -467,7 +479,7 @@ The module defines the following functions and data items: (2) The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing leap seconds and value ``61`` is supported + timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. (3) @@ -572,12 +584,28 @@ The module defines the following functions and data items: .. function:: time() - Return the time in seconds since the epoch as a floating point number. + Return the time in seconds since the epoch_ as a floating point + number. The specific date of the epoch and the handling of + `leap seconds`_ is platform dependent. + On Windows and most Unix systems, the epoch is January 1, 1970, + 00:00:00 (UTC) and leap seconds are not counted towards the time + in seconds since the epoch. This is commonly referred to as + `Unix time `_. + To find out what the epoch is on a given platform, look at + ``gmtime(0)``. + Note that even though the time is always returned as a floating point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back between - the two calls. + lower value than a previous call if the system clock has been set back + between the two calls. + + The number returned by :func:`.time` may be converted into a more common + time format (i.e. year, month, day, hour, etc...) in UTC by passing it to + :func:`gmtime` function or in local time by passing it to the + :func:`localtime` function. In both cases a + :class:`struct_time` object is returned, from which the components + of the calendar date may be accessed as attributes. .. data:: timezone From 93692b1a7cfd70be2aed33d77da05518d3a41f51 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 2 Mar 2017 21:56:06 -0800 Subject: [PATCH 0074/1345] getpass: update docstrings (GH-49) (GH-419) (cherry picked from commit baf7bb30a02aabde260143136bdf5b3738a1d409) --- Lib/getpass.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/getpass.py b/Lib/getpass.py index be511211585a48..36e17e4cb6965d 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -7,7 +7,6 @@ echoing of the password contents while reading. On Windows, the msvcrt module will be used. -On the Mac EasyDialogs.AskPassword is used, if available. """ From 2adc66848135be8e86ba08690a858737a65ae40d Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 3 Mar 2017 01:05:44 -0500 Subject: [PATCH 0075/1345] Restore NEWS file after incorrect git automerge --- Misc/NEWS | 788 +----------------------------------------------------- 1 file changed, 3 insertions(+), 785 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index a13d8c74f23228..2692043b6cbab6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,8 +4,6 @@ Python News What's New in Python 3.6.1 release candidate 1? =============================================== -What's New in Python 3.7.0 alpha 1? -=================================== *Release date: XXXX-XX-XX* @@ -22,7 +20,7 @@ Core and Builtins It should raise TypeError when kwargs is not a dict. But it might cause segv when args=NULL and kwargs is not a dict. -- bpo-28598: Support __rmod__ for subclasses of str being called before +- Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. @@ -35,17 +33,8 @@ Core and Builtins - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. -- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, - and AsyncFunctionDef ast nodes. docstring is not first stmt in their body - anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object - for module and class. - - bpo-29438: Fixed use-after-free problem in key sharing dict. -- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. - -- bpo-29546: Improve from-import error message with location - - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. @@ -58,12 +47,6 @@ Core and Builtins - Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. -- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. - Calling function is up to 5% faster. - -- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII - whitespace, not only spaces. Patch by Robert Xiao. - - Issue #28932: Do not include if it does not exist. - Issue #25677: Correct the positioning of the syntax error caret for @@ -72,12 +55,6 @@ Core and Builtins - Issue #29000: Fixed bytes formatting of octals with zero padding in alternate form. -- Issue #18896: Python function can now have more than 255 parameters. - collections.namedtuple() now supports tuples with more than 255 elements. - -- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by - Chi Hsuan Yen. - - Issue #26919: On Android, operating system data is now always encoded/decoded to/from UTF-8, instead of the locale encoding to avoid inconsistencies with os.fsencode() and os.fsdecode() which are already using UTF-8. @@ -85,10 +62,6 @@ Core and Builtins - Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy bug triggerable by a monkey-patched len() function. -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. Patch written by INADA - Naoki. - - Issue #28739: f-string expressions are no longer accepted as docstrings and by ast.literal_eval() even if they do not include expressions. @@ -98,161 +71,9 @@ Core and Builtins - Issue #28918: Fix the cross compilation of xxlimited when Python has been built with Py_DEBUG defined. -- Issue #23722: Rather than silently producing a class that doesn't support - zero-argument ``super()`` in methods, failing to pass the new - ``__classcell__`` namespace entry up to ``type.__new__`` now results in a - ``DeprecationWarning`` and a class that supports zero-argument ``super()``. - -- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of - a descriptor that is used inside that class no longer prevents calling the - __set_name__ method of other descriptors. - -- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate - the untested and undocumented ``sys.callstats()`` function. Remove the - ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, - :mod:`cProfile` or :mod:`profile` to profile function calls. - -- Issue #12844: More than 255 arguments can now be passed to a function. - -- Issue #28782: Fix a bug in the implementation ``yield from`` when checking - if the next instruction is YIELD_FROM. Regression introduced by WORDCODE - (issue #26647). - -- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 - encoders when a string returned by the error handler contains multiple - non-encodable characters (non-ASCII for the ASCII codec, characters out - of the U+0000-U+00FF range for Latin1). - - Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. Improve speed of dict literal with constant keys up to 30%. -- Issue #28532: Show sys.version when -V option is supplied twice. - -- Issue #27100: The with-statement now checks for __enter__ before it - checks for __exit__. This gives less confusing error messages when - both methods are missing. Patch by Jonathan Ellington. - -- Issue #28746: Fix the set_inheritable() file descriptor method on platforms - that do not have the ioctl FIOCLEX and FIONCLEX commands. - -- Issue #26920: Fix not getting the locale's charset upon initializing the - interpreter, on platforms that do not have langinfo. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28621: Sped up converting int to float by reusing faster bits counting - implementation. Patch by Adrian Wielgosik. - -- Issue #28580: Optimize iterating split table values. - Patch by Xiang Zhang. - -- Issue #28583: PyDict_SetDefault didn't combine split table when needed. - Patch by Xiang Zhang. - -- Issue #28128: Deprecation warning for invalid str and byte escape - sequences now prints better information about where the error - occurs. Patch by Serhiy Storchaka and Eric Smith. - -- Issue #28509: dict.update() no longer allocate unnecessary large memory. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused - keeping unreachable code. - -- Issue #28214: Improved exception reporting for problematic __set_name__ - attributes. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28183: Optimize and cleanup dict iteration. - -- Issue #26081: Added C implementation of asyncio.Future. - Original patch by Yury Selivanov. - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: Creating instances of range_iterator by calling range_iterator - type now is disallowed. Calling iter() on range instance is the only way. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when - hashes have same lower bits. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #28289: ImportError.__init__ now resets not specified attributes. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. - Patch by Soumya Sharma. - -- Issue #28086: Single var-positional argument of tuple subtype was passed - unscathed to the C-defined function. Now it is converted to exact tuple. - -- Issue #28214: Now __set_name__ is looked up on the class instead of the - instance. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28192: Don't import readline in isolated mode. - -- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. - Thanks Oren Milman. - -- Issue #27222: Clean up redundant code in long_rshift function. Thanks - Oren Milman. - -- Upgrade internal unicode databases to Unicode version 9.0.0. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly - optimize memcpy(). - -- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a - "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. - -- Issue #26182: Raise DeprecationWarning when async and await keywords are - used as variable/attribute/class/function name. - -- Issue #26182: Fix a refleak in code that raises DeprecationWarning. - -- Issue #28721: Fix asynchronous generators aclose() and athrow() to - handle StopAsyncIteration propagation properly. - -- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD - opcodes. - Extension Modules ----------------- @@ -264,611 +85,8 @@ Library - bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback implemented in C. -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C - to accept None argument as their pure Python implementation. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher - and its subclasses can now be used as a decorator. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts - PathLike objects. Patch by Sayan Chowdhury. - -- bpo-28518: Start a transaction implicitly before a DML statement. - Patch by Aviv Palivoda. - -- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes - '~' in the set of characters that is not quoted by default. Patch by - Christian Theune and Ratnadeep Debnath. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - -- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, - improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, - Manuel Krebber, and Łukasz Langa. - -- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python - 3.6.0: check minimum and maximum years. - -- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir - -- Issue #29444: Fixed out-of-bounds buffer access in the group() method of - the match object. Based on patch by WGH. - -- Issue #29377: Add SlotWrapperType, MethodWrapperType, and - MethodDescriptorType built-in types to types module. - Original patch by Manuel Krebber. - -- Issue #29218: Unused install_misc command is now removed. It has been - documented as unused since 2000. Patch by Eric N. Vander Weele. - -- Issue #29368: The extend() method is now called instead of the append() - method when unpickle collections.deque and other list-like objects. - This can speed up unpickling to 2 times. - -- Issue #29338: The help of a builtin or extension class now includes the - constructor signature if __text_signature__ is provided for the class. - -- Issue #29335: Fix subprocess.Popen.wait() when the child process has - exited to a stopped instead of terminated state (ex: when under ptrace). - -- Issue #29290: Fix a regression in argparse that help messages would wrap at - non-breaking spaces. - -- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. - -- Issue #29197: Removed deprecated function ntpath.splitunc(). - -- Issue #29210: Removed support of deprecated argument "exclude" in - tarfile.TarFile.add(). - -- Issue #29219: Fixed infinite recursion in the repr of uninitialized - ctypes.CDLL instances. - -- Issue #29192: Removed deprecated features in the http.cookies module. - -- Issue #29193: A format string argument for string.Formatter.format() - is now positional-only. - -- Issue #29195: Removed support of deprecated undocumented keyword arguments - in methods of regular expression objects. - -- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. - KeyError could be raised when cached function with full cache was - simultaneously called from differen threads with the same uncached arguments. - -- Issue #20804: The unittest.mock.sentinel attributes now preserve their - identity when they are copied or pickled. - -- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with - leading dots could match related hostnames again (e.g. .b.c matches a.b.c). - Patch by Milan Oberkirch. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #28985: Update authorizer constants in sqlite3 module. - Patch by Dingyuan Wang. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that - doesn't own its elements as limits. - -- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, - instead of /bin/sh. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #26937: The chown() method of the tarfile.TarFile class does not fail - now when the grp module cannot be imported, as for example on Android - platforms. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. A deprecation warning is now - emitted if the index file is missed and recreated in the 'r' and 'w' modes - (will be an error in future Python releases). - -- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in - re.sub() replacement templates regular expressions now are errors. - -- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): - call warnings.showwarning() if it was overriden inside the context manager. - -- Issue #27172: To assist with upgrades from 2.7, the previously documented - deprecation of ``inspect.getfullargspec()`` has been reversed. This decision - may be revisited again after the Python 2.7 branch is no longer officially - supported. - -- Issue #28740: Add sys.getandroidapilevel(): return the build time API version - of Android as an integer. Function only available on Android. - -- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and - :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by - Omar Sandoval. - -- Issue #28752: Restored the __reduce__() methods of datetime objects. - -- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created - by re.compile(), become comparable (only x==y and x!=y operators). This - change should fix the issue #18383: don't duplicate warning filters when the - warnings module is reloaded (thing usually only done in unit tests). - -- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was - deprecated in 3.4 and undocumented prior to that. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #28548: In the "http.server" module, parse the protocol version if - possible, to avoid using HTTP 0.9 in some error responses. - -- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. - Patch by Vajrasky Kok - -- Issue #28563: Fixed possible DoS and arbitrary code execution when handle - plural form selections in the gettext module. The expression parser now - supports exact syntax supported by GNU gettext. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #24241: The webbrowser in an X environment now prefers using the - default browser directly. Also, the webbrowser register() function now has - a documented 'preferred' argument, to specify browsers to be returned by - get() with no arguments. Patch by David Steele - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after - a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant - newline after a year's calendar. Based on patch by Xiang Zhang. - -- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept - non-None value is passed to it.send(val). - -- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. - -- Issue #25953: re.sub() now raises an error for invalid numerical group - reference in replacement template even if the pattern is not found in - the string. Error message for invalid group reference now includes the - group index and the position of the reference. - Based on patch by SilentGhost. - -- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead - of 1, 10, 100,... for autoranging. - -- Issue #28115: Command-line interface of the zipfile module now uses argparse. - Added support of long options. - -- Issue #18219: Optimize csv.DictWriter for large number of columns. - Patch by Mariatta Wijaya. - -- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. - -- Issue #23214: In the "io" module, the argument to BufferedReader and - BytesIO's read1() methods is now optional and can be -1, matching the - BufferedIOBase specification. - -- Issue #28480: Fix error building socket module when multithreading is - disabled. - -- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line - options which were deprecated since Python 3.3. - -- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 - to make benchmarks more reliable. - -- Issue #28240: timeit autorange now uses a single loop iteration if the - benchmark takes less than 10 seconds, instead of 10 iterations. - "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds - instead of 40 seconds. - -- Distutils.sdist now looks for README and setup.py files with case - sensitivity. This behavior matches that found in Setuptools 6.0 and - later. See `setuptools 100 - `_ for rationale. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by - Ned Batchelder. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #27998: Fixed bytes path support in os.scandir() on Windows. - Patch by Eryk Sun. - -- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28380: unittest.mock Mock autospec functions now properly support - assert_called, assert_not_called, and assert_called_once. - -- Issue #28229: lzma module now supports pathlib. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. - -- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. - -- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. - Original patch by Oren Milman. - -- Issue #27358: Optimized merging var-keyword arguments and improved error - message when passing a non-mapping as a var-keyword argument. - -- Issue #28257: Improved error message when passing a non-iterable as - a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Issue #28228: imghdr now supports pathlib. - -- Issue #28226: compileall now supports pathlib. - -- Issue #28314: Fix function declaration (C flags) for the getiterator() method - of xml.etree.ElementTree.Element. - -- Issue #28148: Stop using localtime() and gmtime() in the time - module. - - Introduced platform independent _PyTime_localtime API that is - similar to POSIX localtime_r, but available on all platforms. Patch - by Ed Schouten. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18844: random.choices() now has k as a keyword-only argument - to improve the readability of common cases and come into line - with the signature used in other languages. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - Added the master parameter in the DisplayStyle constructor. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is - interrupted by a signal and a signal handler raises a Python exception. - -- Issue #28200: Fix memory leak on Windows in the os module (fix - path_converter() function). - -- Issue #25400: RobotFileParser now correctly returns default values for - crawl_delay and request_rate. Initial patch by Peter Wirtz. - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #22493: Warning message emitted by using inline flags in the middle of - regular expression now contains a (truncated) regex pattern. - Patch by Tim Graham. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. - Patch by Eryk Sun. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. - -- Issue #28368: Refuse monitoring processes if the child watcher has - no loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by Коренберг Марк. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by Коренберг Марк. - -- Issue #27972: Prohibit Tasks to await on themselves. - -- Issue #24142: Reading a corrupt config file left configparser in an - invalid state. Original patch by Florian Höch. - -Windows -------- - -- bpo-29579: Removes readme.txt from the installer. - -- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) - -- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default - -- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c - -- Issue #28402: Adds signed catalog files for stdlib on Windows. - -- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by - Eryk Sun) - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #28161: Opening CON for write access fails - -- Issue #28162: WindowsConsoleIO readall() fails if first line starts with - Ctrl+Z - -- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to - _open_osfhandle - -- Issue #28164: _PyIO_get_console_type fails for various paths - -- Issue #28137: Renames Windows path file to ._pth - -- Issue #28138: Windows ._pth file should allow import site - -C API ------ - -- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with - a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 - and 0x03060000 (not including) or 0x03060100 or higher. Added functions - PySlice_Unpack() and PySlice_AdjustIndices(). - -- Issue #29083: Fixed the declaration of some public API functions. - PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in - limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and - Py_BuildValue() were not available in limited API of version < 3.3 when - PY_SSIZE_T_CLEAN is defined. - -- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() - is now of type ``const char *`` rather of ``char *``. - -- Issue #29058: All stable API extensions added after Python 3.2 are now - available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of - the minimum Python version supporting this API. - -- Issue #28822: The index parameters *start* and *end* of PyUnicode_FindChar() - are now adjusted to behave like ``str[start:end]``. - -- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. - -- Issue #28761: The fields name and doc of structures PyMemberDef, PyGetSetDef, - PyStructSequence_Field, PyStructSequence_Desc, and wrapperbase are now of - type ``const char *`` rather of ``char *``. - -- Issue #28748: Private variable _Py_PackageContext is now of type ``const char *`` - rather of ``char *``. - -- Issue #19569: Compiler warnings are now emitted if use most of deprecated - functions. - -- Issue #28426: Deprecated undocumented functions PyUnicode_AsEncodedObject(), - PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and - PyUnicode_AsEncodedUnicode(). - -Documentation -------------- - -- bpo-28929: Link the documentation to its source file on GitHub. - -- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to - aiosmtpd, a third-party asyncio-based replacement. - -- Issue #26355: Add canonical header link on each page to corresponding major - version of the documentation. Patch by Matthias Bussonnier. - -- Issue #29349: Fix Python 2 syntax in code for building the documentation. - -*Release date: XXXX-XX-XX* - -Core and Builtins ------------------ - -- bpo-28893: Set correct __cause__ for errors about invalid awaitables - returned from __aiter__ and __anext__. - -- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by - Brian Coleman. - -- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. - It should raise TypeError when kwargs is not a dict. But it might - cause segv when args=NULL and kwargs is not a dict. - -- Issue #28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. - Patch by Matthieu Dartiailh. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - -- bpo-29347: Fixed possibly dereferencing undefined pointers - when creating weakref objects. - -- bpo-29438: Fixed use-after-free problem in key sharing dict. - -- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - -- Issue #29337: Fixed possible BytesWarning when compare the code objects. - Warnings could be emitted at compile time. - -- Issue #29327: Fixed a crash when pass the iterable keyword argument to - sorted(). - -- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). - -- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. - -- Issue #28932: Do not include if it does not exist. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #26919: On Android, operating system data is now always encoded/decoded - to/from UTF-8, instead of the locale encoding to avoid inconsistencies with - os.fsencode() and os.fsdecode() which are already using UTF-8. - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug triggerable by a monkey-patched len() function. - -- Issue #28739: f-string expressions are no longer accepted as docstrings and - by ast.literal_eval() even if they do not include expressions. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28918: Fix the cross compilation of xxlimited when Python has been - built with Py_DEBUG defined. - -- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. - Improve speed of dict literal with constant keys up to 30%. - -Extension Modules ------------------ - -- Issue #29169: Update zlib to 1.2.11. - -Library -------- +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes + before all pipes are closed. - bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C to accept None argument as their pure Python implementation. From c7ff163abf428d73d4054754db3c99d06487e3ad Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 3 Mar 2017 18:07:18 +0300 Subject: [PATCH 0076/1345] Add Python version since deprecation in base64 methods. (#33) (#429) Allow developers to not have to either test on N Python versions or looked through multiple versions of the docs to know whether they can easily update. (cherry picked from commit c643a967dd7d33ccefa5b61b38caf40b448057ce) --- Doc/library/base64.rst | 17 +++++++++++++---- Lib/base64.py | 6 ++++-- Lib/test/test_base64.py | 8 ++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 080d9d77ec984e..ceecf17cba23e3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -237,14 +237,18 @@ The legacy interface: .. function:: decodebytes(s) - decodestring(s) Decode the :term:`bytes-like object` *s*, which must contain one or more lines of base64 encoded data, and return the decoded :class:`bytes`. - ``decodestring`` is a deprecated alias. .. versionadded:: 3.1 +.. function:: decodestring(s) + + Deprecated alias of :func:`decodebytes`. + + .. deprecated:: 3.1 + .. function:: encode(input, output) @@ -257,14 +261,19 @@ The legacy interface: .. function:: encodebytes(s) - encodestring(s) Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary data, and return :class:`bytes` containing the base64-encoded data, with newlines (``b'\n'``) inserted after every 76 bytes of output, and ensuring that there is a trailing newline, as per :rfc:`2045` (MIME). - ``encodestring`` is a deprecated alias. + .. versionadded:: 3.1 + +.. function:: encodestring(s) + + Deprecated alias of :func:`encodebytes`. + + .. deprecated:: 3.1 An example usage of the module: diff --git a/Lib/base64.py b/Lib/base64.py index 58f6ad6816ee2f..eb8f258a2d1977 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -541,7 +541,8 @@ def encodebytes(s): def encodestring(s): """Legacy alias of encodebytes().""" import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) @@ -554,7 +555,8 @@ def decodebytes(s): def decodestring(s): """Legacy alias of decodebytes().""" import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 4f86aaa0c0eb49..47547396b8cb54 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,14 @@ def check_type_errors(self, f): int_data = memoryview(b"1234").cast('I') self.assertRaises(TypeError, f, int_data) + def test_encodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.encodestring(b"www.python.org") + + def test_decodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.decodestring(b"d3d3LnB5dGhvbi5vcmc=\n") + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") From 21ce65aa67f0dc63002ab0a5fb21ef921cf5279e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 3 Mar 2017 20:48:37 +0300 Subject: [PATCH 0077/1345] [3.6] bpo-29623: Make PathLike objects work with ConfigParser.read() (#242) (#432) (cherry picked from commit 85b8d01c916b482dac937b93ede1e53b1db0361c) Conflicts: Lib/test/test_configparser.py --- Doc/library/configparser.rst | 20 +++++++++++++------- Lib/configparser.py | 5 ++++- Lib/test/test_configparser.py | 12 +++++++++++- Misc/NEWS | 3 +++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e09562dc9046c7..af57cba2b5661d 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -983,13 +983,16 @@ ConfigParser Objects .. method:: read(filenames, encoding=None) Attempt to read and parse a list of filenames, returning a list of - filenames which were successfully parsed. If *filenames* is a string, it - is treated as a single filename. If a file named in *filenames* cannot - be opened, that file will be ignored. This is designed so that you can - specify a list of potential configuration file locations (for example, - the current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the list will be - read. If none of the named files exist, the :class:`ConfigParser` + filenames which were successfully parsed. + + If *filenames* is a string or :term:`path-like object`, it is treated as + a single filename. If a file named in *filenames* cannot be opened, that + file will be ignored. This is designed so that you can specify a list of + potential configuration file locations (for example, the current + directory, the user's home directory, and some system-wide directory), + and all existing configuration files in the list will be read. + + If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires initial values to be loaded from a file should load the required file or files using :meth:`read_file` before calling :meth:`read` for any @@ -1006,6 +1009,9 @@ ConfigParser Objects The *encoding* parameter. Previously, all files were read using the default encoding for :func:`open`. + .. versionadded:: 3.6.1 + The *filenames* parameter accepts a :term:`path-like object`. + .. method:: read_file(f, source=None) diff --git a/Lib/configparser.py b/Lib/configparser.py index af5aca1feae34a..230ab2b017eade 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -143,6 +143,7 @@ import functools import io import itertools +import os import re import sys import warnings @@ -687,7 +688,7 @@ def read(self, filenames, encoding=None): Return list of successfully read files. """ - if isinstance(filenames, str): + if isinstance(filenames, (str, os.PathLike)): filenames = [filenames] read_ok = [] for filename in filenames: @@ -696,6 +697,8 @@ def read(self, filenames, encoding=None): self._read(fp, filename) except OSError: continue + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) read_ok.append(filename) return read_ok diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 0d06080da33c52..72c3f19fb41f56 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,7 @@ import configparser import io import os -import sys +import pathlib import textwrap import unittest import warnings @@ -721,6 +721,16 @@ def test_read_returns_file_list(self): parsed_files = cf.read(file1) self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we pass only a Path object: + cf = self.newconfig() + parsed_files = cf.read(pathlib.Path(file1)) + self.assertEqual(parsed_files, [file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we passed both a filename and a Path object: + cf = self.newconfig() + parsed_files = cf.read([pathlib.Path(file1), file1]) + self.assertEqual(parsed_files, [file1, file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() parsed_files = cf.read(["nonexistent-file"]) diff --git a/Misc/NEWS b/Misc/NEWS index 2692043b6cbab6..692dc189f4f1fd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -82,6 +82,9 @@ Extension Modules Library ------- +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + - bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback implemented in C. From 9d07aceedabcdc9826489f8b9baffff056283bb3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Mar 2017 12:58:17 -0800 Subject: [PATCH 0078/1345] bpo-29455: Mention coverage.py in trace module documentation (GH-435) (cherry picked from commit 5dfccb06dc513ae67fac5fee66356ad58a4de170) --- Doc/library/trace.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b0ac81271c5c18..5cb7029adf5e9e 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -13,6 +13,12 @@ annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. +.. seealso:: + + `Coverage.py `_ + A popular third-party coverage tool that provides HTML + output along with advanced features such as branch coverage. + .. _trace-cli: Command-Line Usage From 1936ba93c900d47d0c8c915184ac2fa7773c952e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 3 Mar 2017 13:24:13 -0800 Subject: [PATCH 0079/1345] bpo-29709: Improve Boolean Operations documentation (#433) (#437) (cherry picked from commit 8eb531d9db0861e14222445fcaebe1a373bba170) --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9a4f42caa408a5..6729a20a9c2ecc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -108,11 +108,11 @@ Notes: (1) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`False`. + argument if the first one is false. (2) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`True`. + argument if the first one is true. (3) ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is From 03f7cb060414b17d88cb406f3e09f26c902d8658 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 3 Mar 2017 16:07:25 -0600 Subject: [PATCH 0080/1345] bpo-29572: Update Windows build to OpenSSL 1.0.2k (GH-442) --- Misc/NEWS | 2 ++ PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 692dc189f4f1fd..2bdf5403c3e1be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -276,6 +276,8 @@ Tests Build ----- +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - Issue #26851: Set Android compilation and link flags. - Issue #28768: Fix implicit declaration of function _setmode. Patch by diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a5185becee5a96..98a755d870ad90 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2j +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 diff --git a/PCbuild/python.props b/PCbuild/python.props index dde94f733a13ae..d6bfd0877e09df 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -45,7 +45,7 @@ $(ExternalsDir)sqlite-3.14.2.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2j\ + $(ExternalsDir)openssl-1.0.2k\ $(opensslDir)include32 $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c04ba4e46b9894..8924b3fcfa2e80 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: http://tukaani.org/xz/ _ssl - Python wrapper for version 1.0.2j of the OpenSSL secure sockets + Python wrapper for version 1.0.2k of the OpenSSL secure sockets library, which is built by ssl.vcxproj Homepage: http://www.openssl.org/ From 226af23e858e2914cf78dfa6fd441c7b90a4cc91 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Mar 2017 14:47:06 -0800 Subject: [PATCH 0081/1345] bpo-26213: Document _UNPACK bytecodes and BUILD_MAP changes (GH-440) (cherry picked from commit 0705f66eb369aa6a6cdb699e24ff61e1ab2e0c56) --- Doc/library/dis.rst | 55 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a15690ba489a66..1c1e1a2aa5d4b3 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -772,8 +772,13 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_MAP (count) - Pushes a new dictionary object onto the stack. The dictionary is pre-sized - to hold *count* entries. + Pushes a new dictionary object onto the stack. Pops ``2 * count`` items + so that the dictionary holds *count* entries: + ``{..., TOS3: TOS2, TOS1: TOS}``. + + .. versionchanged:: 3.5 + The dictionary is created from stack items instead of creating an + empty dictionary pre-sized to hold *count* items. .. opcode:: BUILD_CONST_KEY_MAP (count) @@ -793,6 +798,52 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: BUILD_TUPLE_UNPACK (count) + + Pops *count* iterables from the stack, joins them in a single tuple, + and pushes the result. Implements iterable unpacking in tuple + displays ``(*x, *y, *z)``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_LIST_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list + instead of tuple. Implements iterable unpacking in list + displays ``[*x, *y, *z]``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_SET_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set + instead of tuple. Implements iterable unpacking in set + displays ``{*x, *y, *z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK (count) + + Pops *count* mappings from the stack, merges them into a single dictionary, + and pushes the result. Implements dictionary unpacking in dictionary + displays ``{**x, **y, **z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (oparg) + + This is similar to :opcode:`BUILD_MAP_UNPACK`, + but is used for ``f(**x, **y, **z)`` call syntax. The lowest byte of + *oparg* is the count of mappings, the relative position of the + corresponding callable ``f`` is encoded in the second byte of *oparg*. + + .. versionadded:: 3.5 + + .. opcode:: LOAD_ATTR (namei) Replaces TOS with ``getattr(TOS, co_names[namei])``. From d3e1e9df724d97ab83113c2d5fa15179d1dcd560 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 3 Mar 2017 21:32:17 -0800 Subject: [PATCH 0082/1345] bpo-27593: Updates Windows build to use information from git (#262) (#450) * bpo-27593: Updates Windows build to use information from git --- PCbuild/build.bat | 8 ++++---- PCbuild/pythoncore.vcxproj | 24 ++++++++++++------------ Tools/msi/buildrelease.bat | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 70ab340905e4c9..f7f2858d7d9d99 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -105,9 +105,9 @@ if "%platf%"=="x64" ( ) ) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if exist "%HG%" set HGProperty=/p:HG="%HG%" -if not exist "%HG%" echo Cannot find Mercurial on PATH & set HGProperty= +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment call "%dir%env.bat" %vs_platf% >nul @@ -145,7 +145,7 @@ msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %HGProperty%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6b23d8ecacce3b..8f8bede8a8818e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -407,24 +407,24 @@ - hg - <_HG>$(HG) - <_HG Condition="$(HG.Contains(` `))">"$(HG)" + git + <_GIT>$(GIT) + <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" - + - - - + + + - $([System.IO.File]::ReadAllText('$(IntDir)hgbranch.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgversion.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgtag.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) - + - HGVERSION="$(HgVersion)";HGTAG="$(HgTag)";HGBRANCH="$(HgBranch)";%(PreprocessorDefinitions) + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 4659a32b838cc1..81a3f86cab36b0 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -65,8 +65,8 @@ if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if not exist "%HG%" echo Cannot find Mercurial on PATH && exit /B 1 +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" From 793f822068335cff68f25304a915c726f3f3a599 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 3 Mar 2017 21:57:47 -0800 Subject: [PATCH 0083/1345] Fixes git command (#451) (#452) --- PCbuild/pythoncore.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8f8bede8a8818e..d7e9473638cbb2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -415,7 +415,7 @@ - + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) From 95c50e5aed9e5683676e18349dd94b11901a66b3 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 01:05:06 -0500 Subject: [PATCH 0084/1345] [3.6] bpo-27593: Get SCM build info from git instead of hg. (#446) (#454) * bpo-27593: Get SCM build info from git instead of hg. (#446) sys.version and the platform module python_build(), python_branch(), and python_revision() functions now use git information rather than hg when building from a repo. Based on original patches by Brett Cannon and Steve Dower. (cherry picked from commit 5c4b0d063aba0a68c325073f5f312a2c9f40d178) --- Include/pylifecycle.h | 4 ++-- Lib/platform.py | 4 +++- Lib/test/test_platform.py | 12 +++++----- Makefile.pre.in | 12 +++++----- Misc/NEWS | 6 +++++ Modules/getbuildinfo.c | 46 ++++++++++++++++++------------------- Python/sysmodule.c | 6 ++--- configure | 48 +++++++++++++++++++-------------------- configure.ac | 26 ++++++++++----------- 9 files changed, 86 insertions(+), 78 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5a67666874da94..01abfa9fcd6fb7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -70,8 +70,8 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void); PyAPI_FUNC(const char *) Py_GetCompiler(void); PyAPI_FUNC(const char *) Py_GetBuildInfo(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_hgidentifier(void); -PyAPI_FUNC(const char *) _Py_hgversion(void); +PyAPI_FUNC(const char *) _Py_gitidentifier(void); +PyAPI_FUNC(const char *) _Py_gitversion(void); #endif /* Internal -- various one-time initializations */ diff --git a/Lib/platform.py b/Lib/platform.py index 062347537531ba..cc2db9870d84b6 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1198,7 +1198,9 @@ def _sys_version(sys_version=None): elif buildtime: builddate = builddate + ' ' + buildtime - if hasattr(sys, '_mercurial'): + if hasattr(sys, '_git'): + _, branch, revision = sys._git + elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial elif hasattr(sys, 'subversion'): # sys.subversion was added in Python 2.5 diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5f875eff6c7ede..ea6752b61d8f77 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -67,12 +67,12 @@ def test_processor(self): def setUp(self): self.save_version = sys.version - self.save_mercurial = sys._mercurial + self.save_git = sys._git self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._mercurial = self.save_mercurial + sys._git = self.save_git sys.platform = self.save_platform def test_sys_version(self): @@ -102,7 +102,7 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ): # branch and revision are not "parsed", but fetched - # from sys._mercurial. Ignore them + # from sys._git. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -149,10 +149,10 @@ def test_sys_version(self): sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_mercurial"): - del sys._mercurial + if hasattr(sys, "_git"): + del sys._git else: - sys._mercurial = subversion + sys._git = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Makefile.pre.in b/Makefile.pre.in index 93aae9124726bd..8f27d7320c1d69 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,9 +41,9 @@ RANLIB= @RANLIB@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -HGVERSION= @HGVERSION@ -HGTAG= @HGTAG@ -HGBRANCH= @HGBRANCH@ +GITVERSION= @GITVERSION@ +GITTAG= @GITTAG@ +GITBRANCH= @GITBRANCH@ PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ @@ -740,9 +740,9 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ - -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \ - -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \ - -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile diff --git a/Misc/NEWS b/Misc/NEWS index 2bdf5403c3e1be..debadc1e54c141 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,8 @@ Core and Builtins - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. @@ -276,6 +278,10 @@ Tests Build ----- +- bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. + - bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. - Issue #26851: Set Android compilation and link flags. diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 0971a64fccbadc..5f941a26e1d54c 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -21,47 +21,47 @@ #endif /* XXX Only unix build process has been tested */ -#ifndef HGVERSION -#define HGVERSION "" +#ifndef GITVERSION +#define GITVERSION "" #endif -#ifndef HGTAG -#define HGTAG "" +#ifndef GITTAG +#define GITTAG "" #endif -#ifndef HGBRANCH -#define HGBRANCH "" +#ifndef GITBRANCH +#define GITBRANCH "" #endif const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(HGVERSION) + - ((sizeof(HGTAG) > sizeof(HGBRANCH)) ? - sizeof(HGTAG) : sizeof(HGBRANCH))]; - const char *revision = _Py_hgversion(); + static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; - const char *hgid = _Py_hgidentifier(); - if (!(*hgid)) - hgid = "default"; + const char *gitid = _Py_gitidentifier(); + if (!(*gitid)) + gitid = "default"; PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", hgid, sep, revision, + "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); return buildinfo; } const char * -_Py_hgversion(void) +_Py_gitversion(void) { - return HGVERSION; + return GITVERSION; } const char * -_Py_hgidentifier(void) +_Py_gitidentifier(void) { - const char *hgtag, *hgid; - hgtag = HGTAG; - if ((*hgtag) && strcmp(hgtag, "tip") != 0) - hgid = hgtag; + const char *gittag, *gitid; + gittag = GITTAG; + if ((*gittag) && strcmp(gittag, "undefined") != 0) + gitid = gittag; else - hgid = HGBRANCH; - return hgid; + gitid = GITBRANCH; + return gitid; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 52034ff7fb5ddf..99e6b5ea018ea1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1942,9 +1942,9 @@ _PySys_Init(void) PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", PyLong_FromLong(PY_VERSION_HEX)); - SET_SYS_FROM_STRING("_mercurial", - Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), - _Py_hgversion())); + SET_SYS_FROM_STRING("_git", + Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), + _Py_gitversion())); SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", diff --git a/configure b/configure index 1500cea4a8292b..d4eccb1d68d856 100755 --- a/configure +++ b/configure @@ -760,10 +760,10 @@ build_os build_vendor build_cpu build -HAS_HG -HGBRANCH -HGTAG -HGVERSION +HAS_GIT +GITBRANCH +GITTAG +GITVERSION BASECPPFLAGS target_alias host_alias @@ -2698,17 +2698,17 @@ fi -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -# Extract the first word of "hg", so it can be a program name with args. -set dummy hg; ac_word=$2 +# Extract the first word of "git", so it can be a program name with args. +set dummy git; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_HAS_HG+:} false; then : +if ${ac_cv_prog_HAS_GIT+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$HAS_HG"; then - ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. + if test -n "$HAS_GIT"; then + ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2717,7 +2717,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_HAS_HG="found" + ac_cv_prog_HAS_GIT="found" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2725,13 +2725,13 @@ done done IFS=$as_save_IFS - test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" + test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" fi fi -HAS_HG=$ac_cv_prog_HAS_HG -if test -n "$HAS_HG"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 -$as_echo "$HAS_HG" >&6; } +HAS_GIT=$ac_cv_prog_HAS_GIT +if test -n "$HAS_GIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_GIT" >&5 +$as_echo "$HAS_GIT" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } @@ -2739,17 +2739,17 @@ fi else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse HEAD" + GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi diff --git a/configure.ac b/configure.ac index 64f12f28790a7d..f00a2a6fa498f7 100644 --- a/configure.ac +++ b/configure.ac @@ -25,25 +25,25 @@ else BASECPPFLAGS="" fi -AC_SUBST(HGVERSION) -AC_SUBST(HGTAG) -AC_SUBST(HGBRANCH) +AC_SUBST(GITVERSION) +AC_SUBST(GITTAG) +AC_SUBST(GITBRANCH) -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -AC_CHECK_PROG(HAS_HG, hg, found, not-found) +AC_CHECK_PROG(HAS_GIT, git, found, not-found) else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse HEAD" + GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi AC_CONFIG_SRCDIR([Include/object.h]) From 5ddddb166d9979161624c17cce3bdc11e1d9f3b8 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 02:55:09 -0500 Subject: [PATCH 0085/1345] bpo-29550: Temporarily skip "make touch" in Mac installer build. (#456) (#458) (cherry picked from commit 7cd9b22e98a5b7a99a7539500ea946a7a58e1c8d) --- Mac/BuildScript/build-installer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8dfd092d08b512..5649911c24c505 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of hg and a third-party version of +requires an installed version of git and a third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), hg, and for the documentation +(2.5 is needed for Python parser updates), git, and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -635,9 +635,9 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to hg and to sphinx-build. + # Ensure ws have access to git and to sphinx-build. # You may have to create links in /usr/bin for them. - runCommand('hg --version') + runCommand('git --version') runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1142,8 +1142,9 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make touch") - runCommand("make touch") + # bpo-29550: avoid using make touch until it is fixed for git + # print("Running make touch") + # runCommand("make touch") print("Running make") runCommand("make") From 343d48379c31f31cb0bcc843c537990c4283703d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 03:04:48 -0500 Subject: [PATCH 0086/1345] bpo-29572: Update macOS installer build to OpenSSL 1.0.2k (#457) (#459) (cherry picked from commit cfcd76777e35c83d548d8736f5d7dc92fe56d806) --- Mac/BuildScript/build-installer.py | 6 +++--- Mac/BuildScript/openssl_sdk_makedepend.patch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5649911c24c505..f59ae34df2549a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2j", - url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz", - checksum='96322138f0b69e61b7212bc53d5e912b', + name="OpenSSL 1.0.2k", + url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", + checksum='f965fc0bf01bf882b31314b61391ae65', patches=[ "openssl_sdk_makedepend.patch", ], diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch index e22d67e4e45143..0caac0a64c1ec6 100644 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ b/Mac/BuildScript/openssl_sdk_makedepend.patch @@ -1,6 +1,6 @@ # HG changeset patch # -# using openssl 1.0.2j +# using openssl 1.0.2k # # - support building with an OS X SDK From 1d391f926b37484b8d4b326003a72c0084db19ec Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 06:19:05 -0500 Subject: [PATCH 0087/1345] [3.6] bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS (#463) Skip some tests of select.poll when running on macOS due to unresolved issues with the underlying system poll function on some macOS versions. (cherry picked from commit de04644627f82d9dc48b3423def7ff5b4aa1926a) --- Lib/test/eintrdata/eintr_tester.py | 2 ++ Lib/test/test_asyncore.py | 3 +++ Misc/NEWS | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index d194e775eaeb8f..c619b3d6c38a59 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -437,6 +437,8 @@ def test_select(self): self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') def test_poll(self): poller = select.poll() diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 51c65737a441de..d05462b7efce35 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -661,6 +661,9 @@ def test_handle_expt(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + class TestClient(BaseClient): def handle_expt(self): self.socket.recv(1024, socket.MSG_OOB) diff --git a/Misc/NEWS b/Misc/NEWS index debadc1e54c141..9aee4974344e10 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -261,6 +261,10 @@ Documentation Tests ----- +- bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS. + Skip some tests of select.poll when running on macOS due to unresolved + issues with the underlying system poll function on some macOS versions. + - Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to determine the candidate encoding for the test regex (allowing it to correctly From 5976b9a88b8510789ed6e1679cc3578f7b345eb1 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 07:04:41 -0500 Subject: [PATCH 0088/1345] Refresh Python/importlib.h and Python/importlib_external.h (#464) --- Python/importlib.h | 2 +- Python/importlib_external.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/importlib.h b/Python/importlib.h index 195fbfd13b6227..4607c40de60e7c 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -392,7 +392,7 @@ const unsigned char _Py_M__importlib[] = { 111,118,101,100,197,0,0,0,115,2,0,0,0,0,8,114, 58,0,0,0,114,33,0,0,0,41,1,218,9,118,101,114, 98,111,115,105,116,121,99,1,0,0,0,1,0,0,0,3, - 0,0,0,5,0,0,0,71,0,0,0,115,54,0,0,0, + 0,0,0,4,0,0,0,71,0,0,0,115,54,0,0,0, 116,0,106,1,106,2,124,1,107,5,114,50,124,0,106,3, 100,6,131,1,115,30,100,3,124,0,23,0,125,0,116,4, 124,0,106,5,124,2,142,0,116,0,106,6,100,4,141,2, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 2f40978fb78e13..886aa68009ec03 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -578,7 +578,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,17,95,102,105,110,100,95,109,111,100,117,108,101,95,115, 104,105,109,157,1,0,0,115,10,0,0,0,0,10,14,1, 16,1,4,1,22,1,114,127,0,0,0,99,4,0,0,0, - 0,0,0,0,11,0,0,0,22,0,0,0,67,0,0,0, + 0,0,0,0,11,0,0,0,19,0,0,0,67,0,0,0, 115,136,1,0,0,105,0,125,4,124,2,100,1,107,9,114, 22,124,2,124,4,100,2,60,0,110,4,100,3,125,2,124, 3,100,1,107,9,114,42,124,3,124,4,100,4,60,0,124, From cefca3d7b5ee3f8421c082ccf24758fcb641fb6c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 12:18:43 -0500 Subject: [PATCH 0089/1345] Update pydoc topics for 3.6.0rc1 --- Lib/pydoc_data/topics.py | 370 ++++++++++++++++++--------------------- 1 file changed, 171 insertions(+), 199 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index c7fac3395b3be0..2caab630978fa0 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 6 18:51:51 2016 -topics = {'assert': '\n' - 'The "assert" statement\n' +# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' 'Assert statements are a convenient way to insert debugging ' @@ -39,8 +38,7 @@ 'Assignments to "__debug__" are illegal. The value for the ' 'built-in\n' 'variable is determined when the interpreter starts.\n', - 'assignment': '\n' - 'Assignment statements\n' + 'assignment': 'Assignment statements\n' '*********************\n' '\n' 'Assignment statements are used to (re)bind names to values and ' @@ -405,8 +403,7 @@ 'See also: **PEP 526** - Variable and attribute annotation ' 'syntax\n' ' **PEP 484** - Type hints\n', - 'atom-identifiers': '\n' - 'Identifiers (Names)\n' + 'atom-identifiers': 'Identifiers (Names)\n' '*******************\n' '\n' 'An identifier occurring as an atom is a name. See ' @@ -446,8 +443,7 @@ 'happen. If the class name consists only of underscores, ' 'no\n' 'transformation is done.\n', - 'atom-literals': '\n' - 'Literals\n' + 'atom-literals': 'Literals\n' '********\n' '\n' 'Python supports string and bytes literals and various ' @@ -476,8 +472,7 @@ 'may obtain\n' 'the same object or a different object with the same ' 'value.\n', - 'attribute-access': '\n' - 'Customizing attribute access\n' + 'attribute-access': 'Customizing attribute access\n' '****************************\n' '\n' 'The following methods can be defined to customize the ' @@ -851,8 +846,7 @@ '* *__class__* assignment works only if both classes have ' 'the same\n' ' *__slots__*.\n', - 'attribute-references': '\n' - 'Attribute references\n' + 'attribute-references': 'Attribute references\n' '********************\n' '\n' 'An attribute reference is a primary followed by a ' @@ -875,8 +869,7 @@ 'determined by the object. Multiple evaluations of ' 'the same attribute\n' 'reference may yield different objects.\n', - 'augassign': '\n' - 'Augmented assignment statements\n' + 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' 'Augmented assignment is the combination, in a single statement, ' @@ -940,8 +933,7 @@ 'about\n' 'class and instance attributes applies as for regular ' 'assignments.\n', - 'binary': '\n' - 'Binary arithmetic operations\n' + 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' 'The binary arithmetic operations have the conventional priority\n' @@ -1029,8 +1021,7 @@ 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' 'The numeric arguments are first converted to a common type.\n', - 'bitwise': '\n' - 'Binary bitwise operations\n' + 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' 'Each of the three bitwise operations has a different priority ' @@ -1050,8 +1041,7 @@ 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' 'which must be integers.\n', - 'bltin-code-objects': '\n' - 'Code Objects\n' + 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' 'Code objects are used by the implementation to ' @@ -1074,8 +1064,7 @@ '\n' 'See The standard type hierarchy for more ' 'information.\n', - 'bltin-ellipsis-object': '\n' - 'The Ellipsis Object\n' + 'bltin-ellipsis-object': 'The Ellipsis Object\n' '*******************\n' '\n' 'This object is commonly used by slicing (see ' @@ -1087,8 +1076,7 @@ '"Ellipsis" singleton.\n' '\n' 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': '\n' - 'The Null Object\n' + 'bltin-null-object': 'The Null Object\n' '***************\n' '\n' "This object is returned by functions that don't " @@ -1100,8 +1088,7 @@ 'same singleton.\n' '\n' 'It is written as "None".\n', - 'bltin-type-objects': '\n' - 'Type Objects\n' + 'bltin-type-objects': 'Type Objects\n' '************\n' '\n' 'Type objects represent the various object types. An ' @@ -1113,8 +1100,7 @@ 'all standard built-in types.\n' '\n' 'Types are written like this: "".\n', - 'booleans': '\n' - 'Boolean operations\n' + 'booleans': 'Boolean operations\n' '******************\n' '\n' ' or_test ::= and_test | or_test "or" and_test\n' @@ -1163,8 +1149,7 @@ 'its\n' 'argument (for example, "not \'foo\'" produces "False" rather ' 'than "\'\'".)\n', - 'break': '\n' - 'The "break" statement\n' + 'break': 'The "break" statement\n' '*********************\n' '\n' ' break_stmt ::= "break"\n' @@ -1185,8 +1170,7 @@ 'clause, that "finally" clause is executed before really leaving ' 'the\n' 'loop.\n', - 'callable-types': '\n' - 'Emulating callable objects\n' + 'callable-types': 'Emulating callable objects\n' '**************************\n' '\n' 'object.__call__(self[, args...])\n' @@ -1195,8 +1179,7 @@ 'this method\n' ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': '\n' - 'Calls\n' + 'calls': 'Calls\n' '*****\n' '\n' 'A call calls a callable object (e.g., a *function*) with a ' @@ -1217,7 +1200,8 @@ ' ("," "*" expression | "," ' 'keyword_item)*\n' ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "**" expression)*\n' + ' ("," keyword_item | "," "**" ' + 'expression)*\n' ' keyword_item ::= identifier "=" expression\n' '\n' 'An optional trailing comma may be present after the positional and\n' @@ -1382,8 +1366,7 @@ ' The class must define a "__call__()" method; the effect is then ' 'the\n' ' same as if that method was called.\n', - 'class': '\n' - 'Class definitions\n' + 'class': 'Class definitions\n' '*****************\n' '\n' 'A class definition defines a class object (see section The ' @@ -1469,8 +1452,7 @@ '\n' 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' ' Class Decorators\n', - 'comparisons': '\n' - 'Comparisons\n' + 'comparisons': 'Comparisons\n' '***********\n' '\n' 'Unlike C, all comparison operations in Python have the same ' @@ -1623,7 +1605,7 @@ 'restriction that\n' ' ranges do not support order comparison. Equality ' 'comparison across\n' - ' these types results in unequality, and ordering comparison ' + ' these types results in inequality, and ordering comparison ' 'across\n' ' these types raises "TypeError".\n' '\n' @@ -1762,6 +1744,12 @@ ' to sequences, but not to sets or mappings). See also the\n' ' "total_ordering()" decorator.\n' '\n' + '* The "hash()" result should be consistent with equality. ' + 'Objects\n' + ' that are equal should either have the same hash value, or ' + 'be marked\n' + ' as unhashable.\n' + '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' 'not-a-number values are an example for not following these ' @@ -1833,8 +1821,7 @@ 'is determined using the "id()" function. "x is not y" yields ' 'the\n' 'inverse truth value. [4]\n', - 'compound': '\n' - 'Compound statements\n' + 'compound': 'Compound statements\n' '*******************\n' '\n' 'Compound statements contain (groups of) other statements; they ' @@ -2613,7 +2600,8 @@ 'functions, even if they do not contain "await" or "async" ' 'keywords.\n' '\n' - 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'It is a "SyntaxError" to use "yield from" expressions in "async ' + 'def"\n' 'coroutines.\n' '\n' 'An example of a coroutine function:\n' @@ -2724,8 +2712,7 @@ ' body is transformed into the namespace\'s "__doc__" item ' 'and\n' " therefore the class's *docstring*.\n", - 'context-managers': '\n' - 'With Statement Context Managers\n' + 'context-managers': 'With Statement Context Managers\n' '*******************************\n' '\n' 'A *context manager* is an object that defines the ' @@ -2787,8 +2774,7 @@ ' The specification, background, and examples for the ' 'Python "with"\n' ' statement.\n', - 'continue': '\n' - 'The "continue" statement\n' + 'continue': 'The "continue" statement\n' '************************\n' '\n' ' continue_stmt ::= "continue"\n' @@ -2805,8 +2791,7 @@ '"finally" clause, that "finally" clause is executed before ' 'really\n' 'starting the next loop cycle.\n', - 'conversions': '\n' - 'Arithmetic conversions\n' + 'conversions': 'Arithmetic conversions\n' '**********************\n' '\n' 'When a description of an arithmetic operator below uses the ' @@ -2832,8 +2817,7 @@ "left argument to the '%' operator). Extensions must define " 'their own\n' 'conversion behavior.\n', - 'customization': '\n' - 'Basic customization\n' + 'customization': 'Basic customization\n' '*******************\n' '\n' 'object.__new__(cls[, ...])\n' @@ -3152,15 +3136,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only ' - 'required property\n' + ' "__hash__()" should return an integer. The only required ' + 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive ' - 'or) the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the ' + 'components of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -3272,8 +3259,7 @@ ' neither "__len__()" nor "__bool__()", all its instances ' 'are\n' ' considered true.\n', - 'debugger': '\n' - '"pdb" --- The Python Debugger\n' + 'debugger': '"pdb" --- The Python Debugger\n' '*****************************\n' '\n' '**Source code:** Lib/pdb.py\n' @@ -3938,8 +3924,7 @@ '[1] Whether a frame is considered to originate in a certain ' 'module\n' ' is determined by the "__name__" in the frame globals.\n', - 'del': '\n' - 'The "del" statement\n' + 'del': 'The "del" statement\n' '*******************\n' '\n' ' del_stmt ::= "del" target_list\n' @@ -3968,8 +3953,7 @@ 'Changed in version 3.2: Previously it was illegal to delete a name\n' 'from the local namespace if it occurs as a free variable in a nested\n' 'block.\n', - 'dict': '\n' - 'Dictionary displays\n' + 'dict': 'Dictionary displays\n' '*******************\n' '\n' 'A dictionary display is a possibly empty series of key/datum pairs\n' @@ -4013,8 +3997,7 @@ 'should be *hashable*, which excludes all mutable objects.) Clashes\n' 'between duplicate keys are not detected; the last datum (textually\n' 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': '\n' - 'Interaction with dynamic features\n' + 'dynamic-features': 'Interaction with dynamic features\n' '*********************************\n' '\n' 'Name resolution of free variables occurs at runtime, not ' @@ -4050,8 +4033,7 @@ 'override the global and local namespace. If only one ' 'namespace is\n' 'specified, it is used for both.\n', - 'else': '\n' - 'The "if" statement\n' + 'else': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -4068,8 +4050,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'exceptions': '\n' - 'Exceptions\n' + 'exceptions': 'Exceptions\n' '**********\n' '\n' 'Exceptions are a means of breaking out of the normal flow of ' @@ -4145,8 +4126,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'execmodel': '\n' - 'Execution model\n' + 'execmodel': 'Execution model\n' '***************\n' '\n' '\n' @@ -4477,8 +4457,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'exprlists': '\n' - 'Expression lists\n' + 'exprlists': 'Expression lists\n' '****************\n' '\n' ' expression_list ::= expression ( "," expression )* [","]\n' @@ -4515,8 +4494,7 @@ 'value of that expression. (To create an empty tuple, use an ' 'empty pair\n' 'of parentheses: "()".)\n', - 'floating': '\n' - 'Floating point literals\n' + 'floating': 'Floating point literals\n' '***********************\n' '\n' 'Floating point literals are described by the following lexical\n' @@ -4552,8 +4530,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'for': '\n' - 'The "for" statement\n' + 'for': 'The "for" statement\n' '*******************\n' '\n' 'The "for" statement is used to iterate over the elements of a ' @@ -4625,8 +4602,7 @@ '\n' ' for x in a[:]:\n' ' if x < 0: a.remove(x)\n', - 'formatstrings': '\n' - 'Format String Syntax\n' + 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' 'The "str.format()" method and the "Formatter" class share ' @@ -5345,8 +5321,7 @@ ' 9 9 11 1001\n' ' 10 A 12 1010\n' ' 11 B 13 1011\n', - 'function': '\n' - 'Function definitions\n' + 'function': 'Function definitions\n' '********************\n' '\n' 'A function definition defines a user-defined function object ' @@ -5515,8 +5490,7 @@ '\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n', - 'global': '\n' - 'The "global" statement\n' + 'global': 'The "global" statement\n' '**********************\n' '\n' ' global_stmt ::= "global" identifier ("," identifier)*\n' @@ -5560,8 +5534,7 @@ 'code containing the function call. The same applies to the ' '"eval()"\n' 'and "compile()" functions.\n', - 'id-classes': '\n' - 'Reserved classes of identifiers\n' + 'id-classes': 'Reserved classes of identifiers\n' '*******************************\n' '\n' 'Certain classes of identifiers (besides keywords) have ' @@ -5609,8 +5582,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'identifiers': '\n' - 'Identifiers and keywords\n' + 'identifiers': 'Identifiers and keywords\n' '************************\n' '\n' 'Identifiers (also referred to as *names*) are described by ' @@ -5758,8 +5730,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'if': '\n' - 'The "if" statement\n' + 'if': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -5775,8 +5746,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'imaginary': '\n' - 'Imaginary literals\n' + 'imaginary': 'Imaginary literals\n' '******************\n' '\n' 'Imaginary literals are described by the following lexical ' @@ -5796,8 +5766,7 @@ '\n' ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' '3.14_15_93j\n', - 'import': '\n' - 'The "import" statement\n' + 'import': 'The "import" statement\n' '**********************\n' '\n' ' import_stmt ::= "import" module ["as" name] ( "," module ' @@ -6058,8 +6027,7 @@ '\n' ' **PEP 236** - Back to the __future__\n' ' The original proposal for the __future__ mechanism.\n', - 'in': '\n' - 'Membership test operations\n' + 'in': 'Membership test operations\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' @@ -6094,8 +6062,7 @@ '\n' 'The operator "not in" is defined to have the inverse true value of\n' '"in".\n', - 'integers': '\n' - 'Integer literals\n' + 'integers': 'Integer literals\n' '****************\n' '\n' 'Integer literals are described by the following lexical ' @@ -6141,8 +6108,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'lambda': '\n' - 'Lambdas\n' + 'lambda': 'Lambdas\n' '*******\n' '\n' ' lambda_expr ::= "lambda" [parameter_list]: expression\n' @@ -6165,8 +6131,7 @@ 'Note that functions created with lambda expressions cannot ' 'contain\n' 'statements or annotations.\n', - 'lists': '\n' - 'List displays\n' + 'lists': 'List displays\n' '*************\n' '\n' 'A list display is a possibly empty series of expressions enclosed ' @@ -6183,8 +6148,7 @@ 'from left to right and placed into the list object in that order.\n' 'When a comprehension is supplied, the list is constructed from the\n' 'elements resulting from the comprehension.\n', - 'naming': '\n' - 'Naming and binding\n' + 'naming': 'Naming and binding\n' '******************\n' '\n' '\n' @@ -6397,8 +6361,7 @@ 'override the global and local namespace. If only one namespace ' 'is\n' 'specified, it is used for both.\n', - 'nonlocal': '\n' - 'The "nonlocal" statement\n' + 'nonlocal': 'The "nonlocal" statement\n' '************************\n' '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' @@ -6429,8 +6392,7 @@ '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' ' The specification for the "nonlocal" statement.\n', - 'numbers': '\n' - 'Numeric literals\n' + 'numbers': 'Numeric literals\n' '****************\n' '\n' 'There are three types of numeric literals: integers, floating ' @@ -6444,8 +6406,7 @@ 'is actually an expression composed of the unary operator \'"-"\' ' 'and the\n' 'literal "1".\n', - 'numeric-types': '\n' - 'Emulating numeric types\n' + 'numeric-types': 'Emulating numeric types\n' '***********************\n' '\n' 'The following methods can be defined to emulate numeric ' @@ -6621,8 +6582,7 @@ ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n', - 'objects': '\n' - 'Objects, values and types\n' + 'objects': 'Objects, values and types\n' '*************************\n' '\n' "*Objects* are Python's abstraction for data. All data in a " @@ -6750,8 +6710,7 @@ 'created empty lists. (Note that "c = d = []" assigns the same ' 'object\n' 'to both "c" and "d".)\n', - 'operator-summary': '\n' - 'Operator precedence\n' + 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' 'The following table summarizes the operator precedence ' @@ -6924,8 +6883,7 @@ 'arithmetic\n' ' or bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', - 'pass': '\n' - 'The "pass" statement\n' + 'pass': 'The "pass" statement\n' '********************\n' '\n' ' pass_stmt ::= "pass"\n' @@ -6938,8 +6896,7 @@ ' def f(arg): pass # a function that does nothing (yet)\n' '\n' ' class C: pass # a class with no methods (yet)\n', - 'power': '\n' - 'The power operator\n' + 'power': 'The power operator\n' '******************\n' '\n' 'The power operator binds more tightly than unary operators on its\n' @@ -6973,8 +6930,7 @@ 'Raising a negative number to a fractional power results in a ' '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': '\n' - 'The "raise" statement\n' + 'raise': 'The "raise" statement\n' '*********************\n' '\n' ' raise_stmt ::= "raise" [expression ["from" expression]]\n' @@ -7059,8 +7015,7 @@ 'Exceptions, and information about handling exceptions is in ' 'section\n' 'The try statement.\n', - 'return': '\n' - 'The "return" statement\n' + 'return': 'The "return" statement\n' '**********************\n' '\n' ' return_stmt ::= "return" [expression_list]\n' @@ -7087,9 +7042,15 @@ 'generator is done and will cause "StopIteration" to be raised. ' 'The\n' 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': '\n' - 'Emulating container types\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n' + '\n' + 'In an asynchronous generator function, an empty "return" ' + 'statement\n' + 'indicates that the asynchronous generator is done and will cause\n' + '"StopAsyncIteration" to be raised. A non-empty "return" statement ' + 'is\n' + 'a syntax error in an asynchronous generator function.\n', + 'sequence-types': 'Emulating container types\n' '*************************\n' '\n' 'The following methods can be defined to implement ' @@ -7310,8 +7271,7 @@ ' iteration protocol via "__getitem__()", see this ' 'section in the\n' ' language reference.\n', - 'shifting': '\n' - 'Shifting operations\n' + 'shifting': 'Shifting operations\n' '*******************\n' '\n' 'The shifting operations have lower priority than the arithmetic\n' @@ -7335,8 +7295,7 @@ 'operand is\n' ' larger than "sys.maxsize" an "OverflowError" exception is ' 'raised.\n', - 'slicings': '\n' - 'Slicings\n' + 'slicings': 'Slicings\n' '********\n' '\n' 'A slicing selects a range of items in a sequence object (e.g., ' @@ -7387,8 +7346,7 @@ 'as lower bound, upper bound and stride, respectively, ' 'substituting\n' '"None" for missing expressions.\n', - 'specialattrs': '\n' - 'Special Attributes\n' + 'specialattrs': 'Special Attributes\n' '******************\n' '\n' 'The implementation adds a few special read-only attributes ' @@ -7473,8 +7431,7 @@ '[5] To format only a tuple you should therefore provide a\n' ' singleton tuple whose only element is the tuple to be ' 'formatted.\n', - 'specialnames': '\n' - 'Special method names\n' + 'specialnames': 'Special method names\n' '********************\n' '\n' 'A class can implement certain operations that are invoked by ' @@ -7835,15 +7792,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only required ' + ' "__hash__()" should return an integer. The only required ' 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive or) ' - 'the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the components ' + 'of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -9262,8 +9222,7 @@ 'special method *must* be set on the class object itself in ' 'order to be\n' 'consistently invoked by the interpreter).\n', - 'string-methods': '\n' - 'String Methods\n' + 'string-methods': 'String Methods\n' '**************\n' '\n' 'Strings implement all of the common sequence operations, ' @@ -9500,12 +9459,11 @@ 'characters\n' ' and there is at least one character, false otherwise. ' 'Decimal\n' - ' characters are those from general category "Nd". This ' - 'category\n' - ' includes digit characters, and all characters that can ' - 'be used to\n' - ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' - 'DIGIT ZERO.\n' + ' characters are those that can be used to form numbers ' + 'in base 10,\n' + ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' + 'decimal character\n' + ' is a character in the Unicode General Category "Nd".\n' '\n' 'str.isdigit()\n' '\n' @@ -9515,10 +9473,13 @@ 'include decimal\n' ' characters and digits that need special handling, such ' 'as the\n' - ' compatibility superscript digits. Formally, a digit is ' - 'a character\n' - ' that has the property value Numeric_Type=Digit or\n' - ' Numeric_Type=Decimal.\n' + ' compatibility superscript digits. This covers digits ' + 'which cannot\n' + ' be used to form numbers in base 10, like the Kharosthi ' + 'numbers.\n' + ' Formally, a digit is a character that has the property ' + 'value\n' + ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' @@ -10064,8 +10025,7 @@ " '00042'\n" ' >>> "-42".zfill(5)\n' " '-0042'\n", - 'strings': '\n' - 'String and Bytes literals\n' + 'strings': 'String and Bytes literals\n' '*************************\n' '\n' 'String literals are described by the following lexical ' @@ -10299,8 +10259,7 @@ 'followed by a newline is interpreted as those two characters as ' 'part\n' 'of the literal, *not* as a line continuation.\n', - 'subscriptions': '\n' - 'Subscriptions\n' + 'subscriptions': 'Subscriptions\n' '*************\n' '\n' 'A subscription selects an item of a sequence (string, tuple ' @@ -10357,8 +10316,7 @@ "A string's items are characters. A character is not a " 'separate data\n' 'type but a string of exactly one character.\n', - 'truth': '\n' - 'Truth Value Testing\n' + 'truth': 'Truth Value Testing\n' '*******************\n' '\n' 'Any object can be tested for truth value, for use in an "if" or\n' @@ -10390,8 +10348,7 @@ 'otherwise stated. (Important exception: the Boolean operations ' '"or"\n' 'and "and" always return one of their operands.)\n', - 'try': '\n' - 'The "try" statement\n' + 'try': 'The "try" statement\n' '*******************\n' '\n' 'The "try" statement specifies exception handlers and/or cleanup code\n' @@ -10538,8 +10495,7 @@ 'Exceptions, and information on using the "raise" statement to ' 'generate\n' 'exceptions may be found in section The raise statement.\n', - 'types': '\n' - 'The standard type hierarchy\n' + 'types': 'The standard type hierarchy\n' '***************************\n' '\n' 'Below is a list of the types that are built into Python. ' @@ -11097,6 +11053,27 @@ 'statements.\n' ' See also the Coroutine Objects section.\n' '\n' + ' Asynchronous generator functions\n' + ' A function or method which is defined using "async def" and\n' + ' which uses the "yield" statement is called a *asynchronous\n' + ' generator function*. Such a function, when called, returns ' + 'an\n' + ' asynchronous iterator object which can be used in an "async ' + 'for"\n' + ' statement to execute the body of the function.\n' + '\n' + ' Calling the asynchronous iterator\'s "aiterator.__anext__()"\n' + ' method will return an *awaitable* which when awaited will\n' + ' execute until it provides a value using the "yield" ' + 'expression.\n' + ' When the function executes an empty "return" statement or ' + 'falls\n' + ' off the end, a "StopAsyncIteration" exception is raised and ' + 'the\n' + ' asynchronous iterator will have reached the end of the set ' + 'of\n' + ' values to be yielded.\n' + '\n' ' Built-in functions\n' ' A built-in function object is a wrapper around a C function.\n' ' Examples of built-in functions are "len()" and "math.sin()"\n' @@ -11233,14 +11210,14 @@ 'the\n' ' dictionary containing the class\'s namespace; "__bases__" is a ' 'tuple\n' - ' (possibly empty or a singleton) containing the base classes, in ' - 'the\n' - ' order of their occurrence in the base class list; "__doc__" is ' - 'the\n' - ' class\'s documentation string, or "None" if undefined;\n' - ' "__annotations__" (optional) is a dictionary containing ' - '*variable\n' - ' annotations* collected during class body execution.\n' + ' containing the base classes, in the order of their occurrence ' + 'in\n' + ' the base class list; "__doc__" is the class\'s documentation ' + 'string,\n' + ' or "None" if undefined; "__annotations__" (optional) is a\n' + ' dictionary containing *variable annotations* collected during ' + 'class\n' + ' body execution.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -11520,8 +11497,7 @@ ' under "User-defined methods". Class method objects are ' 'created\n' ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': '\n' - 'Functions\n' + 'typesfunctions': 'Functions\n' '*********\n' '\n' 'Function objects are created by function definitions. The ' @@ -11538,8 +11514,7 @@ 'different object types.\n' '\n' 'See Function definitions for more information.\n', - 'typesmapping': '\n' - 'Mapping Types --- "dict"\n' + 'typesmapping': 'Mapping Types --- "dict"\n' '************************\n' '\n' 'A *mapping* object maps *hashable* values to arbitrary ' @@ -11896,8 +11871,7 @@ " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': '\n' - 'Methods\n' + 'typesmethods': 'Methods\n' '*******\n' '\n' 'Methods are functions that are called using the attribute ' @@ -11954,8 +11928,7 @@ " 'my name is method'\n" '\n' 'See The standard type hierarchy for more information.\n', - 'typesmodules': '\n' - 'Modules\n' + 'typesmodules': 'Modules\n' '*******\n' '\n' 'The only special operation on a module is attribute access: ' @@ -11992,8 +11965,7 @@ 'written as\n' '"".\n', - 'typesseq': '\n' - 'Sequence Types --- "list", "tuple", "range"\n' + 'typesseq': 'Sequence Types --- "list", "tuple", "range"\n' '*******************************************\n' '\n' 'There are three basic sequence types: lists, tuples, and range\n' @@ -12141,9 +12113,9 @@ '\n' '3. If *i* or *j* is negative, the index is relative to the end ' 'of\n' - ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' - 'note\n' - ' that "-0" is still "0".\n' + ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' + 'But\n' + ' note that "-0" is still "0".\n' '\n' '4. The slice of *s* from *i* to *j* is defined as the sequence ' 'of\n' @@ -12162,12 +12134,17 @@ ' (j-i)/k". In other words, the indices are "i", "i+k", ' '"i+2*k",\n' ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). If *i* or *j* is greater than "len(s)", use\n' - ' "len(s)". If *i* or *j* are omitted or "None", they become ' - '"end"\n' - ' values (which end depends on the sign of *k*). Note, *k* ' - 'cannot be\n' - ' zero. If *k* is "None", it is treated like "1".\n' + ' including *j*). When *k* is positive, *i* and *j* are ' + 'reduced to\n' + ' "len(s)" if they are greater. When *k* is negative, *i* and ' + '*j* are\n' + ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' + 'are\n' + ' omitted or "None", they become "end" values (which end ' + 'depends on\n' + ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' + '"None", it\n' + ' is treated like "1".\n' '\n' '6. Concatenating immutable sequences always results in a new\n' ' object. This means that building up a sequence by repeated\n' @@ -12685,8 +12662,7 @@ ' * The linspace recipe shows how to implement a lazy version ' 'of\n' ' range that suitable for floating point applications.\n', - 'typesseq-mutable': '\n' - 'Mutable Sequence Types\n' + 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' 'The operations in the following table are defined on ' @@ -12826,8 +12802,7 @@ 'referenced multiple\n' ' times, as explained for "s * n" under Common Sequence ' 'Operations.\n', - 'unary': '\n' - 'Unary arithmetic and bitwise operations\n' + 'unary': 'Unary arithmetic and bitwise operations\n' '***************************************\n' '\n' 'All unary arithmetic and bitwise operations have the same ' @@ -12849,8 +12824,7 @@ 'In all three cases, if the argument does not have the proper type, ' 'a\n' '"TypeError" exception is raised.\n', - 'while': '\n' - 'The "while" statement\n' + 'while': 'The "while" statement\n' '*********************\n' '\n' 'The "while" statement is used for repeated execution as long as an\n' @@ -12874,8 +12848,7 @@ 'executed in the first suite skips the rest of the suite and goes ' 'back\n' 'to testing the expression.\n', - 'with': '\n' - 'The "with" statement\n' + 'with': 'The "with" statement\n' '********************\n' '\n' 'The "with" statement is used to wrap the execution of a block with\n' @@ -12948,8 +12921,7 @@ ' The specification, background, and examples for the Python ' '"with"\n' ' statement.\n', - 'yield': '\n' - 'The "yield" statement\n' + 'yield': 'The "yield" statement\n' '*********************\n' '\n' ' yield_stmt ::= yield_expression\n' From e0fbe5feee4f9c00f09eb9659c2182183036261a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 12:25:04 -0500 Subject: [PATCH 0090/1345] Update docs and patchlevel for 3.6.1rc1. --- Include/patchlevel.h | 8 ++++---- Misc/NEWS | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 49930e85565349..b42a0db2f82007 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL -#define PY_RELEASE_SERIAL 0 +#define PY_MICRO_VERSION 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0+" +#define PY_VERSION "3.6.1rc1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 9aee4974344e10..d715d306a34bc3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,10 @@ Python News +++++++++++ -What's New in Python 3.6.1 release candidate 1? -=============================================== +What's New in Python 3.6.1 release candidate 1 +============================================== -*Release date: XXXX-XX-XX* +*Release date: 2017-03-04* Core and Builtins ----------------- From 43571dde7b81f8426e01875c611d29274135355a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 4 Mar 2017 15:40:48 -0800 Subject: [PATCH 0091/1345] Correct spelling "instanciate" (GH-465) (GH-467) (cherry picked from commit 6abaed0ddaa1dd9be727ede09f6cd801c467c2ec) --- Doc/library/asyncio-subprocess.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74c6dee12..16ba9a3cd6c7f2 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -80,7 +80,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +123,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about From 9619ec99470100969a95a86ed8d7a1fab6c27546 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 4 Mar 2017 16:43:57 -0800 Subject: [PATCH 0092/1345] distutils docs: Fix a typo (GH-470) (GH-471) (cherry picked from commit 2a7bddaab7d6e1f7b243cdbb4fa6f6c8e266b18d) --- Doc/distutils/examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9cdb29a97..4e2761d8a7d046 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' From 38136e2a581ada0382d9bd1df5b32678cbd94b9a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 5 Mar 2017 05:33:38 -0500 Subject: [PATCH 0093/1345] Bump to v3.6.1rc1+. --- Include/patchlevel.h | 2 +- Misc/NEWS | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b42a0db2f82007..b13794521f297c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.1rc1" +#define PY_VERSION "3.6.1rc1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index d715d306a34bc3..b342738ac4b475 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,18 @@ Python News +++++++++++ +What's New in Python 3.6.1 final? +================================= + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 3.6.1 release candidate 1 ============================================== From 3405792b024e9c6b70c0d2355c55a23ac84e1e67 Mon Sep 17 00:00:00 2001 From: Petr Motejlek Date: Sun, 5 Mar 2017 17:14:06 +0100 Subject: [PATCH 0094/1345] bpo-29615: backport to 3.6 (#478) --- Lib/test/test_xmlrpc.py | 90 ++++++++++++++++++++++++++++++++++++++++- Lib/xmlrpc/server.py | 43 +++++++++++--------- Misc/NEWS | 4 ++ 3 files changed, 117 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e3df0689..69de08e9c3b252 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -343,6 +343,94 @@ def run_server(): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -1312,7 +1400,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfddd84816f..6faa2d6f8fa319 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -386,31 +386,36 @@ def _dispatch(self, method, params): not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. diff --git a/Misc/NEWS b/Misc/NEWS index b342738ac4b475..a12229857945c8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + What's New in Python 3.6.1 release candidate 1 ============================================== From b414e349eb6a6140a81ccec249bae3abe939e836 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 5 Mar 2017 20:18:43 +0100 Subject: [PATCH 0095/1345] bpo-29638: Fix spurious refleaks after typing is imported (#469) (#483) --- Lib/test/libregrtest/refleak.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d2458b1f2e0..e2e58bd872f094 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -143,9 +143,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() From e544b40faa5ab61b6aba691577d90b2b641f664d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sun, 5 Mar 2017 19:58:06 -0800 Subject: [PATCH 0096/1345] Fixes the upload script to purge the CDN correctly and display success output. (#466) (#497) --- Tools/msi/uploadrelease.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 305e84fc2d4b28..75840f2f851ecd 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,6 +8,7 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false false @@ -91,6 +92,7 @@ echo." /> + From 4e1a065c20856a00d0fe88ce022b249170608058 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 6 Mar 2017 17:24:28 +0900 Subject: [PATCH 0097/1345] bpo-29719: Remove Date and Release field in whatsnew/3.6 (GH-500) (cherry picked from commit 2225ddaa9e64c086b2b6997b0c9ac50921f7aa85) --- Doc/whatsnew/3.6.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b76..a696af4a999c84 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: From df6d7b406f3d1b2e4e2014751bfa25574c4df222 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 6 Mar 2017 18:17:10 +0800 Subject: [PATCH 0098/1345] [3.6] bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. (GH-504) --- Lib/test/test_bytes.py | 10 ++++++++++ Misc/NEWS | 3 +++ Objects/bytesobject.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d39cae24..cd82fa64570cb5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ def test_mod(self): a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ def test_imod(self): b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS b/Misc/NEWS index a12229857945c8..1fb78365bb89be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + Library ------- diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d484409600852..801711f7e6ccaa 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -619,11 +619,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); From e9effe6bb6b6dcbc9fb13e47811a251a84a807c5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 6 Mar 2017 09:15:41 -0800 Subject: [PATCH 0099/1345] fix minor bug in pymalloc. (GH-335) (GH-476) (cherry picked from commit f669ffff6010a94f2d441200e0fd73e9dea2883e) --- Doc/c-api/memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff545275fb3b8..873fb2ac1d3cad 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: From 130c4ec5fbeb0878b88ec6aa06e47c9672566c17 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 6 Mar 2017 09:31:16 -0800 Subject: [PATCH 0100/1345] bpo-29557: Remove ambiguous line in binhex docs (GH-90) (GH-475) "appears to not work in all cases" does not inspire confidence in this module. I can find no context for what bug this was referencing so it should be removed. (cherry picked from commit 6de2b7817fa9403e81dc38f13f3690f0bbf3d064) --- Doc/library/binhex.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23b2f9787..2966e0dbfbcfe8 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - From 284a2ba140cf11216e9804573d8cea6674ab90d9 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 7 Mar 2017 11:53:29 +0800 Subject: [PATCH 0101/1345] bpo-28728: clarify possible test failure due to ISP (GH-412) (GH-531) --- Lib/test/test_socket.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd76b24a5..5315d475df2218 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ def testHostnameRes(self): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ def test_host_resolution(self): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): From 33d70d1721ed8a23d2744f33d5358657adccf4cd Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 7 Mar 2017 17:18:12 +0900 Subject: [PATCH 0102/1345] PCbuild: Add -q option to svn export (GH-537) Without this option, AppVeyor log is too unreadable. (cherry picked from commit 8886d5f39286dffa7d9337857b151e7fb4af23fd) --- PCbuild/get_externals.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 98a755d870ad90..3a2656e961ef7f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -66,7 +66,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export %SVNROOT%%%e + svn export -q %SVNROOT%%%e ) ) From 23b26c4e206427a9e0a1d39240c17a049cf8c732 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 7 Mar 2017 02:44:11 -0800 Subject: [PATCH 0103/1345] Fixes bpo-29680: Older gdb does not have gdb.error. (GH-363) (GH-534) This change is required to make python-dbg.py compatible with GDB versions before 7.3. (cherry picked from commit 661ca8843fed1183e38db06e52d59ac300bf1c2a) --- Tools/gdb/libpython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cc1afbe16d01d1..d28851f9fd7f27 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -714,7 +714,7 @@ def _get_entries(self, keys): try: # <= Python 3.5 return keys['dk_entries'], dk_size - except gdb.error: + except RuntimeError: # >= Python 3.6 pass From 78ad039bcf1a8c494cbc8e18380cc30665869c3e Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 8 Mar 2017 11:43:35 +0800 Subject: [PATCH 0104/1345] bpo-26915: Test identity first in index() and count() of collections.abc.Sequence (GH-553) --- Lib/_collections_abc.py | 5 +++-- Lib/test/test_collections.py | 17 +++++++++++++---- Misc/NEWS | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b172f3f360e6ef..005d884572465f 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -908,7 +908,8 @@ def index(self, value, start=0, stop=None): i = start while stop is None or i < stop: try: - if self[i] == value: + v = self[i] + if v is value or v == value: return i except IndexError: break @@ -917,7 +918,7 @@ def index(self, value, start=0, stop=None): def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) + return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc6704cb0..47f756213ddf00 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ def test_issue26915(self): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Misc/NEWS b/Misc/NEWS index 1fb78365bb89be..7a03a2d65e35bc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + - bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. From 6a4b04cd337347d074ae0140fb13dca5bd4b11ef Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 7 Mar 2017 23:56:59 -0800 Subject: [PATCH 0105/1345] Revert "bpo-29571: Use correct locale encoding in test_re (#149)" (#554) (#555) This reverts commit ace5c0fdd9b962e6e886c29dbcea72c53f051dc4. --- Lib/test/test_re.py | 2 +- Misc/NEWS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a506b98be4aeb2..b945cf094e99c1 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1402,7 +1402,7 @@ def test_ascii_and_unicode_flag(self): def test_locale_flag(self): import locale - enc = locale.getpreferredencoding(False) + _, enc = locale.getlocale(locale.LC_CTYPE) # Search non-ASCII letter for i in range(128, 256): try: diff --git a/Misc/NEWS b/Misc/NEWS index 7a03a2d65e35bc..bf3876948f0a26 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -134,6 +134,7 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. +- Issue #24932: Use proper command line parsing in _testembed - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, From 666165fddf499e80d10a5d9263846ec4534f6a2a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 Mar 2017 12:29:33 +0200 Subject: [PATCH 0106/1345] [3.6] bpo-28230: Document the pathlib support in tarfile and add tests. (#559) --- Doc/library/tarfile.rst | 28 ++++++++++++--- Lib/test/test_tarfile.py | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f809753dfdd7..337c061107299c 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc03b28abc..fc79055421161c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ def test_extract_directory(self): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ def test_directory_size(self): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ def test_create_existing_taropen(self): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass From eb65edd1029876a4a5bb70b009aeb914088ac749 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 Mar 2017 15:45:43 +0200 Subject: [PATCH 0107/1345] [3.6] bpo-28231: The zipfile module now accepts path-like objects for external paths. (#561) (cherry picked from commit 8606e9524a7a4065042f7f228dc57eb74f88e4d3) --- Doc/library/zipfile.rst | 24 +++++++- Lib/test/test_zipfile.py | 117 ++++++++++++++++++++++++++++++++++----- Lib/zipfile.py | 20 +++++-- Misc/NEWS | 3 + 4 files changed, 143 insertions(+), 21 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f103380661..4c91759c67f5d4 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20e2bcf0d..d39f05fdb7788e 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ def test_open(self): for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -906,22 +913,56 @@ def test_write_pyfile_bad_syntax(self): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +970,50 @@ def test_extract(self): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1253,8 @@ def test_is_zip_erroneous_file(self): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2100,26 @@ def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43ce1cc848..d7f5beba996e6e 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ def from_file(cls, filename, arcname=None): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -1070,6 +1072,8 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1470,11 +1474,10 @@ def extract(self, member, path=None, pwd=None): as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1490,13 @@ def extractall(self, path=None, members=None, pwd=None): if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1517,9 @@ def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1801,6 +1812,7 @@ def writepy(self, pathname, basename="", filterfunc=None): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Misc/NEWS b/Misc/NEWS index bf3876948f0a26..99ce8c73620714 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + - bpo-26915: index() and count() methods of collections.abc.Sequence now check identity before checking equality when do comparisons. From 7253aded71df8d4bf8684fc78d11c596079211b3 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 8 Mar 2017 08:19:57 -0800 Subject: [PATCH 0108/1345] bpo-29176: Fix name of the _curses.window class (#52) (#532) Set name to "_curses.window" instead of "_curses.curses window" (with a space!?). (cherry picked from commit 61e2bc74dfab1ceee332d3f480dcf86c478c87c5) --- Modules/_cursesmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7ce15392..d88d06e78220ec 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2082,7 +2082,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ From 51b646a55a4a1c4d0df764c4404a062fbcc6b356 Mon Sep 17 00:00:00 2001 From: "n.d. parker" Date: Wed, 8 Mar 2017 23:27:46 +0100 Subject: [PATCH 0109/1345] Fix the only non-C90 comment to be C90 compatible. (#568) --- Include/pyport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d056..426822a81f075a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif From 9cef253ae3251b22d2fbb475fcb7a28805b78c86 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Mar 2017 10:51:44 +0200 Subject: [PATCH 0110/1345] [3.6] bpo-29768: Fixed compile-time check for expat version. (#576) (cherry picked from commit 22e707fa04476710ba5cc7e2206e4ac66743931b) --- Modules/pyexpat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d57aa114c..47f70a27f15846 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,7 +1191,7 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) +#if XML_COMBINED_VERSION >= 20100 || defined(XML_HAS_SET_HASH_SALT) /* This feature was added upstream in libexpat 2.1.0. Our expat copy * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT * to indicate that we can still use it. */ From 26d013e00f9d4adbcf3e084bbc890c799ff70407 Mon Sep 17 00:00:00 2001 From: orenmn Date: Thu, 9 Mar 2017 16:06:47 +0200 Subject: [PATCH 0111/1345] [3.6] bpo-28298: make array 'Q', 'L' and 'I' accept big intables as elements (#579) --- Lib/test/test_array.py | 39 +++++++++------ Misc/NEWS | 3 ++ Modules/arraymodule.c | 108 +++++++++++++++++++++++------------------ 3 files changed, 89 insertions(+), 61 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967ccff3e7c..d67f9195eba04c 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ def test_frombytearray(self): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ def test_overflow(self): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ def test_overflow(self): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Misc/NEWS b/Misc/NEWS index 99ce8c73620714..299a8388ff6e39 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + - bpo-28231: The zipfile module now accepts path-like objects for external paths. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee5a8e892..d4221fe4c6a163 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } From 68d29809405dc766966b2b973b8597212fbc3dbd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 9 Mar 2017 18:43:39 +0100 Subject: [PATCH 0112/1345] bpo-29619: Convert st_ino using unsigned integer (#557) (#584) bpo-29619: os.stat() and os.DirEntry.inodeo() now convert inode (st_ino) using unsigned integers. (cherry picked from commit 0f6d73343d342c106cda2219ebb8a6f0c4bd9b3c) (Misc/NEWS conflict handled manually.) --- Include/fileutils.h | 2 +- Modules/posixmodule.c | 13 ++++++++----- Python/fileutils.c | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index b933e985392e73..900c70faad719f 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; - __int64 st_ino; + uint64_t st_ino; unsigned short st_mode; int st_nlink; int st_uid; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba255ec48f6..2ea5e2def9acff 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1932,11 +1932,13 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT +#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); + PyLong_FromUnsignedLongLong(st->st_ino)); #else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); + Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); #endif #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); @@ -11156,7 +11158,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,7 +11421,8 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ #ifdef HAVE_LARGEFILE_SUPPORT return PyLong_FromLongLong((long long)self->d_ino); diff --git a/Python/fileutils.c b/Python/fileutils.c index e84d66e99a473d..f3764e4b3c5a2d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -583,7 +583,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* first clear the S_IFMT bits */ result->st_mode ^= (result->st_mode & S_IFMT); @@ -653,7 +653,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); From 80dfbe30b734f14f76a4e25c7fa8fdb7bdd3936a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Mar 2017 21:01:49 +0200 Subject: [PATCH 0113/1345] [3.6] bpo-29773: Add more cases for testing string to float conversion errors. (#586) (cherry picked from commit 9e6ac83acae31de2b072e665e177db9fcdf7c049) --- Lib/test/test_float.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473db503b76..6491f458c3acc4 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ def test_float_memoryview(self): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): From d9b0b8f63c3db38dc547e0c115e9060f13f2dbcf Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 10 Mar 2017 01:06:43 -0600 Subject: [PATCH 0114/1345] Add Appveyor (GH-324 backport) (GH-491) --- .github/appveyor.yml | 6 ++++++ README.rst | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 .github/appveyor.yml diff --git a/.github/appveyor.yml b/.github/appveyor.yml new file mode 100644 index 00000000000000..5a40005660b781 --- /dev/null +++ b/.github/appveyor.yml @@ -0,0 +1,6 @@ +version: 3.6.1rc1+.{build} +clone_depth: 5 +build_script: +- cmd: PCbuild\build.bat -e +test_script: +- cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 diff --git a/README.rst b/README.rst index 242572c5e361db..6b4c2dba306ca5 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,10 @@ This is Python version 3.6.1 release candidate 1 :alt: CPython build status on Travis CI :target: https://travis-ci.org/python/cpython +.. image:: https://ci.appveyor.com/api/projects/status/4mew1a93xdkbf5ua/branch/3.6?svg=true + :alt: CPython build status on Appveyor + :target: https://ci.appveyor.com/project/python/cpython/branch/3.6 + .. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg :alt: CPython code coverage on Codecov :target: https://codecov.io/gh/python/cpython From faaa80fb9887950e7068a9506e3517bcd6562ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 10 Mar 2017 05:30:05 -0800 Subject: [PATCH 0115/1345] Ignore Include/pydtrace_probes.h (#596) This is a generated file when --with-dtrace is passed. This entry is present in .hgignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ed4ebfbbd9de74..db27597aac57da 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ .gdb_history Doc/build/ Doc/venv/ +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* From ff6f3716279e75b2519133a82b9de0c3601963d9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 10 Mar 2017 09:52:03 -0800 Subject: [PATCH 0116/1345] bpo-28739: Document that f-strings cannot be used as docstring (GH-592) (GH-600) (cherry picked from commit d4e89287b397c7382c12d3f3d9fd901fd8243b3c) --- Doc/reference/lexical_analysis.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017afff0a3d..7f9c66481785c1 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -696,6 +696,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. From 68c5e4c592495f1e0f4f60acb3483beb6aa87be1 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 10 Mar 2017 14:44:48 -0800 Subject: [PATCH 0117/1345] bpo-28810: Document BUILD_TUPLE_UNPACK_WITH_CALL bytecode (GH-605) (cherry picked from commit 7e52c3e7aefb4cdaa0662fc01ff68a5e976b77ca) --- Doc/library/dis.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1c1e1a2aa5d4b3..686118f5201857 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -807,6 +807,15 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + .. opcode:: BUILD_LIST_UNPACK (count) This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list @@ -834,14 +843,16 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 -.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (oparg) +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) This is similar to :opcode:`BUILD_MAP_UNPACK`, - but is used for ``f(**x, **y, **z)`` call syntax. The lowest byte of - *oparg* is the count of mappings, the relative position of the - corresponding callable ``f`` is encoded in the second byte of *oparg*. + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. .. opcode:: LOAD_ATTR (namei) From 4807fa8386f825d0c71192c59f7e33b99a94bebe Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 10 Mar 2017 15:08:52 -0800 Subject: [PATCH 0118/1345] bpo-28810: Document changes to CALL_FUNCTION opcodes (GH-607) (cherry picked from commit 4b2a2a425a906c8e4eb8daee14ab1793e225f726) --- Doc/library/dis.rst | 69 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 686118f5201857..11d056901b7ad5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1009,14 +1009,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -1049,28 +1080,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops From 701f13ab930b62a634579951dc610f737f461135 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 10 Mar 2017 19:36:26 -0800 Subject: [PATCH 0119/1345] bpo-29784: Fix the reference to shutil.copy in the docs (GH-602) (GH-608) (cherry picked from commit 70ee0cd5c2a3dba82cb8e0c0742c012f9134c040) --- Doc/library/shutil.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf339ae1bc8..41e5bafa53d468 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. From 16416c22f9b8d9e067506d3a98f661756c80389c Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Sat, 11 Mar 2017 14:07:30 +0800 Subject: [PATCH 0120/1345] bpo-29770: remove outdated PYO related info (GH-590) (GH-612) --- Doc/glossary.rst | 2 +- Doc/library/zipfile.rst | 6 +++--- Doc/library/zipimport.rst | 2 +- Doc/using/cmdline.rst | 4 ++-- Modules/main.c | 2 +- Modules/zipimport.c | 6 +++--- PC/getpathp.c | 4 ++-- PCbuild/rmpyc.py | 20 +++++++------------- PCbuild/rt.bat | 8 ++++---- Python/pylifecycle.c | 2 +- 10 files changed, 25 insertions(+), 31 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d83b311fa..e07ab0df2fd68c 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -155,7 +155,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4c91759c67f5d4..a5d42118ba5176 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -464,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c245f7b3fe..eaae2bb04b7482 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 195f63f0a319da..08dc311d837be9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -532,8 +532,8 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to + If this is set to a non-empty string, Python won't try to write ``.pyc`` + files on the import of source modules. This is equivalent to specifying the :option:`-B` option. diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..2a0b1b37da69fb 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aaae41f51..cccc033d2d9fab 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1102,7 +1102,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1261,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1403,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe9c19451..e7be704a9a78dd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PCbuild/rmpyc.py b/PCbuild/rmpyc.py index a1e75bb7ae5b4b..0b58f687729f7e 100644 --- a/PCbuild/rmpyc.py +++ b/PCbuild/rmpyc.py @@ -1,25 +1,19 @@ -# Remove all the .pyc and .pyo files under ../Lib. +# Remove all the .pyc files under ../Lib. def deltree(root): import os from os.path import join - npyc = npyo = 0 + npyc = 0 for root, dirs, files in os.walk(root): for name in files: - delete = False - if name.endswith('.pyc'): - delete = True + # to be thorough + if name.endswith(('.pyc', '.pyo')): npyc += 1 - elif name.endswith('.pyo'): - delete = True - npyo += 1 - - if delete: os.remove(join(root, name)) - return npyc, npyo + return npyc -npyc, npyo = deltree("../Lib") -print(npyc, ".pyc deleted,", npyo, ".pyo deleted") +npyc = deltree("../Lib") +print(npyc, ".pyc deleted") diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 35826727f3fa27..e73ac0457646e9 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -4,8 +4,8 @@ rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args rem -d Run Debug build (python_d.exe). Else release build. rem -O Run python.exe or python_d.exe (see -d) with -O. rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .py[co] files reachable from Lib/. -rem -q runs the tests just once, and without deleting .py[co] files. +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. rem -x64 Run the 64-bit build of python (or python_d if -d was specified) rem from the 'amd64' dir instead of the 32-bit build in this dir. rem All leading instances of these switches are shifted off, and @@ -45,7 +45,7 @@ set exe=%prefix%python%suffix%.exe set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode -echo Deleting .pyc/.pyo files ... +echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... @@ -55,7 +55,7 @@ echo on %cmd% @echo off -echo About to run again without deleting .pyc/.pyo first: +echo About to run again without deleting .pyc first: pause :Qmode diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a4f7f823bc695d..af96d6d5fcb63c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -88,7 +88,7 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ From d7a8d32ce2d2b17ea9584759a281235ebfe81a89 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 11 Mar 2017 10:19:51 -0800 Subject: [PATCH 0121/1345] tempfile.rst: Fix some typos (GH-610) (GH-618) (cherry picked from commit d3b8f98696b905367da3975f73b482bac380b51c) --- Doc/library/tempfile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261ffb21f94..c59aca1e189086 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above From b3536df9f8b041142f5e3bf6037c11be06eb7f59 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 15:42:25 -0600 Subject: [PATCH 0122/1345] Only trigger AppVeyor on code or config changes (GH-611 backport) (GH-620) --- .github/appveyor.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/appveyor.yml b/.github/appveyor.yml index 5a40005660b781..a369e52415bb17 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -4,3 +4,20 @@ build_script: - cmd: PCbuild\build.bat -e test_script: - cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 + +# Only trigger AppVeyor if actual code or its configuration changes +only_commits: + files: + - .github/appveyor.yml + - .gitattributes + - Grammar/ + - Include/ + - Lib/ + - Modules/ + - Objects/ + - PC/ + - PCBuild/ + - Parser/ + - Programs/ + - Python/ + - Tools/ From df6f3f6f5743a061d6c3b939136a11ac170ae69e Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 17:16:34 -0600 Subject: [PATCH 0123/1345] fix subprocess on Windows (GH-391 backport) (GH-623) --- Lib/subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb459e2615..0db6571f291dae 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -986,7 +986,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe From 7c6d6e056460753c916348feee0e4916a97d7c86 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 22:39:54 -0600 Subject: [PATCH 0124/1345] bpo-29763: Use support.unlink instead of os.unlink (GH-624) (GH-625) support.unlink waits for the files to be removed before returning --- Lib/test/test_site.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b242766057..342ec9e43d5de3 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -501,15 +501,15 @@ def _create_underpth_exe(self, lines): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): From 2f386254e2cf054cc7ee5103b54c1cf16d9fa979 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 16:17:46 +1000 Subject: [PATCH 0125/1345] bpo-29656: Handle PR branches in 'make patchcheck' (#302) (#626) (cherry picked from commit 482f7a274fa52b7ba34ff308cd9acdcac9f41ba5) --- Tools/scripts/patchcheck.py | 58 +++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 58b081a9c53ff5..7a04aafa82a8a1 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -12,7 +12,6 @@ SRCDIR = sysconfig.get_config_var('srcdir') - def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -46,27 +45,73 @@ def mq_patches_applied(): return st.returncode == 0 and bstdout +def get_git_branch(): + """Get the symbolic name for the current git branch""" + cmd = "git rev-parse --abbrev-ref HEAD".split() + try: + return subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return None + + +def get_git_upstream_remote(): + """Get the remote name to use for upstream branches + + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream".split() + try: + subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + + +@status("Getting base branch for PR", + info=lambda x: x if x is not None else "not a PR branch") +def get_base_branch(): + if not os.path.isdir(os.path.join(SRCDIR, '.git')): + # Not a git checkout, so there's no base branch + return None + version = sys.version_info + if version.releaselevel == 'alpha': + base_branch = "master" + else: + base_branch = "{0.major}.{0.minor}".format(version) + this_branch = get_git_branch() + if this_branch is None or this_branch == base_branch: + # Not on a git PR branch, so there's no base branch + return None + upstream_remote = get_git_upstream_remote() + return upstream_remote + "/" + base_branch + + @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) -def changed_files(): +def changed_files(base_branch=None): """Get the list of changed or added files from Mercurial or git.""" if os.path.isdir(os.path.join(SRCDIR, '.hg')): + if base_branch is not None: + sys.exit('need a git checkout to check PR status') cmd = 'hg status --added --modified --no-status' if mq_patches_applied(): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] elif os.path.isdir(os.path.join(SRCDIR, '.git')): - cmd = 'git status --porcelain' + if base_branch: + cmd = 'git diff --name-status ' + base_branch + else: + cmd = 'git status --porcelain' filenames = [] with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: for line in st.stdout: line = line.decode().rstrip() - status = set(line[:2]) + status_text, filename = line.split(maxsplit=1) + status = set(status_text) # modified, added or unmerged files if not status.intersection('MAU'): continue - filename = line[3:] if ' -> ' in filename: # file is renamed filename = filename.split(' -> ', 2)[1].strip() @@ -165,7 +210,8 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" def main(): - file_paths = changed_files() + base_branch = get_base_branch() + file_paths = changed_files(base_branch) python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc') and From 4652d82a51e6530a4f820588a6209d3295963bdb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Mar 2017 10:05:05 +0200 Subject: [PATCH 0126/1345] bpo-29746: Update marshal docs to Python 3. (#547) (#631) (cherry picked from commit c611a5b1d4fab0123bf622f06c3bfa510221dc32) --- Doc/c-api/marshal.rst | 8 ++++---- Doc/glossary.rst | 7 +++++++ Doc/library/marshal.rst | 19 ++++++++----------- Python/marshal.c | 26 ++++++++++++-------------- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f4688d1b78..c6d1d02a2fa510 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index e07ab0df2fd68c..495934afe74c8b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -966,6 +970,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6effc7142d..d65afc20041133 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Python/marshal.c b/Python/marshal.c index 87a4b240a41c87..7b12ab7510dcfb 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -549,7 +549,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_lnotab, p); } else if (PyObject_CheckBuffer(v)) { - /* Write unknown bytes-like objects as a byte string */ + /* Write unknown bytes-like objects as a bytes object */ Py_buffer view; if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); @@ -1086,7 +1086,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1110,7 +1110,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } goto _read_ascii; @@ -1150,7 +1150,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } if (n != 0) { @@ -1612,7 +1612,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a string"); + "too much marshal data for a bytes object"); return NULL; } if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) @@ -1658,8 +1658,7 @@ PyDoc_STRVAR(dump_doc, "dump(value, file[, version])\n\ \n\ Write the value on the open file. The value must be a supported type.\n\ -The file must be an open file object such as sys.stdout or returned by\n\ -open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ +The file must be a writeable binary file.\n\ \n\ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised - but garbage data will also be written\n\ @@ -1715,8 +1714,7 @@ PyDoc_STRVAR(load_doc, Read one value from the open file and return it. If no valid value is\n\ read (e.g. because the data has a different Python version's\n\ incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ -The file must be an open file object opened in binary mode ('rb' or\n\ -'r+b').\n\ +The file must be a readable binary file.\n\ \n\ Note: If an object containing an unsupported type was marshalled with\n\ dump(), load() will substitute None for the unmarshallable type."); @@ -1735,7 +1733,7 @@ marshal_dumps(PyObject *self, PyObject *args) PyDoc_STRVAR(dumps_doc, "dumps(value[, version])\n\ \n\ -Return the string that would be written to a file by dump(value, file).\n\ +Return the bytes object that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ @@ -1771,8 +1769,8 @@ marshal_loads(PyObject *self, PyObject *args) PyDoc_STRVAR(loads_doc, "loads(bytes)\n\ \n\ -Convert the bytes object to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the input are\n\ +Convert the bytes-like object to a value. If no valid value is found,\n\ +raise EOFError, ValueError or TypeError. Extra bytes in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1810,8 +1808,8 @@ Functions:\n\ \n\ dump() -- write value to a file\n\ load() -- read value from a file\n\ -dumps() -- write value to a string\n\ -loads() -- read value from a string"); +dumps() -- marshal value as a bytes object\n\ +loads() -- read value from a bytes-like object"); From 61a82a5fbf1a674b91764652ab70f5dbb9ba50f2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 20:00:20 +1000 Subject: [PATCH 0127/1345] bpo-29798: Handle git worktree in `make patchcheck` (#629) (#633) In git worktree directories, `.git` is a configuration file rather than a subdirectory (cherry picked from commit 6a6d090612dd7deaac2bc0399fad743e5e2db606) --- Tools/scripts/patchcheck.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 7a04aafa82a8a1..f4ec7d8a30ea23 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -98,7 +98,10 @@ def changed_files(base_branch=None): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - elif os.path.isdir(os.path.join(SRCDIR, '.git')): + elif os.path.exists(os.path.join(SRCDIR, '.git')): + # We just use an existence check here as: + # directory = normal git checkout/clone + # file = git worktree directory if base_branch: cmd = 'git diff --name-status ' + base_branch else: From c60948464fb0ec116ea227f6bce8a4bb8fb75257 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 21:34:22 +1000 Subject: [PATCH 0128/1345] [3.6] bpo-29723: Consistently configure sys.path[0] (#636) Directory and zipfile execution previously added the parent directory of the directory or zipfile as sys.path[0] and then subsequently overwrote it with the directory or zipfile itself. This caused problems in isolated mode, as it overwrote the "stdlib as a zip archive" entry in sys.path, as the parent directory was never added. The attempted fix to that issue in bpo-29319 created the opposite problem in *non*-isolated mode, by potentially leaving the parent directory on sys.path instead of overwriting it. This change fixes the root cause of the problem by removing the whole "add-and-overwrite" dance for sys.path[0], and instead simply never adds the parent directory to sys.path in the first place. (cherry picked from commit d2977a3ae2cc6802921b1e3b6e9d13fcfbda872d) --- Lib/test/test_cmd_line_script.py | 67 +++++++++++++++++++++++++++++ Misc/NEWS | 8 ++++ Modules/main.c | 72 +++++++++++++++++++------------- 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Misc/NEWS b/Misc/NEWS index 299a8388ff6e39..640c783ed7f4be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,14 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. diff --git a/Modules/main.c b/Modules/main.c index 2a0b1b37da69fb..475a2fdc36db9b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; From aac875fa2f03cab61ceeaa2621c4c5534c7bcfc2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Mar 2017 21:52:17 +0200 Subject: [PATCH 0129/1345] [3.6] bpo-8256: Fixed possible failing or crashing input() (#641) if attributes "encoding" or "errors" of sys.stdin or sys.stdout are not set or are not strings. --- Misc/NEWS | 3 +++ Python/bltinmodule.c | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 640c783ed7f4be..eef4dfa3352286 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,9 @@ Core and Builtins Library ------- +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + - bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big intables (objects that have __int__) as elements. Patch by Oren Milman. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ef5a34cae900b6..597e26ec69a95a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1927,12 +1927,15 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *result; size_t len; + /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); - if (!stdin_encoding || !stdin_errors) - /* stdin is a text stream, so it must have an - encoding. */ + if (!stdin_encoding || !stdin_errors || + !PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) { + tty = 0; goto _readline_errors; + } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) @@ -1948,8 +1951,12 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *stringpo; stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); - if (!stdout_encoding || !stdout_errors) + if (!stdout_encoding || !stdout_errors || + !PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) { + tty = 0; goto _readline_errors; + } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); if (!stdout_encoding_str || !stdout_errors_str) @@ -2003,13 +2010,17 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: Py_XDECREF(stdin_encoding); Py_XDECREF(stdout_encoding); Py_XDECREF(stdin_errors); Py_XDECREF(stdout_errors); Py_XDECREF(po); - return NULL; + if (tty) + return NULL; + + PyErr_Clear(); } /* Fallback if we're not interactive */ From fa448de97de85d242491d7ad259ade0732f05db8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 12 Mar 2017 17:04:06 -0400 Subject: [PATCH 0130/1345] Fix wrapping into StopIteration of return values in generators and coroutines (#644) (#647) --- Lib/test/test_coroutines.py | 15 +++++++++++++++ Misc/NEWS | 2 ++ Objects/genobject.c | 3 +-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b4c7b5be6e208c..a69583b5f95ae6 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ async def waiter(coro): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): diff --git a/Misc/NEWS b/Misc/NEWS index eef4dfa3352286..504b977bc87b4b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + - bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a regression by revealing an inconsistency in how sys.path is initialized when executing ``__main__`` from a zipfile, directory, or other import location. diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0e129da4..1c29e296afe2a8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); From 99f8d33a94c8951fd6b2d5f079d20be698150993 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 12 Mar 2017 17:06:16 -0400 Subject: [PATCH 0131/1345] bpo-29742: asyncio get_extra_info() throws exception (#525) (#645) --- Lib/asyncio/sslproto.py | 4 +++- Lib/test/test_asyncio/test_sslproto.py | 12 ++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6aa0089a..ab7ff0bf93d076 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -543,8 +543,10 @@ def eof_received(self): def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default def _start_shutdown(self): if self._in_shutdown: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6967e5b5..f1771c5561afea 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -95,5 +95,17 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 504b977bc87b4b..25a24f52008a6f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,9 @@ Core and Builtins Library ------- +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + - bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" or "errors" of sys.stdin or sys.stdout are not set or are not strings. From 7927abba3449e12d0ae3518b722fdfeb383b4039 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 13 Mar 2017 10:36:34 +0800 Subject: [PATCH 0132/1345] fix the name of argument to ftplib.FTP.set_pasv and fix wording (GH-653) (GH-654) --- Doc/library/ftplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcfef2d98e..1153ce77ddabcb 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. From 388e2568fc919a177df2d2949b575c553aea84ac Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 13 Mar 2017 11:03:36 +0800 Subject: [PATCH 0133/1345] ftplib.FTP.retrbinary callback gets a bytes, not a str (GH-652) (GH-657) --- Doc/library/ftplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 1153ce77ddabcb..7291dfe84811c6 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A From faa2cc63e45bc7d7ffab84bebe5a9f4fe065bd96 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 14 Mar 2017 15:27:01 +0800 Subject: [PATCH 0134/1345] bpo-28856: Let %b format for bytes support objects that follow the buffer protocol (GH-664) --- Lib/test/test_format.py | 8 ++++++-- Misc/NEWS | 3 +++ Objects/bytesobject.c | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 83eb29faf88d6c..b6ba2e566b295f 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -315,10 +315,12 @@ def __bytes__(self): testcommon(b"%b", b"abc", b"abc") testcommon(b"%b", bytearray(b"def"), b"def") testcommon(b"%b", fb, b"123") + testcommon(b"%b", memoryview(b"abc"), b"abc") # # %s is an alias for %b -- should only be used for Py2/3 code testcommon(b"%s", b"abc", b"abc") testcommon(b"%s", bytearray(b"def"), b"def") testcommon(b"%s", fb, b"123") + testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') testcommon(b"%a", 3.14, b"3.14") @@ -377,9 +379,11 @@ def test_exc(formatstr, args, exception, excmsg): test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") test_exc(b"%s", "Wd", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: diff --git a/Misc/NEWS b/Misc/NEWS index 25a24f52008a6f..284a2031eeafc8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ Core and Builtins - bpo-29600: Fix wrapping coroutine return values in StopIteration. +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + - bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a regression by revealing an inconsistency in how sys.path is initialized when executing ``__main__`` from a zipfile, directory, or other import location. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 801711f7e6ccaa..30c1a7e24532da 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } From 53b2667dcf2a7d13af466a5fb91844f5125a920d Mon Sep 17 00:00:00 2001 From: Michael Seifert Date: Wed, 15 Mar 2017 08:42:02 +0100 Subject: [PATCH 0135/1345] bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords are not strings (#649) (#671) --- Lib/test/test_functools.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_functoolsmodule.c | 5 ++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index b7d648d0b15a55..cd4664cec08745 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -402,6 +402,32 @@ def test_attributes_unwritable(self): else: self.fail('partial object allowed __dict__ to be deleted') + def test_manually_adding_non_string_keyword(self): + p = self.partial(capture) + # Adding a non-string/unicode keyword to partial kwargs + p.keywords[1234] = 'value' + r = repr(p) + self.assertIn('1234', r) + self.assertIn("'value'", r) + with self.assertRaises(TypeError): + p() + + def test_keystr_replaces_value(self): + p = self.partial(capture) + + class MutatesYourDict(object): + def __str__(self): + p.keywords[self] = ['sth2'] + return 'astr' + + # Raplacing the value during key formatting should keep the original + # value alive (at least long enough). + p.keywords[MutatesYourDict()] = ['sth'] + r = repr(p) + self.assertIn('astr', r) + self.assertIn("['sth']", r) + + class TestPartialPy(TestPartial, unittest.TestCase): partial = py_functools.partial diff --git a/Misc/ACKS b/Misc/ACKS index c3b29a42a34f75..03afeb8f3876ee 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1371,6 +1371,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Žiga Seilnacht Yury Selivanov Fred Sells diff --git a/Misc/NEWS b/Misc/NEWS index 284a2031eeafc8..4c5efefb659378 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Core and Builtins Library ------- +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + - bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. Patch by Nikolay Kim. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 7abc9f464027f7..1bcf16a7e00f53 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -250,8 +250,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } From 02c6fa573a0a7f417e21ad9e81bcff3ac61a9fa8 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 15 Mar 2017 14:12:22 +0300 Subject: [PATCH 0136/1345] Delete duplicate entry in Misc/NEWS (#673) It has already been added in the build section. --- Misc/NEWS | 2 -- 1 file changed, 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 4c5efefb659378..03f70dc885d818 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,8 +73,6 @@ Core and Builtins - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. -- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. - - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. From 78add3309b53d68c2c5c118ec63f7485bbf93c9a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 15 Mar 2017 14:12:53 +0300 Subject: [PATCH 0137/1345] Change assertRaises to assertRaisesRegex in test_xmlrpc (#481) (#674) (cherry picked from commit c6b448b36d22769c684bb3276f85c1b47d15ab63) --- Lib/test/test_xmlrpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 69de08e9c3b252..30025e388ddf2e 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -412,7 +412,7 @@ def test_registered_func_is_none(self): dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() dispatcher.register_function(None, name='method') - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) def test_instance_has_no_func(self): @@ -420,14 +420,14 @@ def test_instance_has_no_func(self): dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() dispatcher.register_instance(object()) - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) def test_cannot_locate_func(self): """Calls a function that the dispatcher cannot locate""" dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) From 0dfd18a1e14e08b0d6459c2dd1898e5bb38020c5 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 16 Mar 2017 16:22:35 +0300 Subject: [PATCH 0138/1345] Fix stderr bug in json.tool test (#346) (#676) See https://github.com/python/cpython/pull/201#discussion_r103229425.(cherry picked from commit b4e9087e7b77e8f76feac76f9c1ab21b49c0c766) --- Lib/test/test_json/test_tool.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f373664e1278..9d93f931ca3958 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN From 9cd5e87bac51d7b901e3c36bf22728bb1693da59 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 16 Mar 2017 11:03:03 -0700 Subject: [PATCH 0139/1345] Takes vcruntime140.dll from the correct source. (#684) --- Tools/msi/make_zip.proj | 9 +++------ Tools/nuget/make_pkg.proj | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72..b3588b7a0bae90 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c..464ef0456af47a 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + From 3fce38c540681a736b7e5ca7611758f7bc159c9e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 19:56:56 -0700 Subject: [PATCH 0140/1345] bpo-29820: othergui.rst: Remove outdated information (GH-685) (GH-688) (cherry picked from commit 1bb0f3762ec5104014aeed0ae6e9d64598d8fcac) --- Doc/library/othergui.rst | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50b6bd5e1..d40abe167653b7 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for From 4ac01f0ff3e3c9c02f5cc8c55f2fbe4639808f5e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 20:58:42 -0700 Subject: [PATCH 0141/1345] update test_socket AEAD test for kernel 4.9 and up (GH-133) (GH-548) (cherry picked from commit 9764c151c51480a7ca6042b1ccd69be2620ff360) --- Lib/test/test_socket.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 5315d475df2218..1508f20c9f0598 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5479,7 +5479,7 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5502,8 +5502,7 @@ def test_aead_aes_gcm(self): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5511,7 +5510,7 @@ def test_aead_aes_gcm(self): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5522,7 +5521,7 @@ def test_aead_aes_gcm(self): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5530,7 +5529,7 @@ def test_aead_aes_gcm(self): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5540,8 +5539,8 @@ def test_aead_aes_gcm(self): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): From 7c2081122c2a6b8ea27c3111da7f19f7824fe18a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 20:59:36 -0700 Subject: [PATCH 0142/1345] Add sockaddr_alg to sock_addr_t (GH-234) (GH-533) (cherry picked from commit d37c068e695f8ec72b5c1b5a5a5ece2337fda768) --- Modules/socketmodule.c | 30 ------------------------------ Modules/socketmodule.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc062fd6539..f3654c97e76ce9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ if_indextoname(index) -- return the corresponding interface name\n\ #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927e0b35f0..03f982b91083f3 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, From 948171bf999cf8b3e12048851041d2e04ae3a78c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 17 Mar 2017 14:59:16 +0300 Subject: [PATCH 0143/1345] bpo-16355: Clarify when inspect.getcomments() returns None (#428) (#690) Initial patch by Vajrasky Kok. (cherry picked from commit 3f2155ffe683080f2a1b28408fa48d43ba92f943) --- Doc/library/inspect.rst | 4 +++- Lib/test/test_inspect.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3fa44a0c99891c..4ff2187b45f8eb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -442,7 +442,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 88eaabe6765ab2..cfea281c70db11 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -387,6 +387,11 @@ def test_cleandoc(self): def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) def test_getmodule(self): # Check actual module From 46e81d3345b979f835740a228857906dca0f59c0 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 17 Mar 2017 19:52:56 +0000 Subject: [PATCH 0144/1345] bpo-29808: Do not fail in SysLogHandler constructor if syslog isn't available. (#696) bpo-29808: SysLogHandler: Do not fail if initial connect to syslog failed. (cherry picked from commit 1b038e073807ecb6fd176edaf3386a8e3205416e) --- Lib/logging/handlers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d779734f37b0d..2356f8d3d2d8ce 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,7 +815,14 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: From 49fc153342f98e0d4ae5a96b88f3e91e4c5861d7 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 17 Mar 2017 15:41:35 -0700 Subject: [PATCH 0145/1345] Combine the C++ header CI build into the main C build (GH-697) (GH-704) This will eliminate one of the builds in Travis, allowing for CI overall to complete faster. (cherry picked from commit 993d4b3440f2282976901ce66879037c4443868a) --- .travis.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27b63c6c08b376..36961e38790a4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,15 +64,6 @@ matrix: # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate - bash <(curl -s https://codecov.io/bash) - - os: linux - language: cpp - compiler: clang - env: - - TESTING="C++ header compatibility" - before_script: - - ./configure - script: - - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . # Travis provides only 2 cores, so don't overdue the parallelism and waste memory. before_script: @@ -88,6 +79,8 @@ before_script: script: # `-r -w` implicitly provided through `make buildbottest`. - make buildbottest TESTOPTS="-j4" + # Test for C++ header compatibility. + - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . notifications: email: false From b044120f048571030552eefab8ba2329a9e76ceb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 19 Mar 2017 10:03:39 -0700 Subject: [PATCH 0146/1345] Drop C++ header compatibility test (#718) (#719) The $CXX environment variable is not exported under the 'c' language on Travis. (cherry picked from commit 77ed11552da3e01dd235b7d68988076866b1f604) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36961e38790a4d..66f03dc7161a7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,8 +79,6 @@ before_script: script: # `-r -w` implicitly provided through `make buildbottest`. - make buildbottest TESTOPTS="-j4" - # Test for C++ header compatibility. - - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . notifications: email: false From 69eab3123ed1de4bed4b7dedecabe415f6139bb6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:26:26 +0200 Subject: [PATCH 0147/1345] bpo-28749: Fixed the documentation of the mapping codec APIs. (#487) (#714) Added the documentation for PyUnicode_Translate(). (cherry picked from commit c85a26628ceb9624c96c3064e8b99033c026d8a3) --- Doc/c-api/unicode.rst | 95 +++++++++++++++++++++-------------------- Include/unicodeobject.h | 45 ++++++++----------- 2 files changed, 66 insertions(+), 74 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada7be7ee4..6e91576ee8f004 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 587cf03e3697eb..2d0d77e804d9d8 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1609,50 +1609,41 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( This codec uses mappings to encode and decode characters. - Decoding mappings must map single string characters to single - Unicode characters, integers (which are then interpreted as Unicode - ordinals) or None (meaning "undefined mapping" and causing an - error). - - Encoding mappings must map single Unicode characters to single - string characters, integers (which are then interpreted as Latin-1 - ordinals) or None (meaning "undefined mapping" and causing an - error). - - If a character lookup fails with a LookupError, the character is - copied as-is meaning that its ordinal value will be interpreted as - Unicode or Latin-1 ordinal resp. Because of this mappings only need - to contain those mappings which map characters to different code - points. + Decoding mappings must map byte ordinals (integers in the range from 0 to + 255) to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or None. Unmapped data bytes (ones which cause a LookupError) + as well as mapped to None, 0xFFFE or '\ufffe' are treated as "undefined + mapping" and cause an error. + + Encoding mappings must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or None. Unmapped character + ordinals (ones which cause a LookupError) as well as mapped to + None are treated as "undefined mapping" and cause an error. */ PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( const char *string, /* Encoded string */ Py_ssize_t length, /* size of string */ - PyObject *mapping, /* character mapping - (char ordinal -> unicode ordinal) */ + PyObject *mapping, /* decoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( PyObject *unicode, /* Unicode object */ - PyObject *mapping /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping /* encoding mapping */ ); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); #endif @@ -1661,8 +1652,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -1960,8 +1951,8 @@ PyAPI_FUNC(PyObject*) PyUnicode_RSplit( /* Translate a string by applying a character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and From fca705d533970011e50b3f278aab81cead39b00d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:27:16 +0200 Subject: [PATCH 0148/1345] bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. (#514) (#722) (cherry picked from commit a5af6e1af77ee0f9294c5776478a9c24d9fbab94) --- Lib/test/test_fileio.py | 8 +- Lib/test/test_io.py | 20 + Misc/NEWS | 800 +++++++++++++++++++++++++++++++++++++++ Modules/_io/bufferedio.c | 14 +- Modules/_io/fileio.c | 16 +- Modules/_io/textio.c | 23 +- 6 files changed, 872 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210ae1997ad..57a02656206ff4 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ def testReprNoCloseFD(self): finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index aaa64eadffe1ea..8f895fe0bfb885 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1014,6 +1014,16 @@ def test_repr(self): raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2434,16 @@ def test_repr(self): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) diff --git a/Misc/NEWS b/Misc/NEWS index 03f70dc885d818..310e982d78f03a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,806 @@ Core and Builtins - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. +- bpo-29568: Escaped percent "%%" in the format string for classic string + formatting no longer allows any characters between two percents. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +- bpo-29695: bool(), float(), list() and tuple() no longer take keyword arguments. + The first argument of int() can now be passes only as positional argument. + +- bpo-20087: Prefer glibc's list of locale aliases to the X11 ones. + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- bpo-28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, + and AsyncFunctionDef ast nodes. docstring is not first stmt in their body + anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object + for module and class. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + +- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. + +- bpo-29546: Improve from-import error message with location + +- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. + +- Issue #29337: Fixed possible BytesWarning when compare the code objects. + Warnings could be emitted at compile time. + +- Issue #29327: Fixed a crash when pass the iterable keyword argument to + sorted(). + +- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). + +- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. + +- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. + Calling function is up to 5% faster. + +- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII + whitespace, not only spaces. Patch by Robert Xiao. + +- Issue #28932: Do not include if it does not exist. + +- Issue #25677: Correct the positioning of the syntax error caret for + indented blocks. Based on patch by Michael Layzell. + +- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate + form. + +- Issue #18896: Python function can now have more than 255 parameters. + collections.namedtuple() now supports tuples with more than 255 elements. + +- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by + Chi Hsuan Yen. + +- Issue #26919: On Android, operating system data is now always encoded/decoded + to/from UTF-8, instead of the locale encoding to avoid inconsistencies with + os.fsencode() and os.fsdecode() which are already using UTF-8. + +- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy + bug triggerable by a monkey-patched len() function. + +- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() + must not convert combined table into split table. Patch written by INADA + Naoki. + +- Issue #28739: f-string expressions are no longer accepted as docstrings and + by ast.literal_eval() even if they do not include expressions. + +- Issue #28512: Fixed setting the offset attribute of SyntaxError by + PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +- Issue #28918: Fix the cross compilation of xxlimited when Python has been + built with Py_DEBUG defined. + +- Issue #23722: Rather than silently producing a class that doesn't support + zero-argument ``super()`` in methods, failing to pass the new + ``__classcell__`` namespace entry up to ``type.__new__`` now results in a + ``DeprecationWarning`` and a class that supports zero-argument ``super()``. + +- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of + a descriptor that is used inside that class no longer prevents calling the + __set_name__ method of other descriptors. + +- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate + the untested and undocumented ``sys.callstats()`` function. Remove the + ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, + :mod:`cProfile` or :mod:`profile` to profile function calls. + +- Issue #12844: More than 255 arguments can now be passed to a function. + +- Issue #28782: Fix a bug in the implementation ``yield from`` when checking + if the next instruction is YIELD_FROM. Regression introduced by WORDCODE + (issue #26647). + +- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 + encoders when a string returned by the error handler contains multiple + non-encodable characters (non-ASCII for the ASCII codec, characters out + of the U+0000-U+00FF range for Latin1). + +- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. + Improve speed of dict literal with constant keys up to 30%. + +- Issue #28532: Show sys.version when -V option is supplied twice. + +- Issue #27100: The with-statement now checks for __enter__ before it + checks for __exit__. This gives less confusing error messages when + both methods are missing. Patch by Jonathan Ellington. + +- Issue #28746: Fix the set_inheritable() file descriptor method on platforms + that do not have the ioctl FIOCLEX and FIONCLEX commands. + +- Issue #26920: Fix not getting the locale's charset upon initializing the + interpreter, on platforms that do not have langinfo. + +- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X + when decode astral characters. Patch by Xiang Zhang. + +- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. + +- Issue #19398: Extra slash no longer added to sys.path components in case of + empty compile-time PYTHONPATH components. + +- Issue #28621: Sped up converting int to float by reusing faster bits counting + implementation. Patch by Adrian Wielgosik. + +- Issue #28580: Optimize iterating split table values. + Patch by Xiang Zhang. + +- Issue #28583: PyDict_SetDefault didn't combine split table when needed. + Patch by Xiang Zhang. + +- Issue #28128: Deprecation warning for invalid str and byte escape + sequences now prints better information about where the error + occurs. Patch by Serhiy Storchaka and Eric Smith. + +- Issue #28509: dict.update() no longer allocate unnecessary large memory. + +- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug + build. + +- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused + keeping unreachable code. + +- Issue #28214: Improved exception reporting for problematic __set_name__ + attributes. + +- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception + loss in PyTraceBack_Here(). + +- Issue #28183: Optimize and cleanup dict iteration. + +- Issue #26081: Added C implementation of asyncio.Future. + Original patch by Yury Selivanov. + +- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). + Patch by Xiang Zhang. + +- Issue #28376: The type of long range iterator is now registered as Iterator. + Patch by Oren Milman. + +- Issue #28376: Creating instances of range_iterator by calling range_iterator + type now is disallowed. Calling iter() on range instance is the only way. + Patch by Oren Milman. + +- Issue #26906: Resolving special methods of uninitialized type now causes + implicit initialization of the type instead of a fail. + +- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. + Original patch by Niklas Koep. + +- Issue #24098: Fixed possible crash when AST is changed in process of + compiling it. + +- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when + hashes have same lower bits. + +- Issue #28350: String constants with null character no longer interned. + +- Issue #26617: Fix crash when GC runs during weakref callbacks. + +- Issue #27942: String constants now interned recursively in tuples and frozensets. + +- Issue #28289: ImportError.__init__ now resets not specified attributes. + +- Issue #21578: Fixed misleading error message when ImportError called with + invalid keyword args. + +- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. + Patch by Soumya Sharma. + +- Issue #28086: Single var-positional argument of tuple subtype was passed + unscathed to the C-defined function. Now it is converted to exact tuple. + +- Issue #28214: Now __set_name__ is looked up on the class instead of the + instance. + +- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() + syscall fails with EPERM, for example when blocked by SECCOMP. + +- Issue #28192: Don't import readline in isolated mode. + +- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. + Thanks Oren Milman. + +- Issue #27222: Clean up redundant code in long_rshift function. Thanks + Oren Milman. + +- Upgrade internal unicode databases to Unicode version 9.0.0. + +- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport + should use the same optimization level as the interpreter. + +- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly + optimize memcpy(). + +- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a + "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. + +- Issue #26182: Raise DeprecationWarning when async and await keywords are + used as variable/attribute/class/function name. + +- Issue #26182: Fix a refleak in code that raises DeprecationWarning. + +- Issue #28721: Fix asynchronous generators aclose() and athrow() to + handle StopAsyncIteration propagation properly. + +- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD + opcodes. + +Extension Modules +----------------- + +- Issue #29169: Update zlib to 1.2.11. + +Library +------- + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28692: Using non-integer value for selecting a plural form in gettext is + now deprecated. + +- bpo-26121: Use C library implementation for math functions erf() and erfc(). + +- bpo-29619: os.stat() and os.DirEntry.inode() now convert inode (st_ino) using + unsigned integers. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. + +- bpo-29645: Speed up importing the webbrowser module. webbrowser.register() + is now thread-safe. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-28682: Added support for bytes paths in os.fwalk(). + +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + +- bpo-9303: Migrate sqlite3 module to _v2 API. Patch by Aviv Palivoda. + +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher + and its subclasses can now be used as a decorator. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes + '~' in the set of characters that is not quoted by default. Patch by + Christian Theune and Ratnadeep Debnath. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Łukasz Langa. + +- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python + 3.6.0: check minimum and maximum years. + +- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir + +- Issue #29444: Fixed out-of-bounds buffer access in the group() method of + the match object. Based on patch by WGH. + +- Issue #29377: Add SlotWrapperType, MethodWrapperType, and + MethodDescriptorType built-in types to types module. + Original patch by Manuel Krebber. + +- Issue #29218: Unused install_misc command is now removed. It has been + documented as unused since 2000. Patch by Eric N. Vander Weele. + +- Issue #29368: The extend() method is now called instead of the append() + method when unpickle collections.deque and other list-like objects. + This can speed up unpickling to 2 times. + +- Issue #29338: The help of a builtin or extension class now includes the + constructor signature if __text_signature__ is provided for the class. + +- Issue #29335: Fix subprocess.Popen.wait() when the child process has + exited to a stopped instead of terminated state (ex: when under ptrace). + +- Issue #29290: Fix a regression in argparse that help messages would wrap at + non-breaking spaces. + +- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. + +- Issue #29197: Removed deprecated function ntpath.splitunc(). + +- Issue #29210: Removed support of deprecated argument "exclude" in + tarfile.TarFile.add(). + +- Issue #29219: Fixed infinite recursion in the repr of uninitialized + ctypes.CDLL instances. + +- Issue #29192: Removed deprecated features in the http.cookies module. + +- Issue #29193: A format string argument for string.Formatter.format() + is now positional-only. + +- Issue #29195: Removed support of deprecated undocumented keyword arguments + in methods of regular expression objects. + +- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. + KeyError could be raised when cached function with full cache was + simultaneously called from differen threads with the same uncached arguments. + +- Issue #20804: The unittest.mock.sentinel attributes now preserve their + identity when they are copied or pickled. + +- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with + leading dots could match related hostnames again (e.g. .b.c matches a.b.c). + Patch by Milan Oberkirch. + +- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter + anymore. Patch written by Jiajun Huang. + +- Issue #15812: inspect.getframeinfo() now correctly shows the first line of + a context. Patch by Sam Breese. + +- Issue #28985: Update authorizer constants in sqlite3 module. + Patch by Dingyuan Wang. + +- Issue #29094: Offsets in a ZIP file created with extern file object and modes + "w" and "x" now are relative to the start of the file. + +- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows + +- Issue #13051: Fixed recursion errors in large or resized + curses.textpad.Textbox. Based on patch by Tycho Andersen. + +- Issue #9770: curses.ascii predicates now work correctly with negative + integers. + +- Issue #28427: old keys should not remove new values from + WeakValueDictionary when collecting from another thread. + +- Issue 28923: Remove editor artifacts from Tix.py. + +- Issue #28871: Fixed a crash when deallocate deep ElementTree. + +- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and + WeakValueDictionary.pop() when a GC collection happens in another + thread. + +- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that + doesn't own its elements as limits. + +- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, + instead of /bin/sh. + +- Issue #28779: multiprocessing.set_forkserver_preload() would crash the + forkserver process if a preloaded module instantiated some + multiprocessing objects such as locks. + +- Issue #26937: The chown() method of the tarfile.TarFile class does not fail + now when the grp module cannot be imported, as for example on Android + platforms. + +- Issue #28847: dbm.dumb now supports reading read-only files and no longer + writes the index file when it is not changed. A deprecation warning is now + emitted if the index file is missed and recreated in the 'r' and 'w' modes + (will be an error in future Python releases). + +- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in + re.sub() replacement templates regular expressions now are errors. + +- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): + call warnings.showwarning() if it was overriden inside the context manager. + +- Issue #27172: To assist with upgrades from 2.7, the previously documented + deprecation of ``inspect.getfullargspec()`` has been reversed. This decision + may be revisited again after the Python 2.7 branch is no longer officially + supported. + +- Issue #28740: Add sys.getandroidapilevel(): return the build time API version + of Android as an integer. Function only available on Android. + +- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and + :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by + Omar Sandoval. + +- Issue #28752: Restored the __reduce__() methods of datetime objects. + +- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created + by re.compile(), become comparable (only x==y and x!=y operators). This + change should fix the issue #18383: don't duplicate warning filters when the + warnings module is reloaded (thing usually only done in unit tests). + +- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was + deprecated in 3.4 and undocumented prior to that. + +- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and + from_buffer_copy() methods on abstract classes like Array. + +- Issue #28548: In the "http.server" module, parse the protocol version if + possible, to avoid using HTTP 0.9 in some error responses. + +- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. + Patch by Vajrasky Kok + +- Issue #28563: Fixed possible DoS and arbitrary code execution when handle + plural form selections in the gettext module. The expression parser now + supports exact syntax supported by GNU gettext. + +- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when + the garbage collector is invoked in other thread. Based on patch by + Sebastian Cufre. + +- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if + given empty data twice. Patch by Benjamin Fogle. + +- Issue #28549: Fixed segfault in curses's addch() with ncurses6. + +- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar + file with compression before trying to open it without compression. Otherwise + it had 50% chance failed with ignore_zeros=True. + +- Issue #23262: The webbrowser module now supports Firefox 36+ and derived + browsers. Based on patch by Oleg Broytman. + +- Issue #24241: The webbrowser in an X environment now prefers using the + default browser directly. Also, the webbrowser register() function now has + a documented 'preferred' argument, to specify browsers to be returned by + get() with no arguments. Patch by David Steele + +- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused + by representing the scale as float value internally in Tk. tkinter.IntVar + now works if float value is set to underlying Tk variable. + +- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after + a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant + newline after a year's calendar. Based on patch by Xiang Zhang. + +- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space + at the start of new line after printing a month's calendar. Patch by + Xiang Zhang. + +- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. + Based on patch by Kaarle Ritvanen. + +- Issue #28353: os.fwalk() no longer fails on broken links. + +- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept + non-None value is passed to it.send(val). + +- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix + for readability (was "`"). + +- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin + a workaround to Tix library bug. + +- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. + +- Issue #25953: re.sub() now raises an error for invalid numerical group + reference in replacement template even if the pattern is not found in + the string. Error message for invalid group reference now includes the + group index and the position of the reference. + Based on patch by SilentGhost. + +- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead + of 1, 10, 100,... for autoranging. + +- Issue #28115: Command-line interface of the zipfile module now uses argparse. + Added support of long options. + +- Issue #18219: Optimize csv.DictWriter for large number of columns. + Patch by Mariatta Wijaya. + +- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. + +- Issue #23214: In the "io" module, the argument to BufferedReader and + BytesIO's read1() methods is now optional and can be -1, matching the + BufferedIOBase specification. + +- Issue #28480: Fix error building socket module when multithreading is + disabled. + +- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line + options which were deprecated since Python 3.3. + +- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 + to make benchmarks more reliable. + +- Issue #28240: timeit autorange now uses a single loop iteration if the + benchmark takes less than 10 seconds, instead of 10 iterations. + "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds + instead of 40 seconds. + +- Distutils.sdist now looks for README and setup.py files with case + sensitivity. This behavior matches that found in Setuptools 6.0 and + later. See `setuptools 100 + `_ for rationale. + +- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by + Ned Batchelder. + +- Issue #20766: Fix references leaked by pdb in the handling of SIGINT + handlers. + +- Issue #27998: Fixed bytes path support in os.scandir() on Windows. + Patch by Eryk Sun. + +- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. + +- Issue #26293: Fixed writing ZIP files that starts not from the start of the + file. Offsets in ZIP file now are relative to the start of the archive in + conforming to the specification. + +- Issue #28380: unittest.mock Mock autospec functions now properly support + assert_called, assert_not_called, and assert_called_once. + +- Issue #28229: lzma module now supports pathlib. + +- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. + +- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. + +- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. + +- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. + Original patch by Oren Milman. + +- Issue #27358: Optimized merging var-keyword arguments and improved error + message when passing a non-mapping as a var-keyword argument. + +- Issue #28257: Improved error message when passing a non-iterable as + a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. + +- Issue #28322: Fixed possible crashes when unpickle itertools objects from + incorrect pickle data. Based on patch by John Leitch. + +- Issue #28228: imghdr now supports pathlib. + +- Issue #28226: compileall now supports pathlib. + +- Issue #28314: Fix function declaration (C flags) for the getiterator() method + of xml.etree.ElementTree.Element. + +- Issue #28148: Stop using localtime() and gmtime() in the time + module. + + Introduced platform independent _PyTime_localtime API that is + similar to POSIX localtime_r, but available on all platforms. Patch + by Ed Schouten. + +- Issue #28253: Fixed calendar functions for extreme months: 0001-01 + and 9999-12. + + Methods itermonthdays() and itermonthdays2() are reimplemented so + that they don't call itermonthdates() which can cause datetime.date + under/overflow. + +- Issue #28275: Fixed possible use after free in the decompress() + methods of the LZMADecompressor and BZ2Decompressor classes. + Original patch by John Leitch. + +- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() + if pass invalid string-like object as a name. Patch by Xiang Zhang. + +- Issue #18844: random.choices() now has k as a keyword-only argument + to improve the readability of common cases and come into line + with the signature used in other languages. + +- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. + Patch by Madison May. + +- Issue #27611: Fixed support of default root window in the tkinter.tix module. + Added the master parameter in the DisplayStyle constructor. + +- Issue #27348: In the traceback module, restore the formatting of exception + messages like "Exception: None". This fixes a regression introduced in + 3.5a2. + +- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). + +- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is + interrupted by a signal and a signal handler raises a Python exception. + +- Issue #28200: Fix memory leak on Windows in the os module (fix + path_converter() function). + +- Issue #25400: RobotFileParser now correctly returns default values for + crawl_delay and request_rate. Initial patch by Peter Wirtz. + +- Issue #27932: Prevent memory leak in win32_ver(). + +- Fix UnboundLocalError in socket._sendfile_use_sendfile. + +- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of + os.stat(). Patch by Eryk Sun. + +- Issue #22493: Warning message emitted by using inline flags in the middle of + regular expression now contains a (truncated) regex pattern. + Patch by Tim Graham. + +- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when + an empty bytestring is passed. + +- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. + Patch by Gergely Imreh and Markus Holtermann. + +- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. + Patch by Eryk Sun. + +- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +- Issue #27906: Fix socket accept exhaustion during high TCP traffic. + Patch by Kevin Conway. + +- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. + Patch by Seth Michael Larson. + +- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. + Patch by iceboy. + +- Issue #26909: Fix slow pipes IO in asyncio. + Patch by INADA Naoki. + +- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. + Patch by Mark Williams. + +- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. + +- Issue #28368: Refuse monitoring processes if the child watcher has + no loop attached. + Patch by Vincent Michel. + +- Issue #28369: Raise RuntimeError when transport's FD is used with + add_reader, add_writer, etc. + +- Issue #28370: Speedup asyncio.StreamReader.readexactly. + Patch by Коренберг Марк. + +- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. + +- Issue #28372: Fix asyncio to support formatting of non-python coroutines. + +- Issue #28399: Remove UNIX socket from FS before binding. + Patch by Коренберг Марк. + +- Issue #27972: Prohibit Tasks to await on themselves. + +- Issue #24142: Reading a corrupt config file left configparser in an + invalid state. Original patch by Florian Höch. + +- Issue #29581: ABCMeta.__new__ now accepts **kwargs, allowing abstract base + classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. + +Windows +------- + +- bpo-29579: Removes readme.txt from the installer. + +- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) + +- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default + +- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c + +- Issue #28402: Adds signed catalog files for stdlib on Windows. + +- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by + Eryk Sun) + +- Issue #28251: Improvements to help manuals on Windows. + +- Issue #28110: launcher.msi has different product codes between 32-bit and + 64-bit + +- Issue #28161: Opening CON for write access fails + +- Issue #28162: WindowsConsoleIO readall() fails if first line starts with + Ctrl+Z + +- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to + _open_osfhandle + +- Issue #28164: _PyIO_get_console_type fails for various paths + +- Issue #28137: Renames Windows path file to ._pth + +- Issue #28138: Windows ._pth file should allow import site + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with + a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 + and 0x03060000 (not including) or 0x03060100 or higher. Added functions + PySlice_Unpack() and PySlice_AdjustIndices(). + +- Issue #29083: Fixed the declaration of some public API functions. + PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in + limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and + Py_BuildValue() were not available in limited API of version < 3.3 when + PY_SSIZE_T_CLEAN is defined. + +- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() + is now of type ``const char *`` rather of ``char *``. + +- Issue #29058: All stable API extensions added after Python 3.2 are now + available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of + the minimum Python version supporting this API. Library ------- diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425eaefa65..efc7d05b7d01b9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab9625ef5..833ea8e7b50894 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1082,9 +1082,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df55626d4bc23..bc8d11efa5258c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2483,6 +2483,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2491,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2514,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2530,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } From da88596a19d2cad0e85a2b3c3290b5b1d97a793d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:53:42 +0200 Subject: [PATCH 0149/1345] bpo-29845: Mark tests that use _testcapi as CPython-only (#711) (#725) (cherry picked from commit 24c738a9e91b8f46da6166663d8ce7ec18cec784) --- Lib/ctypes/test/test_structures.py | 5 ++++- Lib/test/test_atexit.py | 1 + Lib/test/test_coroutines.py | 1 + Lib/test/test_socket.py | 1 + Lib/test/test_tracemalloc.py | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 3eded7749ed95e..2e778fb1b43740 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -2,8 +2,8 @@ from ctypes import * from ctypes.test import need_symbol from struct import calcsize -import _testcapi import _ctypes_test +import test.support class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -202,7 +202,10 @@ class X(Structure): "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + @test.support.cpython_only + def test_packed_c_limits(self): # Issue 15989 + import _testcapi d = {"_fields_": [("a", c_byte)], "_pack_": _testcapi.INT_MAX + 1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25419cea7..c761076c4a0225 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -143,6 +143,7 @@ def test_bound_methods(self): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a69583b5f95ae6..2b79a17ea703f5 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2117,6 +2117,7 @@ def wrap(gen): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1508f20c9f0598..2497e47c668bc3 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -906,6 +906,7 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7ee606d60..742259b4396b98 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ def test_sys_xoptions_invalid(self): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. From 3d258b1eb453bcbc412d6b252f5bdceae0303f07 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 21:06:29 +0200 Subject: [PATCH 0150/1345] bpo-29116: Fix error messages for concatenating bytes and bytearray with unsupported type. (#709) (#723) (cherry picked from commit 6b5a9ec4788770c652bac3bf5d5a0a3b710b82ae) --- Objects/bytearrayobject.c | 2 +- Objects/bytesobject.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d69802508736..d456678d462f2f 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 30c1a7e24532da..c95a46d1846649 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1438,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } From d1f566f6bddc67e9870eaa75ae477208b9a199be Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 19 Mar 2017 20:55:16 -0700 Subject: [PATCH 0151/1345] bpo-29856: Fix typo in curses documentation (GH-730) (GH-731) From Shifted Dxit -> Shifted Exit in Doc/library/curses.rst (cherry picked from commit 64508780d72769e4c7afc67a511c057261c578f6) --- Doc/library/curses.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eafaea9b5c..d51085506c2a96 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1443,7 +1443,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ From 6fad4090ec9a27f8572bb00661b9890f01fb62f7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 Mar 2017 09:13:47 +0200 Subject: [PATCH 0152/1345] bpo-28876: bool of large range raises OverflowError (#699) (#734) (cherry picked from commit e46fb8611867fa3b407a813f53137929b7cb4a10) --- Lib/test/test_range.py | 17 +++++++++++++---- Misc/NEWS | 3 +++ Objects/rangeobject.c | 12 +++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e518f61e1f..679759ec6d676a 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ def test_large_operands(self): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ def _range_len(x): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Misc/NEWS b/Misc/NEWS index 310e982d78f03a..7e87a28145780b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -39,6 +39,9 @@ Core and Builtins - bpo-28893: Set correct __cause__ for errors about invalid awaitables returned from __aiter__ and __anext__. +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + - bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by Brian Coleman. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24d394..8f5fc434bd4955 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ From fcd79ada4e4cf335372538b34c8eb89cf09f84b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 Mar 2017 14:54:52 +0200 Subject: [PATCH 0153/1345] Fix "NotImplentedError" typo in constants documentation (#692) (#739) `NotImplentedError` --> `NotImplementedError` (cherry picked from commit 05f53735c8912f8df1077e897f052571e13c3496) --- Doc/library/constants.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742cee55bd55..469a3eed606ff0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. From e9213d929d7b0075539e87416f6a6fb86c27454b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 00:35:08 -0400 Subject: [PATCH 0154/1345] bpo-27593: Revise git SCM build info. (#744) (#745) Use --short form of git hash. Use output from "git describe" for tag. Expected outputs: 1. previous hg 2. previous git 3. updated git Release (tagged) build: 1. Python 3.7.0a0 (v3.7.0a0:4def2a2901a5, ... 2. Python 3.7.0a0 (v3.7.0a0^0:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (v3.7.0a0:05f53735c8, ... Development build: 1. Python 3.7.0a0 (default:41df79263a11, ... 2. Python 3.7.0a0 (master:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (heads/master-dirty:05f53735c8, ... "dirty" means the working tree has uncommitted changes. See "git help describe" for more info. (cherry picked from commit 554626ada769abf82a5dabe6966afa4265acb6a6) --- PCbuild/pythoncore.vcxproj | 4 ++-- configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7e9473638cbb2..6ea184877f99cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -414,8 +414,8 @@ - - + + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) diff --git a/configure b/configure index d4eccb1d68d856..abe1dc5abd4e1c 100755 --- a/configure +++ b/configure @@ -2743,8 +2743,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" diff --git a/configure.ac b/configure.ac index f00a2a6fa498f7..9eacf52559e8c2 100644 --- a/configure.ac +++ b/configure.ac @@ -37,8 +37,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" From 6efc812280b9303d37f51fbd061f7c31379fb72e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sun, 5 Mar 2017 19:58:06 -0800 Subject: [PATCH 0155/1345] Fixes the upload script to purge the CDN correctly and display success output. (#466) (#497) (cherry picked from commit e544b40faa5ab61b6aba691577d90b2b641f664d) --- Tools/msi/uploadrelease.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 305e84fc2d4b28..75840f2f851ecd 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,6 +8,7 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false false @@ -91,6 +92,7 @@ echo." /> + From 0acdea79cba3883c5e7035c7336fb0ce54435c03 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 6 Mar 2017 17:24:28 +0900 Subject: [PATCH 0156/1345] bpo-29719: Remove Date and Release field in whatsnew/3.6 (GH-500) (cherry picked from commit 2225ddaa9e64c086b2b6997b0c9ac50921f7aa85) (cherry picked from commit 4e1a065c20856a00d0fe88ce022b249170608058) --- Doc/whatsnew/3.6.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b76..a696af4a999c84 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: From 07e6cbd7b9e5b6047a608cedd5cc9b20891a3972 Mon Sep 17 00:00:00 2001 From: "n.d. parker" Date: Wed, 8 Mar 2017 23:27:46 +0100 Subject: [PATCH 0157/1345] Fix the only non-C90 comment to be C90 compatible. (#568) (cherry picked from commit 51b646a55a4a1c4d0df764c4404a062fbcc6b356) --- Include/pyport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d056..426822a81f075a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif From 75345c552d0889f4f63039d6063f371846c8f41f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 21:34:22 +1000 Subject: [PATCH 0158/1345] [3.6] bpo-29723: Consistently configure sys.path[0] (#636) Directory and zipfile execution previously added the parent directory of the directory or zipfile as sys.path[0] and then subsequently overwrote it with the directory or zipfile itself. This caused problems in isolated mode, as it overwrote the "stdlib as a zip archive" entry in sys.path, as the parent directory was never added. The attempted fix to that issue in bpo-29319 created the opposite problem in *non*-isolated mode, by potentially leaving the parent directory on sys.path instead of overwriting it. This change fixes the root cause of the problem by removing the whole "add-and-overwrite" dance for sys.path[0], and instead simply never adds the parent directory to sys.path in the first place. (cherry picked from commit d2977a3ae2cc6802921b1e3b6e9d13fcfbda872d) (cherry picked from commit c60948464fb0ec116ea227f6bce8a4bb8fb75257) --- Lib/test/test_cmd_line_script.py | 67 +++++++++++++++++++++++++++++ Misc/NEWS | 36 ++++++++++++++++ Modules/main.c | 72 +++++++++++++++++++------------- 3 files changed, 147 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Misc/NEWS b/Misc/NEWS index d715d306a34bc3..653510bb411791 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,42 @@ Python News +++++++++++ +What's New in Python 3.6.1 final? +================================= + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + + What's New in Python 3.6.1 release candidate 1 ============================================== diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..dd502119d985e9 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; From 360c49b9cf3c164fa1ee3228e1a68282d77c5e4e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 16 Mar 2017 11:03:03 -0700 Subject: [PATCH 0159/1345] Takes vcruntime140.dll from the correct source. (#684) (cherry picked from commit 9cd5e87bac51d7b901e3c36bf22728bb1693da59) --- Tools/msi/make_zip.proj | 9 +++------ Tools/nuget/make_pkg.proj | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72..b3588b7a0bae90 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c..464ef0456af47a 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + From 8c18fbeed1c7721b67f1726a6e9c41acef823135 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 00:35:08 -0400 Subject: [PATCH 0160/1345] bpo-27593: Revise git SCM build info. (#744) (#745) Use --short form of git hash. Use output from "git describe" for tag. Expected outputs: 1. previous hg 2. previous git 3. updated git Release (tagged) build: 1. Python 3.7.0a0 (v3.7.0a0:4def2a2901a5, ... 2. Python 3.7.0a0 (v3.7.0a0^0:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (v3.7.0a0:05f53735c8, ... Development build: 1. Python 3.7.0a0 (default:41df79263a11, ... 2. Python 3.7.0a0 (master:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (heads/master-dirty:05f53735c8, ... "dirty" means the working tree has uncommitted changes. See "git help describe" for more info. (cherry picked from commit 554626ada769abf82a5dabe6966afa4265acb6a6) (cherry picked from commit e9213d929d7b0075539e87416f6a6fb86c27454b) --- PCbuild/pythoncore.vcxproj | 4 ++-- configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7e9473638cbb2..6ea184877f99cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -414,8 +414,8 @@ - - + + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) diff --git a/configure b/configure index d4eccb1d68d856..abe1dc5abd4e1c 100755 --- a/configure +++ b/configure @@ -2743,8 +2743,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" diff --git a/configure.ac b/configure.ac index f00a2a6fa498f7..9eacf52559e8c2 100644 --- a/configure.ac +++ b/configure.ac @@ -37,8 +37,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" From 69c0db5050f623e8895b72dfe970392b1f9a0e2e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 02:32:38 -0400 Subject: [PATCH 0161/1345] Update docs and patchlevel for 3.6.1 final --- Include/patchlevel.h | 6 +++--- Misc/NEWS | 26 +++++++------------------- README.rst | 4 ++-- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b42a0db2f82007..54a9ab6ff78896 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 1 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.1rc1" +#define PY_VERSION "3.6.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 653510bb411791..4807bd0fcc3b3f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,10 @@ Python News +++++++++++ -What's New in Python 3.6.1 final? -================================= +What's New in Python 3.6.1? +=========================== -*Release date: XXXX-XX-XX* +*Release date: 2017-03-21* Core and Builtins ----------------- @@ -18,24 +18,12 @@ Core and Builtins are inadvertently modified when inserting the import location named on the command line. -- bpo-29714: Fix a regression that bytes format may fail when containing zero - bytes inside. - -Library -------- - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. Patch by Oren Milman. - -- bpo-28231: The zipfile module now accepts path-like objects for external - paths. +Build +----- -- bpo-26915: index() and count() methods of collections.abc.Sequence now - check identity before checking equality when do comparisons. +- bpo-27593: fix format of git information used in sys.version -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. +- Fix incompatible comment in python.h What's New in Python 3.6.1 release candidate 1 diff --git a/README.rst b/README.rst index 242572c5e361db..9ea036066877c9 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.6.1 release candidate 1 -================================================ +This is Python version 3.6.1 +============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI From fe30339534c602af1123e1402e44a1463f91f2e5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 03:46:26 +0900 Subject: [PATCH 0162/1345] bpo-29859: Fix error messages from return codes for pthread_* calls (GH-753) (cherry picked from commit d7fa6b259e00fca04dbf816bfcf4115fdda14bb7) --- Misc/NEWS | 3 +++ Python/thread_pthread.h | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7e87a28145780b..aad92480693e6a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + - bpo-29600: Fix wrapping coroutine return values in StopIteration. - bpo-28856: Fix an oversight that %b format for bytes should support objects diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 27e0dc84bcb52e..ba7393f03de6ae 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -143,6 +143,8 @@ typedef struct { } pthread_lock; #define CHECK_STATUS(name) if (status != 0) { perror(name); error = 1; } +#define CHECK_STATUS_PTHREAD(name) if (status != 0) { fprintf(stderr, \ + "%s: %s\n", name, strerror(status)); error = 1; } /* * Initialization. @@ -417,7 +419,7 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); - CHECK_STATUS("pthread_mutex_init"); + CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the Python-level mutex as a mutex because it can be @@ -427,7 +429,7 @@ PyThread_allocate_lock(void) status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); - CHECK_STATUS("pthread_cond_init"); + CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { PyMem_RawFree((void *)lock); @@ -452,10 +454,10 @@ PyThread_free_lock(PyThread_type_lock lock) * and must have the cond destroyed first. */ status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_destroy"); + CHECK_STATUS_PTHREAD("pthread_cond_destroy"); status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS("pthread_mutex_destroy"); + CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); PyMem_RawFree((void *)thelock); } @@ -472,7 +474,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); if (thelock->locked == 0) { success = PY_LOCK_ACQUIRED; @@ -494,13 +496,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, &thelock->mut, &ts); if (status == ETIMEDOUT) break; - CHECK_STATUS("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); } else { status = pthread_cond_wait( &thelock->lock_released, &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); } if (intr_flag && status == 0 && thelock->locked) { @@ -518,7 +520,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, } if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", @@ -536,16 +538,16 @@ PyThread_release_lock(PyThread_type_lock lock) dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); thelock->locked = 0; /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_signal"); + CHECK_STATUS_PTHREAD("pthread_cond_signal"); status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); } #endif /* USE_SEMAPHORES */ From 677df6ecade6ff503d3c3c93cf157ed12fec9386 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 20:21:49 -0400 Subject: [PATCH 0163/1345] Revert and fix Misc/NEWS after merge error in fca705d53397. (#755) --- Misc/NEWS | 805 +----------------------------------------------------- 1 file changed, 3 insertions(+), 802 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index aad92480693e6a..2e0f00be111427 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins - bpo-29859: Show correct error messages when any of the pthread_* calls in thread_pthread.h fails. +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + - bpo-29600: Fix wrapping coroutine return values in StopIteration. - bpo-28856: Fix an oversight that %b format for bytes should support objects @@ -28,813 +31,12 @@ Core and Builtins - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. -- bpo-29568: Escaped percent "%%" in the format string for classic string - formatting no longer allows any characters between two percents. - -- bpo-29714: Fix a regression that bytes format may fail when containing zero - bytes inside. - -- bpo-29695: bool(), float(), list() and tuple() no longer take keyword arguments. - The first argument of int() can now be passes only as positional argument. - -- bpo-20087: Prefer glibc's list of locale aliases to the X11 ones. - -- bpo-28893: Set correct __cause__ for errors about invalid awaitables - returned from __aiter__ and __anext__. - -- bpo-28876: ``bool(range)`` works even if ``len(range)`` - raises :exc:`OverflowError`. - -- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by - Brian Coleman. - -- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. - It should raise TypeError when kwargs is not a dict. But it might - cause segv when args=NULL and kwargs is not a dict. - -- bpo-28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. - Patch by Matthieu Dartiailh. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - -- bpo-29347: Fixed possibly dereferencing undefined pointers - when creating weakref objects. - -- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, - and AsyncFunctionDef ast nodes. docstring is not first stmt in their body - anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object - for module and class. - -- bpo-29438: Fixed use-after-free problem in key sharing dict. - -- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. - -- bpo-29546: Improve from-import error message with location - -- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - -- Issue #29337: Fixed possible BytesWarning when compare the code objects. - Warnings could be emitted at compile time. - -- Issue #29327: Fixed a crash when pass the iterable keyword argument to - sorted(). - -- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). - -- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. - -- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. - Calling function is up to 5% faster. - -- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII - whitespace, not only spaces. Patch by Robert Xiao. - -- Issue #28932: Do not include if it does not exist. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #18896: Python function can now have more than 255 parameters. - collections.namedtuple() now supports tuples with more than 255 elements. - -- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by - Chi Hsuan Yen. - -- Issue #26919: On Android, operating system data is now always encoded/decoded - to/from UTF-8, instead of the locale encoding to avoid inconsistencies with - os.fsencode() and os.fsdecode() which are already using UTF-8. - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug triggerable by a monkey-patched len() function. - -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. Patch written by INADA - Naoki. - -- Issue #28739: f-string expressions are no longer accepted as docstrings and - by ast.literal_eval() even if they do not include expressions. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28918: Fix the cross compilation of xxlimited when Python has been - built with Py_DEBUG defined. - -- Issue #23722: Rather than silently producing a class that doesn't support - zero-argument ``super()`` in methods, failing to pass the new - ``__classcell__`` namespace entry up to ``type.__new__`` now results in a - ``DeprecationWarning`` and a class that supports zero-argument ``super()``. - -- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of - a descriptor that is used inside that class no longer prevents calling the - __set_name__ method of other descriptors. - -- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate - the untested and undocumented ``sys.callstats()`` function. Remove the - ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, - :mod:`cProfile` or :mod:`profile` to profile function calls. - -- Issue #12844: More than 255 arguments can now be passed to a function. - -- Issue #28782: Fix a bug in the implementation ``yield from`` when checking - if the next instruction is YIELD_FROM. Regression introduced by WORDCODE - (issue #26647). - -- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 - encoders when a string returned by the error handler contains multiple - non-encodable characters (non-ASCII for the ASCII codec, characters out - of the U+0000-U+00FF range for Latin1). - -- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. - Improve speed of dict literal with constant keys up to 30%. - -- Issue #28532: Show sys.version when -V option is supplied twice. - -- Issue #27100: The with-statement now checks for __enter__ before it - checks for __exit__. This gives less confusing error messages when - both methods are missing. Patch by Jonathan Ellington. - -- Issue #28746: Fix the set_inheritable() file descriptor method on platforms - that do not have the ioctl FIOCLEX and FIONCLEX commands. - -- Issue #26920: Fix not getting the locale's charset upon initializing the - interpreter, on platforms that do not have langinfo. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28621: Sped up converting int to float by reusing faster bits counting - implementation. Patch by Adrian Wielgosik. - -- Issue #28580: Optimize iterating split table values. - Patch by Xiang Zhang. - -- Issue #28583: PyDict_SetDefault didn't combine split table when needed. - Patch by Xiang Zhang. - -- Issue #28128: Deprecation warning for invalid str and byte escape - sequences now prints better information about where the error - occurs. Patch by Serhiy Storchaka and Eric Smith. - -- Issue #28509: dict.update() no longer allocate unnecessary large memory. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused - keeping unreachable code. - -- Issue #28214: Improved exception reporting for problematic __set_name__ - attributes. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28183: Optimize and cleanup dict iteration. - -- Issue #26081: Added C implementation of asyncio.Future. - Original patch by Yury Selivanov. - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: Creating instances of range_iterator by calling range_iterator - type now is disallowed. Calling iter() on range instance is the only way. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when - hashes have same lower bits. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #28289: ImportError.__init__ now resets not specified attributes. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. - Patch by Soumya Sharma. - -- Issue #28086: Single var-positional argument of tuple subtype was passed - unscathed to the C-defined function. Now it is converted to exact tuple. - -- Issue #28214: Now __set_name__ is looked up on the class instead of the - instance. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28192: Don't import readline in isolated mode. - -- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. - Thanks Oren Milman. - -- Issue #27222: Clean up redundant code in long_rshift function. Thanks - Oren Milman. - -- Upgrade internal unicode databases to Unicode version 9.0.0. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly - optimize memcpy(). - -- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a - "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. - -- Issue #26182: Raise DeprecationWarning when async and await keywords are - used as variable/attribute/class/function name. - -- Issue #26182: Fix a refleak in code that raises DeprecationWarning. - -- Issue #28721: Fix asynchronous generators aclose() and athrow() to - handle StopAsyncIteration propagation properly. - -- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD - opcodes. - -Extension Modules ------------------ - -- Issue #29169: Update zlib to 1.2.11. Library ------- - bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. -- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords - are not strings. Patch by Michael Seifert. - -- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" - or "errors" of sys.stdin or sys.stdout are not set or are not strings. - -- bpo-28692: Using non-integer value for selecting a plural form in gettext is - now deprecated. - -- bpo-26121: Use C library implementation for math functions erf() and erfc(). - -- bpo-29619: os.stat() and os.DirEntry.inode() now convert inode (st_ino) using - unsigned integers. - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. - -- bpo-29645: Speed up importing the webbrowser module. webbrowser.register() - is now thread-safe. - -- bpo-28231: The zipfile module now accepts path-like objects for external - paths. - -- bpo-26915: index() and count() methods of collections.abc.Sequence now - check identity before checking equality when do comparisons. - -- bpo-28682: Added support for bytes paths in os.fwalk(). - -- bpo-29623: Allow use of path-like object as a single argument in - ConfigParser.read(). Patch by David Ellis. - -- bpo-9303: Migrate sqlite3 module to _v2 API. Patch by Aviv Palivoda. - -- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback - implemented in C. - -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C - to accept None argument as their pure Python implementation. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher - and its subclasses can now be used as a decorator. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts - PathLike objects. Patch by Sayan Chowdhury. - -- bpo-28518: Start a transaction implicitly before a DML statement. - Patch by Aviv Palivoda. - -- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. - Patch by Nikolay Kim. - -- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes - '~' in the set of characters that is not quoted by default. Patch by - Christian Theune and Ratnadeep Debnath. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - -- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, - improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, - Manuel Krebber, and Łukasz Langa. - -- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python - 3.6.0: check minimum and maximum years. - -- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir - -- Issue #29444: Fixed out-of-bounds buffer access in the group() method of - the match object. Based on patch by WGH. - -- Issue #29377: Add SlotWrapperType, MethodWrapperType, and - MethodDescriptorType built-in types to types module. - Original patch by Manuel Krebber. - -- Issue #29218: Unused install_misc command is now removed. It has been - documented as unused since 2000. Patch by Eric N. Vander Weele. - -- Issue #29368: The extend() method is now called instead of the append() - method when unpickle collections.deque and other list-like objects. - This can speed up unpickling to 2 times. - -- Issue #29338: The help of a builtin or extension class now includes the - constructor signature if __text_signature__ is provided for the class. - -- Issue #29335: Fix subprocess.Popen.wait() when the child process has - exited to a stopped instead of terminated state (ex: when under ptrace). - -- Issue #29290: Fix a regression in argparse that help messages would wrap at - non-breaking spaces. - -- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. - -- Issue #29197: Removed deprecated function ntpath.splitunc(). - -- Issue #29210: Removed support of deprecated argument "exclude" in - tarfile.TarFile.add(). - -- Issue #29219: Fixed infinite recursion in the repr of uninitialized - ctypes.CDLL instances. - -- Issue #29192: Removed deprecated features in the http.cookies module. - -- Issue #29193: A format string argument for string.Formatter.format() - is now positional-only. - -- Issue #29195: Removed support of deprecated undocumented keyword arguments - in methods of regular expression objects. - -- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. - KeyError could be raised when cached function with full cache was - simultaneously called from differen threads with the same uncached arguments. - -- Issue #20804: The unittest.mock.sentinel attributes now preserve their - identity when they are copied or pickled. - -- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with - leading dots could match related hostnames again (e.g. .b.c matches a.b.c). - Patch by Milan Oberkirch. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #28985: Update authorizer constants in sqlite3 module. - Patch by Dingyuan Wang. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that - doesn't own its elements as limits. - -- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, - instead of /bin/sh. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #26937: The chown() method of the tarfile.TarFile class does not fail - now when the grp module cannot be imported, as for example on Android - platforms. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. A deprecation warning is now - emitted if the index file is missed and recreated in the 'r' and 'w' modes - (will be an error in future Python releases). - -- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in - re.sub() replacement templates regular expressions now are errors. - -- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): - call warnings.showwarning() if it was overriden inside the context manager. - -- Issue #27172: To assist with upgrades from 2.7, the previously documented - deprecation of ``inspect.getfullargspec()`` has been reversed. This decision - may be revisited again after the Python 2.7 branch is no longer officially - supported. - -- Issue #28740: Add sys.getandroidapilevel(): return the build time API version - of Android as an integer. Function only available on Android. - -- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and - :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by - Omar Sandoval. - -- Issue #28752: Restored the __reduce__() methods of datetime objects. - -- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created - by re.compile(), become comparable (only x==y and x!=y operators). This - change should fix the issue #18383: don't duplicate warning filters when the - warnings module is reloaded (thing usually only done in unit tests). - -- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was - deprecated in 3.4 and undocumented prior to that. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #28548: In the "http.server" module, parse the protocol version if - possible, to avoid using HTTP 0.9 in some error responses. - -- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. - Patch by Vajrasky Kok - -- Issue #28563: Fixed possible DoS and arbitrary code execution when handle - plural form selections in the gettext module. The expression parser now - supports exact syntax supported by GNU gettext. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #24241: The webbrowser in an X environment now prefers using the - default browser directly. Also, the webbrowser register() function now has - a documented 'preferred' argument, to specify browsers to be returned by - get() with no arguments. Patch by David Steele - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after - a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant - newline after a year's calendar. Based on patch by Xiang Zhang. - -- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept - non-None value is passed to it.send(val). - -- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. - -- Issue #25953: re.sub() now raises an error for invalid numerical group - reference in replacement template even if the pattern is not found in - the string. Error message for invalid group reference now includes the - group index and the position of the reference. - Based on patch by SilentGhost. - -- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead - of 1, 10, 100,... for autoranging. - -- Issue #28115: Command-line interface of the zipfile module now uses argparse. - Added support of long options. - -- Issue #18219: Optimize csv.DictWriter for large number of columns. - Patch by Mariatta Wijaya. - -- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. - -- Issue #23214: In the "io" module, the argument to BufferedReader and - BytesIO's read1() methods is now optional and can be -1, matching the - BufferedIOBase specification. - -- Issue #28480: Fix error building socket module when multithreading is - disabled. - -- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line - options which were deprecated since Python 3.3. - -- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 - to make benchmarks more reliable. - -- Issue #28240: timeit autorange now uses a single loop iteration if the - benchmark takes less than 10 seconds, instead of 10 iterations. - "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds - instead of 40 seconds. - -- Distutils.sdist now looks for README and setup.py files with case - sensitivity. This behavior matches that found in Setuptools 6.0 and - later. See `setuptools 100 - `_ for rationale. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by - Ned Batchelder. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #27998: Fixed bytes path support in os.scandir() on Windows. - Patch by Eryk Sun. - -- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28380: unittest.mock Mock autospec functions now properly support - assert_called, assert_not_called, and assert_called_once. - -- Issue #28229: lzma module now supports pathlib. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. - -- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. - -- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. - Original patch by Oren Milman. - -- Issue #27358: Optimized merging var-keyword arguments and improved error - message when passing a non-mapping as a var-keyword argument. - -- Issue #28257: Improved error message when passing a non-iterable as - a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Issue #28228: imghdr now supports pathlib. - -- Issue #28226: compileall now supports pathlib. - -- Issue #28314: Fix function declaration (C flags) for the getiterator() method - of xml.etree.ElementTree.Element. - -- Issue #28148: Stop using localtime() and gmtime() in the time - module. - - Introduced platform independent _PyTime_localtime API that is - similar to POSIX localtime_r, but available on all platforms. Patch - by Ed Schouten. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18844: random.choices() now has k as a keyword-only argument - to improve the readability of common cases and come into line - with the signature used in other languages. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - Added the master parameter in the DisplayStyle constructor. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is - interrupted by a signal and a signal handler raises a Python exception. - -- Issue #28200: Fix memory leak on Windows in the os module (fix - path_converter() function). - -- Issue #25400: RobotFileParser now correctly returns default values for - crawl_delay and request_rate. Initial patch by Peter Wirtz. - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #22493: Warning message emitted by using inline flags in the middle of - regular expression now contains a (truncated) regex pattern. - Patch by Tim Graham. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. - Patch by Eryk Sun. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. - -- Issue #28368: Refuse monitoring processes if the child watcher has - no loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by Коренберг Марк. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by Коренберг Марк. - -- Issue #27972: Prohibit Tasks to await on themselves. - -- Issue #24142: Reading a corrupt config file left configparser in an - invalid state. Original patch by Florian Höch. - -- Issue #29581: ABCMeta.__new__ now accepts **kwargs, allowing abstract base - classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. - -Windows -------- - -- bpo-29579: Removes readme.txt from the installer. - -- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) - -- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default - -- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c - -- Issue #28402: Adds signed catalog files for stdlib on Windows. - -- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by - Eryk Sun) - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #28161: Opening CON for write access fails - -- Issue #28162: WindowsConsoleIO readall() fails if first line starts with - Ctrl+Z - -- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to - _open_osfhandle - -- Issue #28164: _PyIO_get_console_type fails for various paths - -- Issue #28137: Renames Windows path file to ._pth - -- Issue #28138: Windows ._pth file should allow import site - -C API ------ - -- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with - a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 - and 0x03060000 (not including) or 0x03060100 or higher. Added functions - PySlice_Unpack() and PySlice_AdjustIndices(). - -- Issue #29083: Fixed the declaration of some public API functions. - PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in - limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and - Py_BuildValue() were not available in limited API of version < 3.3 when - PY_SSIZE_T_CLEAN is defined. - -- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() - is now of type ``const char *`` rather of ``char *``. - -- Issue #29058: All stable API extensions added after Python 3.2 are now - available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of - the minimum Python version supporting this API. - -Library -------- - - bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords are not strings. Patch by Michael Seifert. @@ -966,7 +168,6 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. -- Issue #24932: Use proper command line parsing in _testembed - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, From 1688e64925967ba41df682211efaa8d066c4b3e2 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 20:39:58 -0400 Subject: [PATCH 0164/1345] Bump to 3.6.2rc1 development. --- Include/patchlevel.h | 2 +- Misc/NEWS | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 54a9ab6ff78896..a658e9f5840e1c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.1" +#define PY_VERSION "3.6.1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 4807bd0fcc3b3f..bb37f0479ebc34 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,56 @@ Python News +++++++++++ +What's New in Python 3.6.2 release candidate 1? +=============================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + + What's New in Python 3.6.1? =========================== From af839fe2fef12dbbc8083fc7f01daee48a85a916 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 22 Mar 2017 07:45:23 +0200 Subject: [PATCH 0165/1345] bpo-29864: Don't use Py_SIZE for dict object. (#747) (#750) --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a0c1977037b9e6..8bcc931148beaa 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1929,7 +1929,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } From 72c51136c6269112f4c37ef14f972c89b92c272c Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 22 Mar 2017 15:26:53 +0800 Subject: [PATCH 0166/1345] fix function name in tabnanny documentation (GH-762) --- Doc/library/tabnanny.rst | 8 ++++---- Lib/tabnanny.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fbabb2023..dfe688a2f93e0c 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56a392ae6..bfb670c9027d76 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): From c0f3e21a23356133c30926b6cdf3e0e70cbc8486 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 17:14:03 +0900 Subject: [PATCH 0167/1345] doc: minor fix for library/profile (GH-766) (cherry picked from commit bd3d8ba3b22da0bad018b53a3e6610ae03c5aa49) --- Doc/library/profile.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index bd67fe486abf77..b4b1479e2ceba4 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` From 906118d8c68160ed4a3d15cec803d1ee57836517 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 21:09:41 +0900 Subject: [PATCH 0168/1345] bpo-28331: fix impl-detail label is removed when content is translated. (GH-769) (cherry picked from commit c351ce6a2c923c5016e48ecbf7b1e4833031d154) --- Doc/tools/extensions/pyspecific.py | 14 ++++++++++++-- Doc/tools/templates/dummy.html | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Doc/tools/templates/dummy.html diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1141d6c5ac1891..865f2edee02205 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -21,6 +21,7 @@ from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 00000000000000..6e43be23230b54 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} From ee51327a2329eb9c2bfbb6ea673cfab4299450b0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 22 Mar 2017 18:56:56 -0700 Subject: [PATCH 0169/1345] Remove an outdated statement in execution model docs (GH-754) (GH-775) (cherry picked from commit fad7f1560669af1766c583c7ef242c55d8c8de41) --- Doc/reference/executionmodel.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92ed460f3..f0dbbd1edf6321 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full From 90eafdb1546604c0e9bc207723484b6816570cfe Mon Sep 17 00:00:00 2001 From: Christophe Zeitouny Date: Fri, 24 Mar 2017 04:20:40 -0700 Subject: [PATCH 0170/1345] faulthandler: Restore the old sigaltstack during teardown (GH-777) (GH-797) (cherry picked from commit 20fbf8accd494fd15b0fc4c84928178c71ead4d1) --- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/faulthandler.c | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Misc/ACKS b/Misc/ACKS index 03afeb8f3876ee..2b23702f22910c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1709,6 +1709,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS index 1835d1e389d725..687c5c06ef9bdb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + - bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. - bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb7d17760..2f8b624fd160d9 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -124,6 +124,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -1310,7 +1311,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1367,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } From cc3331fec8b7a61c3f06c097eac85bfa38490758 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 24 Mar 2017 14:45:34 +0100 Subject: [PATCH 0171/1345] bpo-29861: release references to multiprocessing Pool tasks (#743) (#800) * bpo-29861: release references to multiprocessing Pool tasks (#743) * bpo-29861: release references to multiprocessing Pool tasks Release references to tasks, their arguments and their results as soon as they are finished, instead of keeping them alive until another task arrives. * Comments in test (cherry picked from commit 8988945cdc27ffa86ba8c624e095b51c459f5154) * Fix Misc/NEWS (hopefully) --- Lib/multiprocessing/pool.py | 7 ++++++- Lib/test/_test_multiprocessing.py | 28 ++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf42614d59eb..ae8cec44796b21 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -128,6 +128,8 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) @@ -402,10 +404,11 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): if set_length: util.debug('doing set_length()') set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +448,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +465,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5f47825466de6..1d3bb0f8bae781 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -1738,6 +1739,19 @@ def raise_large_valuerror(wait): time.sleep(wait) raise ValueError("x" * 1024**2) +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1746,6 +1760,7 @@ def exception_throwing_generator(total, when): raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -2000,6 +2015,19 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.time() - t_start, 0.9) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index 687c5c06ef9bdb..0c7eaea5a8b65c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. + - bpo-29884: faulthandler: Restore the old sigaltstack during teardown. Patch by Christophe Zeitouny. From 8c8785b2f8e4048cef350f89c686266f4519b67c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 24 Mar 2017 21:46:25 +0200 Subject: [PATCH 0172/1345] =?UTF-8?q?bpo-25803:=20Avoid=20incorrect=20erro?= =?UTF-8?q?rs=20raised=20by=20Path.mkdir(exist=5Fok=3DTrue)=E2=80=A6=20(#8?= =?UTF-8?q?06)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit when the OS gives priority to errors such as EACCES over EEXIST. (cherry picked from commit af7b9ec5c855366feef4c67dc492d64b3baf84ca) --- Lib/pathlib.py | 32 +++++++++++++++----------------- Lib/test/test_pathlib.py | 5 +++++ Misc/NEWS | 3 +++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f347216b183a9..8c1cb96bad3075 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1220,25 +1220,23 @@ def touch(self, mode=0o666, exist_ok=True): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True) + self._accessor.mkdir(self, mode) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5802c1cb..3ff9726d4359ec 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1776,6 +1776,11 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': diff --git a/Misc/NEWS b/Misc/NEWS index 0c7eaea5a8b65c..9798a3a4490cbe 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + - bpo-29861: Release references to tasks, their arguments and their results as soon as they are finished in multiprocessing.Pool. From 74bfcc314b188e1e8c90e442270e36d6510755ee Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 24 Mar 2017 14:18:53 -0700 Subject: [PATCH 0173/1345] bpo-28810: Document remaining bytecode changes in 3.6 (GH-651) (GH-808) (cherry picked from commit 8f9e1bbf2dbdf46a0bf920279568a31460043376) --- Doc/library/dis.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 11d056901b7ad5..c795782034a464 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -1105,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: From 5dafaece67cb576ef96579fb5ddcbf62e0827b0d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Mar 2017 20:04:40 -0700 Subject: [PATCH 0174/1345] bpo-29892: Fix wrong markup on doc-lib-functions (GH-802) (GH-810) (cherry picked from commit 29540cdf6c66df9f806375a95078c0c63192ef78) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037bd9eef19..64c9569766cc7d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1072,7 +1072,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 From 8b82236952619c3838865ff535e5ce77b59b4a78 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Mar 2017 03:41:59 -0700 Subject: [PATCH 0175/1345] bpo-29862: Fix grammar in importlib.reload() exception (GH-809) (GH-811) (cherry picked from commit 9f0aa4843f8c26937d5817f27cac4aae9c0a034f) --- Lib/importlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e05f268..8b11d22b024ffa 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: From ceff32fb833e7be42508ffa73344e2e56865e54b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 26 Mar 2017 13:58:13 -0400 Subject: [PATCH 0176/1345] bpo-29888: Fix the link referring to the "Python download page" (GH-824) (GH-826) (cherry picked from commit f8beb9831acd5cf80b9c56aea5864e16118c5400) --- Doc/tools/templates/download.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3abc81c7..3a05cb697938fd 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,7 +42,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

From d2fc7824104172b47cdcea09d7ce699d5bfc2ba6 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Mar 2017 19:53:37 -0700 Subject: [PATCH 0177/1345] import sys before we use it on line 9 (GH-828) (GH-833) (cherry picked from commit 0579e81f30d00da562f021760d5b6a9c35186520) --- Lib/idlelib/pyshell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391ca006fc..dd6c997abc587b 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -25,7 +27,6 @@ import re import socket import subprocess -import sys import threading import time import tokenize From abc68484b2c523b500802cb8e8913531e0ff9a8d Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sun, 26 Mar 2017 22:53:57 -0500 Subject: [PATCH 0178/1345] Treat Sphinx warnings as errors (GH-832) (GH-834) * Treat Sphinx warnings as errors (GH-832) (cherry picked from commit 334e9ec938ea9876baadef15edb135d6d2aff30c) * Remove unused suspicious rules --- .travis.yml | 2 +- Doc/tools/susp-ignored.csv | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66f03dc7161a7f..3496a1abba2c97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: - cd Doc - make venv script: - - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q" + - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" - os: linux language: c compiler: clang diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e03119ae9730..166374cb90cec1 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -324,6 +324,4 @@ whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" whatsnew/changelog,,`,"for readability (was ""`"")." From bc2031470eac4b199692921b4e97730c85ceb678 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 27 Mar 2017 00:35:56 -0500 Subject: [PATCH 0179/1345] Minor test cleanup (GH-837) (GH-838) * Remove unused test file * Remove duplicated text in sndhdrdata README (cherry picked from commit b8a7daf077dab18e9e3701c5380b542ae0aa9a94) --- Lib/test/185test.db | Bin 16384 -> 0 bytes Lib/test/sndhdrdata/README | 7 ------- 2 files changed, 7 deletions(-) delete mode 100644 Lib/test/185test.db diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc0961fa22527dcb5d947a6a0fd5480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI!F$%&k6vpwdEl39?E*_+VxVZETo - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - From 08f9f71162dedeec51ad99bac1b26eb0ab014400 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Mar 2017 16:46:31 -0700 Subject: [PATCH 0180/1345] Drop the standard gcc test build on Travis (GH-853) (GH-859) Instead have gcc be used for the coverage build so gcc is exercised in at least one place. (cherry picked from commit ad2f9e2c8a0b44b3e6aec9d28ba59e13239236f7) --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3496a1abba2c97..5fee6cd7673324 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,11 @@ os: - linux # macOS builds are disabled as the machines are under-provisioned on Travis, # adding up to an extra hour completing a full CI run. - #- osx compiler: - clang - - gcc + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. env: - TESTING=cpython @@ -32,7 +32,7 @@ matrix: include: - os: linux language: python - python: 3.5 + python: 3.6 env: - TESTING=docs before_script: @@ -42,7 +42,7 @@ matrix: - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" - os: linux language: c - compiler: clang + compiler: gcc env: - TESTING=coverage before_script: From 596506216104613591218c2896cdb49fa0b7e5bb Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 27 Mar 2017 17:02:29 -0700 Subject: [PATCH 0181/1345] bpo-29677: DOC: clarify documentation for `round` (GH-357) (GH-862) (cherry picked from commit 6003db7db5fec545c01923c198a5fdfca5a91538) --- Doc/library/functions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 64c9569766cc7d..2c0ea71c47f413 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1262,7 +1262,8 @@ are always available. They are listed here in alphabetical order. closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. .. note:: From f01de61a8efea8319c65365898982f929d59a895 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 29 Mar 2017 00:25:26 +0900 Subject: [PATCH 0182/1345] bpo-29643: Fix check for --enable-optimizations (GH-869) The presence of the ``--enable-optimizations`` flag is indicated by the value of ``$enableval``, but the configure script was checking ``$withval``, resulting in the ``--enable-optimizations`` flag being effectively ignored. (cherry picked from commit 8cea5929f52801b0ce5928b46ef836e99a24321a) --- Misc/NEWS | 5 +++++ configure | 2 +- configure.ac | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 9798a3a4490cbe..45c5c7828701f8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -60,6 +60,11 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +Build +----- + +- bpo-29643: Fix ``--enable-optimization`` didn't work. + What's New in Python 3.6.1? =========================== diff --git a/configure b/configure index abe1dc5abd4e1c..8bdb82ba4a7bce 100755 --- a/configure +++ b/configure @@ -6521,7 +6521,7 @@ $as_echo_n "checking for --enable-optimizations... " >&6; } # Check whether --enable-optimizations was given. if test "${enable_optimizations+set}" = set; then : enableval=$enable_optimizations; -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 diff --git a/configure.ac b/configure.ac index 9eacf52559e8c2..6b331d5742745f 100644 --- a/configure.ac +++ b/configure.ac @@ -1274,7 +1274,7 @@ Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), [ -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' AC_MSG_RESULT(yes); From c4021af50526f488c0c280e7c7eaa83ef80ae1df Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Mar 2017 09:33:54 -0700 Subject: [PATCH 0183/1345] bpo-16011: clarify that 'in' always returns a boolean value (GH-874) (cherry picked from commit 0ae7c8bd614d3aa1fcaf2d71a10ff1148c80d9b5) --- Doc/reference/expressions.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f4a82699b0be12..d80768ac07dc0e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). From 346dcd65e6b832a35b4cfc15b7309b51a38e9ca2 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 29 Mar 2017 12:50:28 +0800 Subject: [PATCH 0184/1345] bpo-28699: fix abnormal behaviour of pools in multiprocessing.pool (GH-882) an exception raised at the very first of an iterable would cause pools behave abnormally (swallow the exception or hang) --- Lib/multiprocessing/pool.py | 79 +++++++++++++++++++++---------- Lib/test/_test_multiprocessing.py | 59 ++++++++++++++++++++++- Misc/NEWS | 4 ++ 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ae8cec44796b21..a545f3c1a18961 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -133,6 +133,10 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -277,6 +281,17 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -285,15 +300,23 @@ def imap(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -304,15 +327,23 @@ def imap_unordered(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -323,7 +354,7 @@ def apply_async(self, func, args=(), kwds={}, callback=None, if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -354,8 +385,14 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -377,33 +414,27 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) finally: task = taskseq = job = None else: diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1d3bb0f8bae781..771bbf24265b91 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1755,6 +1755,8 @@ def __del__(self): class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): + if when == -1: + raise SayWhenError("Somebody said when") for i in range(total): if i == when: raise SayWhenError("Somebody said when") @@ -1833,6 +1835,32 @@ def test_map_chunksize(self): except multiprocessing.TimeoutError: self.fail("pool.map_async with chunksize stalled on null list") + def test_map_handle_iterable_exception(self): + if self.TYPE == 'manager': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # SayWhenError seen at the very first of the iterable + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + # again, make sure it's reentrant + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(10, 3), 1) + + class SpecialIterable: + def __iter__(self): + return self + def __next__(self): + raise SayWhenError + def __len__(self): + return 1 + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + def test_async(self): res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) get = TimingWrapper(res.get) @@ -1863,6 +1891,13 @@ def test_imap_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1) for i in range(3): self.assertEqual(next(it), i*i) @@ -1889,6 +1924,17 @@ def test_imap_unordered_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap_unordered(sqr, exception_throwing_generator(10, 3), 1) @@ -1970,7 +2016,7 @@ def test_traceback(self): except Exception as e: exc = e else: - raise AssertionError('expected RuntimeError') + self.fail('expected RuntimeError') self.assertIs(type(exc), RuntimeError) self.assertEqual(exc.args, (123,)) cause = exc.__cause__ @@ -1984,6 +2030,17 @@ def test_traceback(self): sys.excepthook(*sys.exc_info()) self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + # _helper_reraises_exception should not make the error + # a remote exception + with self.Pool(1) as p: + try: + p.map(sqr, exception_throwing_generator(1, -1), 1) + except Exception as e: + exc = e + else: + self.fail('expected SayWhenError') + self.assertIs(type(exc), SayWhenError) + self.assertIs(exc.__cause__, None) @classmethod def _test_wrapped_exception(cls): diff --git a/Misc/NEWS b/Misc/NEWS index 45c5c7828701f8..fbaa840638ff97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,10 @@ Core and Builtins Library ------- +- bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an + exception at the very first of an iterable may swallow the exception or + make the program hang. Patch by Davin Potts and Xiang Zhang. + - bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) when the OS gives priority to errors such as EACCES over EEXIST. From 2609c9ee78c53d501914a5a90dbe094d9a8c3c97 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 29 Mar 2017 19:10:07 -0700 Subject: [PATCH 0185/1345] bpo-29677: DOC: clarify documentation for `round` (GH-877) (GH-892) (cherry picked from commit 85deefcf61d3cc192846f41a4ccc6df17da60c98) --- Doc/library/functions.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2c0ea71c47f413..90fb07af227d98 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1254,9 +1254,9 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are @@ -1266,6 +1266,9 @@ are always available. They are listed here in alphabetical order. negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, From da6ad2f780d187fbfdea330d1037766ae7bdb778 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 29 Mar 2017 22:29:00 -0700 Subject: [PATCH 0186/1345] bpo-29917: DOC: Remove link from PyMethodDef (#890) (#894) (cherry picked from commit c3c7ef088583cc12bd218138036d1edb6de9c63f) --- Doc/c-api/structures.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f48119391f2bbd..c080f317bee9d9 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. From 1b43a959fb1dd2e08dbf51bbc644df5ff2eb357e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 29 Mar 2017 23:56:52 -0700 Subject: [PATCH 0187/1345] Remove an unrequired TODO in test_urllib2. (#897) (#900) (cherry picked from commit e6911a44f69c0d302db60f49952a9cf69da69a2b) --- Lib/test/test_urllib2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f87162a62..308acb3724cad1 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1414,7 +1414,6 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { From a6b4e1902250d6f28ca6d083ce1c8d7e9b91974b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 Mar 2017 18:08:21 +0300 Subject: [PATCH 0188/1345] bpo-27863: Fixed multiple crashes in ElementTree. (#765) (#903) (cherry picked from commit 576def096ec7b64814e038f03290031f172886c3) --- Lib/test/test_xml_etree.py | 112 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 + Modules/_elementtree.c | 100 +++++++++++++++++---------------- 3 files changed, 167 insertions(+), 48 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1cb8febf..dbdad23a7423f2 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1879,6 +1879,118 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) diff --git a/Misc/NEWS b/Misc/NEWS index fbaa840638ff97..2caa8f360d122d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions + and wrong types. + - bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an exception at the very first of an iterable may swallow the exception or make the program hang. Patch by Davin Potts and Xiang Zhang. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e61127d6..e3350d194da18a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -2147,6 +2149,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2404,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2498,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2602,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { From bf4bb2e43030661e568d5d4b046e8b9351cc164c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 Mar 2017 19:46:59 +0300 Subject: [PATCH 0189/1345] bpo-29935: Fixed error messages in the index() method of tuple, list and deque (#887) (#907) when pass indices of wrong type. (cherry picked from commit d4edfc9abffca965e76ebc5957a92031a4d6c4d4) --- Include/ceval.h | 1 + Misc/NEWS | 3 +++ Modules/_collectionsmodule.c | 4 ++-- Objects/listobject.c | 4 ++-- Objects/tupleobject.c | 4 ++-- Python/ceval.c | 26 +++++++++++++++++++++----- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 89c6062f11e17c..1e482729a1cc98 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -216,6 +216,7 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif diff --git a/Misc/NEWS b/Misc/NEWS index 2caa8f360d122d..e8cd8f0221102e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-29935: Fixed error messages in the index() method of tuple, list and deque + when pass indices of wrong type. + - bpo-29859: Show correct error messages when any of the pthread_* calls in thread_pthread.h fails. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c64e77de2..30157701d70b47 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5efe5b0fe..cde281a0f6f9dc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499e72c7dd..52f20f4fe8a2b1 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); diff --git a/Python/ceval.c b/Python/ceval.c index 9cac771abdf65b..bce86ab12cf2fb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5074,14 +5074,10 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. Return 0 on error, 1 on success. */ -/* Note: If v is NULL, return success without storing into *pi. This - is because_PyEval_SliceIndex() is called by apply_slice(), which can be - called by the SLICE opcode with v and/or w equal to NULL. -*/ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { - if (v != NULL) { + if (v != Py_None) { Py_ssize_t x; if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5099,6 +5095,26 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + Py_ssize_t x; + if (PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} + + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" From 7b5b1379ac2c1e89ebf90b88b5d32457910e975e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 30 Mar 2017 12:27:19 -0700 Subject: [PATCH 0190/1345] bpo-29928: Add f-string to the Glossary (GH-864) (GH-914) (cherry picked from commit 33db068dac7686e37736f7ecf8abb2aee0345cf2) --- Doc/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 495934afe74c8b..81238414efe49d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -320,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending From 599bb181036f724629a515317f0f39520950d51c Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Thu, 30 Mar 2017 12:48:23 -0700 Subject: [PATCH 0191/1345] bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. (#911) * bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. Fix the use of recursion in itertools.chain.from_iterable. Using recursion is unnecessary, and can easily cause stack overflows, especially when building in low optimization modes or with Py_DEBUG enabled. (cherry picked from commit 5466d4af5fe76ec0a5fbc8a05675287d9e8e9d14) --- Lib/test/test_itertools.py | 8 ++++++ Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 52 ++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57caade12b..c431f0dc6e1d7f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,14 @@ def gen2(x): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Misc/NEWS b/Misc/NEWS index e8cd8f0221102e..5ea4f66e76fb39 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering + long runs of empty iterables. + - bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions and wrong types. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cbee31c35..8e22ec9c4b7cc1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1864,33 +1864,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * From 8d1f935945ca8cf3665315e0f82358fffe689353 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 30 Mar 2017 23:15:51 -0700 Subject: [PATCH 0192/1345] Remove catching OSError in ftphandler test. Only URLError is raised in urllib.request module. (#918) (#920) (cherry picked from commit ed3dd1c02af6872bd0748f7b9a5dadb89f7b830f) --- Lib/test/test_urllib2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 308acb3724cad1..b537fca3c31adf 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -830,7 +830,6 @@ def test_file(self): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -839,7 +838,7 @@ def test_file(self): try: h.file_open(req) # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1689,7 +1688,6 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): From 4c75fbb485c0e42181aab95c2ae92c597915827c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 31 Mar 2017 15:43:10 +0900 Subject: [PATCH 0193/1345] bpo-29952: Use usual terminology of dict (GH-922) s/keys and elements/keys and values/ (cherry picked from commit cdcac039fb447f2ab04efcacbe663751bb2cb4ec) --- Doc/reference/expressions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index d80768ac07dc0e..c4f6c55c7ca3aa 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. From 0a17e584461b14ff65ec287048f53911dbb22222 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 31 Mar 2017 02:12:18 -0700 Subject: [PATCH 0194/1345] bpo-28810: Update lnotab_notes.txt (GH-665) (GH-919) (cherry picked from commit 9135275cba680902e6caf29461f0423dc570190d) --- Objects/lnotab_notes.txt | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 515375772e6317..3dab2b98661695 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. From 7d5d13d8d003ae5b62bb8c9ef1d1f310eaabc506 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 31 Mar 2017 23:23:49 +0300 Subject: [PATCH 0195/1345] =?UTF-8?q?bpo-29953:=20Fix=20memory=20leaks=20i?= =?UTF-8?q?n=20the=20replace()=20method=20of=20datetime=20and=20t=E2=80=A6?= =?UTF-8?q?=20(#933)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objects when pass out of bound fold argument. (cherry picked from commit 314d6fca36a4eaa0541218431d14804fadec6488) --- Lib/test/datetimetester.py | 5 +++++ Misc/NEWS | 3 +++ Modules/_datetimemodule.c | 21 ++++++++++----------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2350125f6d6dbe..bccd97aa3c7f60 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4313,6 +4313,11 @@ def test_replace(self): dt = dt.replace(fold=1, tzinfo=Eastern) self.assertEqual(t.replace(tzinfo=None).fold, 1) self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) # Check that fold is a keyword-only argument with self.assertRaises(TypeError): t.replace(1, 1, 1, None, 1) diff --git a/Misc/NEWS b/Misc/NEWS index 5ea4f66e76fb39..c1bcdc1e93fc5d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- bpo-29953: Fixed memory leaks in the replace() method of datetime and time + objects when pass out of bound fold argument. + - bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering long runs of empty iterables. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f4a90999..c2ad9a203e9652 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3926,16 +3926,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5019,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); From efde51ad54c58353f25ff80c8d30dbee82ef33a3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 1 Apr 2017 23:29:31 +0900 Subject: [PATCH 0196/1345] bpo-29949: Fix set memory usage regression (GH-945) Revert "Minor factoring: move redundant resize scaling logic into the resize function." This reverts commit 4897300276d870f99459c82b937f0ac22450f0b6. (cherry picked from commit e82cf8675bacd7a03de508ed11865fc2701dcef5) --- Misc/NEWS | 2 ++ Objects/setobject.c | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c1bcdc1e93fc5d..e61a1652381fa0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-29949: Fix memory usage regression of set and frozenset object. + - bpo-29935: Fixed error messages in the index() method of tuple, list and deque when pass indices of wrong type. diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d3600d9fad..9fe28138c0012b 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -646,8 +645,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -990,7 +989,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1510,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * From 4415b823a08163a8728d13ff06565efb12353fc0 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Apr 2017 19:57:04 -0700 Subject: [PATCH 0197/1345] Add helpful explaination to test_password_manager tests. (#936) (#941) Also uncomment and fix a path test. (cherry picked from commit 1f5425ff69ea0531d869b4f9fa28bd3f66ca3de7) --- Lib/test/test_urllib2.py | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index b537fca3c31adf..d50211c59857e1 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) From a71a3ad54d14483cfaebd8e0fb96c97c798d9e32 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Apr 2017 20:00:36 -0700 Subject: [PATCH 0198/1345] bpo-26947: DOC: clarify wording on hashable in glossary (#948) (#957) (cherry picked from commit 64c887ab3a400cf91bde4f0c5ef69eacc88bc5e1) --- Doc/glossary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 81238414efe49d..dba9186d935a6a 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -467,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. From 90e3518225bafaff01469ed48c472ae7db5686f0 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Sat, 1 Apr 2017 20:20:05 -0700 Subject: [PATCH 0199/1345] bpo-29941: Assert fixes (#886) (#955) Make a non-Py_DEBUG, asserts-enabled build of CPython possible. This means making sure helper functions are defined when NDEBUG is not defined, not just when Py_DEBUG is defined. Also fix a division-by-zero in obmalloc.c that went unnoticed because in Py_DEBUG mode, elsize is never zero. (cherry picked from commit a00c3fd12d421e41b769debd7df717d17b0deed5 and 06bb4873d6a9ac303701d08a851d6cd9a51e02a3) --- Include/unicodeobject.h | 4 ++++ Objects/dictobject.c | 2 +- Objects/obmalloc.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 2d0d77e804d9d8..5b877185f258d3 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -2309,6 +2309,10 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); +#elif !defined(NDEBUG) +/* For asserts that call _PyUnicode_CheckConsistency(), which would + * otherwise be a problem when building with asserts but without Py_DEBUG. */ +#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op) #endif #ifndef Py_LIMITED_API diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 8bcc931148beaa..566d1a5ac8d893 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3b09ad9e..32e7ecbe1e0436 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND From c4e557ed9933499e1c97644ea12ad77ba064e829 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 2 Apr 2017 01:36:58 -0700 Subject: [PATCH 0200/1345] Minor spell fix and formatting fixes in urllib tests. (#959) (#961) (cherry picked from commit efbd4ea65dbb9f87b1afeec6a760802756badee5) --- Lib/test/test_urllib.py | 6 +++++- Lib/test/test_urllibnet.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486e5ab479..fa3757cc94be52 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ def test_proxy_bypass_environment_host_match(self): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ def test_getproxies_environment_prefer_lowercase(self): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ def test_ftp_cache_pruning(self): finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ def test_cafile_and_context(self): "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c2b567e0..865a7f4f5fb172 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -29,7 +29,7 @@ def testURLread(self): class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no From d184c20e359907e326696724a3395ce1d542f42e Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 4 Apr 2017 01:26:32 +0900 Subject: [PATCH 0201/1345] Keep the c-api exception doc up-to-date (#966) cherry-pick'ed from ec1f5df..e3d6db3 --- Doc/c-api/exceptions.rst | 205 ++++++++++++++++++++++++++++----------- Misc/ACKS | 1 + 2 files changed, 150 insertions(+), 56 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85cfd11754..3516a1690384f7 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -750,6 +750,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +814,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +822,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +858,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +884,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +926,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +951,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +962,58 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +Standard Warnings +================= + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DepricationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDepricationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Misc/ACKS b/Misc/ACKS index 2b23702f22910c..b5da02a63ad0aa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -775,6 +775,7 @@ Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes From cb1e002c07622e027e80a3843d27a623d1617430 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 3 Apr 2017 22:27:15 -0700 Subject: [PATCH 0202/1345] bpo-29725: DOC: add text for arraysize in sqlite3.Cursor (#947) (#985) (cherry picked from commit 02e12138000da834f23719521a011fa93763384d) --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d1f7a6f120620b..9fef7d7f03f1f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To From ce92be1825c5ca1d3e6bff24598f3aa54c000b22 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 4 Apr 2017 11:49:14 +0200 Subject: [PATCH 0203/1345] remove merge=union attribute for Misc/NEWS (GH-460) (GH-988) (cherry picked from commit 060d2d776a29341c079cce37220324f9775140ba) --- .gitattributes | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 55a3f49922bf19..82694d81f276b2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,3 @@ -Misc/NEWS merge=union - *.pck binary Lib/test/cjkencodings/* binary Lib/test/decimaltestdata/*.decTest binary From ba980e8ef01dfaf484defd6e7a19ea5375de1338 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Apr 2017 13:18:18 +0300 Subject: [PATCH 0204/1345] Miscellaneous minor fixes of Misc/NEWS formatting. (#1002) (#1003) (cherry picked from commit a0157b5f11e621f2196af4e918b9f07688a6cd1c) --- Misc/NEWS | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e61a1652381fa0..fa20ea16cd0144 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -290,7 +290,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #29055: Neaten-up empty population error on random.choice() by suppressing the upstream exception. @@ -1058,7 +1058,7 @@ Core and Builtins alpha releases, where backslashes are allowed anywhere in an f-string. Also, require that expressions inside f-strings be enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. + ``f'\x7b"hi"\x7d'``. - Issue #28046: Remove platform-specific directories from sys.path. @@ -1277,7 +1277,7 @@ Library - Issue #24277: The new email API is no longer provisional, and the docs have been reorganized and rewritten to emphasize the new API. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -1296,9 +1296,9 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27331: The email.mime classes now all accept an optional policy keyword. +- Issue #27331: The email.mime classes now all accept an optional policy keyword. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #16113: Add SHA-3 and SHAKE support to hashlib module. @@ -2406,7 +2406,7 @@ Core and Builtins in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used @@ -2964,7 +2964,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -3100,7 +3100,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -3735,7 +3735,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #28871: Fixed a crash when deallocate deep ElementTree. @@ -3858,7 +3858,7 @@ Library - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line +- Issue #19003:m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per the RFC, instead of all unicode line endings. - Issue #28019: itertools.count() no longer rounds non-integer step in range @@ -3881,7 +3881,7 @@ Library - Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -3896,7 +3896,7 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 certs. @@ -4347,7 +4347,7 @@ Core and Builtins cookie names. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #27138: Fix the doc comment for FileFinder.find_spec(). @@ -4727,7 +4727,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -5137,7 +5137,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -6680,7 +6680,7 @@ Library - Issue #23521: Corrected pure python implementation of timedelta division. - * Eliminated OverflowError from timedelta * float for some floats; + * Eliminated OverflowError from ``timedelta * float`` for some floats; * Corrected rounding in timedlta true division. - Issue #21619: Popen objects no longer leave a zombie after exit in the with @@ -7479,7 +7479,7 @@ Library character instead of truncating it. Based on patch by Victor Stinner. - Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. + subdirectories using the ``**`` pattern. - Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with empty string or tuple argument. From ed278448daa4afb78d1af57b0c179097477f9832 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 6 Apr 2017 08:38:38 -0700 Subject: [PATCH 0205/1345] correct parse_qs and parse_qsl test case descriptions. (#968) (#997) (cherry picked from commit 257b980b316a5206ecf6c23b958e2b7c4df4f3de) --- Lib/test/test_urlparse.py | 12 ++++++------ Lib/urllib/parse.py | 30 +++++++++++++++++------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c033e30d20..3c89ed928ff51f 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ def test_RFC2368(self): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ def test_RFC2396(self): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ def test_RFC3986(self): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08d7e77..32de25b374da93 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -612,6 +612,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +634,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +647,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] From 19d61b265fabed01ebfb8eae9855f2ee59ff591b Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Fri, 7 Apr 2017 00:56:59 -0700 Subject: [PATCH 0206/1345] Remove Invalid comment in test_urllib2.py. (#1022) (cherry picked from commit fd0cd07a5a3c964c084f4efc5bbcb89dd2193ee6) --- Lib/test/test_urllib2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d50211c59857e1..876fcd4199fb92 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -848,7 +848,6 @@ def test_file(self): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed except urllib.error.URLError: self.assertFalse(ftp) else: From 616d3eb7ef96f1a27bbff68fba6e5d229257d936 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 7 Apr 2017 16:50:13 -0400 Subject: [PATCH 0207/1345] Fix a minor typo. (#1032) (#1037) (cherry picked from commit dd9a0a14c89d57e43898d4b866b8c161e4ff8506) --- Doc/library/asyncio-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 482ffbbed35b1e..3f55506c669a29 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -384,7 +384,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). From d0d575a6db8cb3b2a720be9f404af3d754da9a5d Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 7 Apr 2017 23:23:08 +0100 Subject: [PATCH 0208/1345] bpo-29939: suppress compiler warnings in _ctypes_test (#1038) bpo-29939: Changed test code to suppress a compiler warning, while taking care to avoid the code being optimized out by the compiler. (cherry picked from commit 164d30eb1e66575dafee6af4fca4cbf52c7fbe6a) --- Modules/_ctypes/_ctypes_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 59d56d0d218732..6119ecdaf90dc6 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -52,9 +52,9 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) EXPORT(void) _testfunc_large_struct_update_value(Test in) { - in.first = 0x0badf00d; - in.second = 0x0badf00d; - in.third = 0x0badf00d; + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; } EXPORT(void)testfunc_array(int values[4]) From c26b19d5c7aba51b50a4d7fb5f8291036cb9da24 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Apr 2017 11:18:14 +0300 Subject: [PATCH 0209/1345] Expand the PySlice_GetIndicesEx macro. (#1023) (#1044) (cherry picked from commit b879fe82e7e5c3f7673c9a7fa4aad42bd05445d8) --- Modules/_ctypes/_ctypes.c | 10 ++++------ Modules/_elementtree.c | 12 ++++++------ Modules/_testbuffer.c | 7 ++++--- Modules/arraymodule.c | 11 ++++++----- Modules/mmapmodule.c | 9 ++++----- Objects/bytearrayobject.c | 12 ++++++------ Objects/bytesobject.c | 6 +++--- Objects/listobject.c | 10 ++++++---- Objects/memoryobject.c | 4 ++-- Objects/tupleobject.c | 6 +++--- Objects/unicodeobject.c | 5 +++-- 11 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aedec6eeab5..12234e254e1a7a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4273,11 +4273,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4413,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e3350d194da18a..bef702ebe69c93 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1712,11 +1712,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1768,11 +1768,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce6851a5eb6..6b8ab34d931cd3 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d4221fe4c6a163..64e0f172fd7d82 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2297,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2368,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615ff82992c..426b7cacebb573 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index d456678d462f2f..a9c8ca6f1f35a8 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c95a46d1846649..4c55294d9d8c8a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1683,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/listobject.c b/Objects/listobject.c index cde281a0f6f9dc..547bdf0b95f1ee 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2073d108..e1ac7281783f38 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 52f20f4fe8a2b1..0dada74dc7c986 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454b495dee..949205adf9347e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13915,10 +13915,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); From af685f9050416da8050e0ec11a8dff9afd4130e7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Apr 2017 11:25:47 +0300 Subject: [PATCH 0210/1345] bpo-29998: Pickling and copying ImportError now preserves name and path (#1010) (#1042) attributes. (cherry picked from commit b785396ab451b0c9d6ae9ee5a9e56c810209a6cb) --- Lib/test/test_exceptions.py | 20 ++++++++++++++++ Misc/NEWS | 3 +++ Objects/exceptions.c | 48 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 48379222c37659..960fc0f1bf9486 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -1119,6 +1120,25 @@ def test_non_str_argument(self): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index fa20ea16cd0144..83f9e7a6852560 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- bpo-29998: Pickling and copying ImportError now preserves name and path + attributes. + - bpo-29953: Fixed memory leaks in the replace() method of datetime and time objects when pass out of bound fold argument. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145bcb1..d158b9768d45f5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; From 4c116cb3b3ee38cae229f7e0df0cd2045fe73c27 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 9 Apr 2017 19:22:36 +1000 Subject: [PATCH 0211/1345] bpo-29798: Handle git worktree in patchcheck (#1058) (#1060) The original attempted fix missed an `isdir()` call in `get_base_branch()`. (cherry picked from commit 2abfdf5a81383d3b1ed6b7321903a9a168c373c5) --- Tools/scripts/patchcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index f4ec7d8a30ea23..33a9fead879325 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -70,7 +70,7 @@ def get_git_upstream_remote(): @status("Getting base branch for PR", info=lambda x: x if x is not None else "not a PR branch") def get_base_branch(): - if not os.path.isdir(os.path.join(SRCDIR, '.git')): + if not os.path.exists(os.path.join(SRCDIR, '.git')): # Not a git checkout, so there's no base branch return None version = sys.version_info From a524d6307273e3920d410452ace89610c843f051 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 9 Apr 2017 20:57:04 +1000 Subject: [PATCH 0212/1345] bpo-29506: Clarify deep copy note in copy module The reference to administrative data was confusing to readers, so this simplifies the note to explain that deep copying may copy more then you intended, such as data that you expected to be shared between copies. (cherry picked from commit 19e04942562a980ad2519f6ff79c455a7472783b) --- Doc/library/copy.rst | 4 ++-- Misc/ACKS | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d469bc05..2041d9175ea587 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Misc/ACKS b/Misc/ACKS index b5da02a63ad0aa..72070e19f65701 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -770,6 +770,7 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim From 854ca174a92970d11d9b063af46e00d132832944 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 9 Apr 2017 10:15:06 -0700 Subject: [PATCH 0213/1345] Remove invalid comment in urllib.request. (#1055) (cherry picked from commit a2a9ddd923a849124bdd1c484f70f02df6fde0e9) --- Lib/urllib/request.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3d0e4bf5..3e82000466432f 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1658,14 +1658,10 @@ def pathname2url(pathname): of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need From 90eb7a9ff4ce38b5c4e3662bdd42a0ed513893a2 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:15:57 -0700 Subject: [PATCH 0214/1345] [3.6] Correct typo in configparser.rst (GH-1012) (GH-1025) (cherry picked from commit 01fa9ae5460b00bf1ced500c797176ebd3fb060d) --- Doc/library/configparser.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index af57cba2b5661d..04c2a820921759 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also From 577fc04a7157f6e904cffd6a0e1ad83d3460acd6 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:17:06 -0700 Subject: [PATCH 0215/1345] [3.6] bpo-29549: Fixes docstring for str.index (GH-256) (GH-1028) (cherry picked from commit 43ba8861e0ad044efafa46a7cc04e12ac5df640e) --- Objects/bytes_methods.c | 13 ++++++++++--- Objects/unicodeobject.c | 12 ++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6346fc53..625e242d563d89 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 949205adf9347e..7871043681533d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11701,7 +11701,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12762,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) From 28ddd3c905836907bf0c4df909067d3a054358c1 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:17:28 -0700 Subject: [PATCH 0216/1345] [3.6] Improvements to typing documentation (GH-967) (GH-1053) Documents a few omitted classes and adds NamedTuple methods. (cherry picked from commit 45d22c256bce3afcf57f49032a6b20fdec4f26ad) --- Doc/library/typing.rst | 50 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d130e1759d8f59..2cfc37f695ef2d 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -508,6 +508,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -658,7 +666,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -742,9 +762,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -756,6 +779,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -782,10 +808,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -794,7 +830,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -972,9 +1008,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance From 03066a05f5cdf96bc3bf15718f586877860f2d47 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 9 Apr 2017 20:37:44 -0700 Subject: [PATCH 0217/1345] Remove OSError related comment in urllib.request. (#1072) (cherry picked from commit 6dfcc81f6b1c82a71a1c876e14424fb8b3573447) --- Lib/urllib/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 3e82000466432f..96921440edea11 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ def open_local_file(self, req): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') From f0c416799b33742af36d7c6608b9a422615fb3d2 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:02:21 -0700 Subject: [PATCH 0218/1345] [3.6] bpo-30047: Fix a typo in Doc/library/select.rst (GH-1086) (GH-1098) (cherry picked from commit 3e0f1fc4e0ffcfcc706015fa3d67c262948ef171) --- Doc/library/select.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118ebe05788..bd5442c6a27aa0 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | From 14944c62300f741488c2f28cb91ad1e3fef7343b Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Thu, 13 Apr 2017 11:14:17 +0800 Subject: [PATCH 0219/1345] bpo-26985: Add missing info of code object in inspect documentation (GH-1090) (GH-1099) --- Doc/library/inspect.rst | 379 +++++++++++++++++++++------------------- Lib/inspect.py | 30 ++-- Misc/NEWS | 5 + 3 files changed, 223 insertions(+), 191 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 4ff2187b45f8eb..6be28a2b31cb66 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -1268,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1281,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5d41b47f..a2dcb888a0c609 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -253,18 +253,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): diff --git a/Misc/NEWS b/Misc/NEWS index 83f9e7a6852560..dd1fa12d7c0be8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -86,6 +86,11 @@ Build - bpo-29643: Fix ``--enable-optimization`` didn't work. +Documentation +------------- + +- bpo-26985: Add missing info of code object in inspect documentation. + What's New in Python 3.6.1? =========================== From bc9025012fc5ae007283e9ef345dba0ba08cf2b9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:45:54 -0700 Subject: [PATCH 0220/1345] [3.6] Remove superfluous comment in urllib.error. (GH-1076) (GH-1101) (cherry picked from commit 6fab78e9027f9ebd6414995580781b480433e595) --- Lib/urllib/error.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16188b8..851515bc307d62 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. From bd1173f202f5a3990063d980368e7ad1edc9b5b5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:52:39 -0700 Subject: [PATCH 0221/1345] [3.6] Clarify exception handler scope in contextlib (GH-1103) Moved explicit raise from inside try to try...else. (cherry picked from commit 680e20beee8bbce9f857b8e7795009191f98b0ba) --- Lib/contextlib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 8421968525947e..e91cf460e53bfd 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -98,7 +98,6 @@ def __exit__(self, type, value, traceback): value = type() try: self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -124,6 +123,8 @@ def __exit__(self, type, value, traceback): # if sys.exc_info()[1] is not value: raise + else: + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): From 9b409ff41ceb2d7ea7e8d25a7bbf5eb7d46625f3 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 02:50:21 -0700 Subject: [PATCH 0222/1345] [3.6] bpo-29692: contextlib.contextmanager may incorrectly unchain RuntimeError (GH-949) (#1105) contextlib._GeneratorContextManager.__exit__ includes a special case to deal with PEP 479 RuntimeErrors created when `StopIteration` is thrown into the context manager body. Previously this check was too permissive, and undid one level of chaining on *all* RuntimeError instances, not just those that wrapped a StopIteration instance. (cherry picked from commit 00c75e9a45ff0366c185e9e8a2e23af5a35481b0) --- Lib/contextlib.py | 12 ++++++------ Lib/test/test_contextlib.py | 23 +++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index e91cf460e53bfd..5e47054954ba5a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -88,7 +88,7 @@ def __exit__(self, type, value, traceback): try: next(self.gen) except StopIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: @@ -110,7 +110,7 @@ def __exit__(self, type, value, traceback): # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). - if exc.__cause__ is value: + if type is StopIteration and exc.__cause__ is value: return False raise except: @@ -121,10 +121,10 @@ def __exit__(self, type, value, traceback): # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # - if sys.exc_info()[1] is not value: - raise - else: - raise RuntimeError("generator didn't stop after throw()") + if sys.exc_info()[1] is value: + return False + raise + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804af57047..b1a467d952da0f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -152,6 +152,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Misc/NEWS b/Misc/NEWS index dd1fa12d7c0be8..6e2db58a92f451 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -31,6 +31,9 @@ Core and Builtins Library ------- +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. + Patch by Siddharth Velankar. - bpo-29998: Pickling and copying ImportError now preserves name and path attributes. From 97a40b7a5b2979fb17e1751c139fd4ba1ebd5276 Mon Sep 17 00:00:00 2001 From: Nevada Sanchez Date: Thu, 13 Apr 2017 06:21:05 -0400 Subject: [PATCH 0223/1345] bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-752) --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..fba0fa2c00dccd 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. From 84c2d75489a84174d8993aea292828662e35a50f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 04:03:16 -0700 Subject: [PATCH 0224/1345] Revert "bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-752)" (GH-1109) This reverts commit 97a40b7a5b2979fb17e1751c139fd4ba1ebd5276. The commit is supposed to go to the master branch first. --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ---------------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index fba0fa2c00dccd..d14db60f7da89e 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') +Binnumber = r'0[bB][01]*' +Hexnumber = r'0[xX][\da-fA-F]*[lL]?' +Octnumber = r'0[oO]?[0-7]*[lL]?' +Decnumber = r'[1-9]\d*[lL]?' Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent +Exponent = r'[eE][-+]?\d+' +Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) +Expfloat = r'\d+' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index 0b9bee0ab42ea6..cf31a5411a855a 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,28 +72,6 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testUnderscoresInNumbers(self): - # Integers - x = 1_0 - x = 123_456_7_89 - x = 0xabc_123_4_5 - x = 0X_abc_123 - x = 0B11_01 - x = 0b_11_01 - x = 0o45_67 - x = 0O_45_67 - - # Floats - x = 3_1.4 - x = 03_1.4 - x = 3_1. - x = .3_1 - x = 3.1_4 - x = 0_3.1_4 - x = 3e1_4 - x = 3_1e+4_1 - x = 3_1E-4_1 - def testFloats(self): x = 3.14 x = 314. From 119d94ad37a99ecb0b8329467d04cd9d909e310e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 13 Apr 2017 16:17:01 +0300 Subject: [PATCH 0225/1345] bpo-29791: Clarify that flush is keyword-only argument (GH-1093) Reported by Lucio Ricardo Montero Valenzuela. (cherry picked from commit 61b9ac93712df8092a25223cd56fa6528359792b) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 90fb07af227d98..9126535eecb85c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1125,7 +1125,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and From 936633282220768a6fec9bd6bb53ee9e3ec354df Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Apr 2017 19:41:19 +0300 Subject: [PATCH 0226/1345] bpo-30021: Add examples for re.escape(). (#1048) (#1115) And fix the parameter name. (cherry picked from commit 8fc7bc2b7631ee819ee614e47b6f44bacebe1574) --- Doc/library/re.rst | 17 ++++++++++++++--- Doc/tools/susp-ignored.csv | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9cced512706554..944c6fb10d6c25 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -784,11 +784,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 166374cb90cec1..57b0cf12f53712 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -302,6 +302,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional," Date: Thu, 13 Apr 2017 14:11:48 -0700 Subject: [PATCH 0227/1345] convert from long long to PyLong loselessly (#1106) (#1121) --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f63cc5ab..b25f4434ffa35c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3812,7 +3812,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); From c5c24cfaf9fe9a38f0901756cd797bd12524a8b7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 16:37:51 -0700 Subject: [PATCH 0228/1345] [3.6] Fix a typo in Doc/library/functions.rst (GH-1117) (GH-1123) Replace `For object's ... ` with `For objects ...` (cherry picked from commit 873ef20d0007b4b120933473e6252d2309a70102) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 9126535eecb85c..138f7d027cb86f 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -610,7 +610,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. From 2cdf087d1fd48f7d0f95b5a0b31b9a624fa84751 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 16:54:49 -0700 Subject: [PATCH 0229/1345] [3.6] bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119) (GH-1122) * Allow underscores in numeric literals in lib2to3. * Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize. * Add test case for underscores in literals in Python 3. (cherry picked from commit a6e395dffadf8c5124903c01ad69fefa36b1a935) --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..fba0fa2c00dccd 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. From cbc46afa59dcc43c2c8c90ae7a0a0dc404325a89 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 19:26:16 -0700 Subject: [PATCH 0230/1345] [3.6] bpo-29694: race condition in pathlib mkdir with flags parents=True (GH-1089). (GH-1126) (cherry picked from commit 22a594a0047d7706537ff2ac676cdc0f1dcb329c) --- Lib/pathlib.py | 4 ++-- Lib/test/test_pathlib.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8c1cb96bad3075..70f5cba76ffa8a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1230,8 +1230,8 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system # could give priority to other errors like EACCES or EROFS diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 3ff9726d4359ec..846f721e8d9abb 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1816,6 +1817,35 @@ def test_mkdir_no_parents_file(self): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Misc/NEWS b/Misc/NEWS index 6e2db58a92f451..e623457cdd18ce 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -31,6 +31,10 @@ Core and Builtins Library ------- + +- bpo-29694: Fixed race condition in pathlib mkdir with flags + parents=True. Patch by Armin Rigo. + - bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in contextlib.contextmanager. Patch by Siddharth Velankar. From 413a8913aee255d0635e4ddbb9c343d9e5e76fea Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 14 Apr 2017 13:37:46 +0200 Subject: [PATCH 0231/1345] Relax test timing (bpo-29861) to avoid sporadic failures (#1120) (#1132) (cherry picked from commit 685cdb9acc3fca04a9897d88b89771ddfd50e772) --- Lib/test/_test_multiprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 771bbf24265b91..57ada127465d75 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2080,6 +2080,7 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs + time.sleep(DELTA) # let threaded cleanup code run self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. From 7ac96ec60e944bbfa290c869225dc48b542b6884 Mon Sep 17 00:00:00 2001 From: Jack McCracken Date: Fri, 14 Apr 2017 10:44:34 -0400 Subject: [PATCH 0232/1345] Fix misplaced positional argument in OS X support library (#1137) --- Lib/_osx_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index eadf06f20e2507..e37852e2536c33 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -210,7 +210,7 @@ def _remove_universal_flags(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags) From 2e30eb6a98fb9b47e840528b41f1fc4446c69f36 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 14 Apr 2017 09:59:34 -0700 Subject: [PATCH 0233/1345] improve alignment autoconf test (#1129) (#1139) Replace the deprecated AC_TRY_RUN with AC_RUN_IFELSE. Also, standardize the variable name and allow for caching of the result. --- aclocal.m4 | 4 ++-- configure | 35 +++++++++++++++++++++++++---------- configure.ac | 19 +++++++++---------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/aclocal.m4 b/aclocal.m4 index 9a9cc557281571..2a745e57466cae 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -13,7 +13,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -dnl serial 11 (pkg-config-0.29) +dnl serial 11 (pkg-config-0.29.1) dnl dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ diff --git a/configure b/configure index 8bdb82ba4a7bce..6a1b9e4edc2c23 100755 --- a/configure +++ b/configure @@ -784,6 +784,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -894,6 +895,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1146,6 +1148,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1283,7 +1294,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1436,6 +1447,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -9697,8 +9709,11 @@ esac # check for systems that require aligned memory access { $as_echo "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 $as_echo_n "checking aligned memory access is required... " >&6; } -if test "$cross_compiling" = yes; then : - aligned_required=yes +if ${ac_cv_aligned_required+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + ac_cv_aligned_required=yes else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9715,26 +9730,26 @@ int main() return 1; return 0; } - _ACEOF if ac_fn_c_try_run "$LINENO"; then : - aligned_required=no + ac_cv_aligned_required=no else - aligned_required=yes + ac_cv_aligned_required=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi -if test "$aligned_required" = yes ; then +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_aligned_required" >&5 +$as_echo "$ac_cv_aligned_required" >&6; } +if test "$ac_cv_aligned_required" = yes ; then $as_echo "#define HAVE_ALIGNED_REQUIRED 1" >>confdefs.h fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $aligned_required" >&5 -$as_echo "$aligned_required" >&6; } - # str, bytes and memoryview hash algorithm diff --git a/configure.ac b/configure.ac index 6b331d5742745f..2c8e246f22f53e 100644 --- a/configure.ac +++ b/configure.ac @@ -2696,7 +2696,8 @@ esac # check for systems that require aligned memory access AC_MSG_CHECKING(aligned memory access is required) -AC_TRY_RUN([ +AC_CACHE_VAL(ac_cv_aligned_required, +[AC_RUN_IFELSE([AC_LANG_SOURCE([[ int main() { char s[16]; @@ -2708,18 +2709,16 @@ int main() if (*p1 == *p2) return 1; return 0; -} - ], - [aligned_required=no], - [aligned_required=yes], - [aligned_required=yes]) - -if test "$aligned_required" = yes ; then +}]])], +[ac_cv_aligned_required=no], +[ac_cv_aligned_required=yes], +[ac_cv_aligned_required=yes]) +]) +AC_MSG_RESULT($ac_cv_aligned_required) +if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], [Define if aligned memory access is required]) fi -AC_MSG_RESULT($aligned_required) - # str, bytes and memoryview hash algorithm AH_TEMPLATE(Py_HASH_ALGORITHM, From 8e7201342dc6eef201bfa4f533ad89a8144fd693 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 14 Apr 2017 18:34:11 -0700 Subject: [PATCH 0234/1345] [3.6] bpo-29738: Fix memory leak in _get_crl_dp (GH-526) (GH-1142) * Remove conditional on free of `dps`, since `dps` is now allocated for all versions of OpenSSL * Remove call to `x509_check_ca` since it was only used to cache the `crldp` field of the certificate CRL_DIST_POINTS_free is available in all supported versions of OpenSSL (recent 0.9.8+) and LibreSSL. (cherry picked from commit 2849cc34a8db93d448a62d69c462402347b50dcb) --- Modules/_ssl.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c0a7b8e1052c33..2a2c18fe2f7309 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1210,10 +1210,6 @@ _get_crl_dp(X509 *certificate) { int i, j; PyObject *lst, *res = NULL; -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - /* Calls x509v3_cache_extensions and sets up crldp */ - X509_check_ca(certificate); -#endif dps = X509_get_ext_d2i(certificate, NID_crl_distribution_points, NULL, NULL); if (dps == NULL) @@ -1258,9 +1254,7 @@ _get_crl_dp(X509 *certificate) { done: Py_XDECREF(lst); -#if OPENSSL_VERSION_NUMBER < 0x10001000L - sk_DIST_POINT_free(dps); -#endif + CRL_DIST_POINTS_free(dps); return res; } From 05bfbcd233b2f5ba0d0634a380092d6ead6b35e1 Mon Sep 17 00:00:00 2001 From: Michael Seifert Date: Sat, 15 Apr 2017 04:03:41 +0200 Subject: [PATCH 0235/1345] [3.6] bpo-30059: Include Py_Ellipsis in C API documentation (GH-1018) (GH-1149) --- Doc/c-api/slice.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index a825164918f446..8b695e065aeffd 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -56,3 +56,14 @@ Slice Objects .. versionchanged:: 3.2 The parameter type for the *slice* parameter was ``PySliceObject*`` before. + + +Ellipsis Object +--------------- + + +.. c:var:: PyObject *Py_Ellipsis + + The Python ``Ellipsis`` object. This object has no methods. It needs to be + treated just like any other object with respect to reference counts. Like + :c:data:`Py_None` it is a singleton object. From d5fa5f3ce7d9003bbd3975d1bf634043305ae18f Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Sat, 15 Apr 2017 13:25:15 +0800 Subject: [PATCH 0236/1345] bpo-30068: add missing iter(self) in _io._IOBase.readlines when hint is present (#1130) (#1150) --- Lib/test/test_io.py | 1 + Misc/NEWS | 8 +++++--- Modules/_io/iobase.c | 25 +++++++++++++++++-------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 8f895fe0bfb885..e04baefef081d4 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3498,6 +3498,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.readinto1, bytearray(1024)) self.assertRaises(ValueError, f.readline) self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) self.assertRaises(ValueError, f.seek, 0) self.assertRaises(ValueError, f.tell) self.assertRaises(ValueError, f.truncate) diff --git a/Misc/NEWS b/Misc/NEWS index e623457cdd18ce..7628ed5cab7987 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,12 +32,14 @@ Core and Builtins Library ------- +- bpo-30068: _io._IOBase.readlines will check if it's closed first when + hint is present. + - bpo-29694: Fixed race condition in pathlib mkdir with flags parents=True. Patch by Armin Rigo. -- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in - contextlib.contextmanager. - Patch by Siddharth Velankar. +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. Patch by Siddharth Velankar. - bpo-29998: Pickling and copying ImportError now preserves name and path attributes. diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 472ef3b97cd38e..f7986d7e52f957 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -650,7 +650,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) /*[clinic end generated code: output=2f50421677fa3dea input=1961c4a95e96e661]*/ { Py_ssize_t length = 0; - PyObject *result; + PyObject *result, *it = NULL; result = PyList_New(0); if (result == NULL) @@ -664,19 +664,22 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) PyObject *ret = _PyObject_CallMethodId(result, &PyId_extend, "O", self); if (ret == NULL) { - Py_DECREF(result); - return NULL; + goto error; } Py_DECREF(ret); return result; } + it = PyObject_GetIter(self); + if (it == NULL) { + goto error; + } + while (1) { - PyObject *line = PyIter_Next(self); + PyObject *line = PyIter_Next(it); if (line == NULL) { if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + goto error; } else break; /* StopIteration raised */ @@ -684,8 +687,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) if (PyList_Append(result, line) < 0) { Py_DECREF(line); - Py_DECREF(result); - return NULL; + goto error; } length += PyObject_Size(line); Py_DECREF(line); @@ -693,7 +695,14 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) if (length > hint) break; } + + Py_DECREF(it); return result; + + error: + Py_XDECREF(it); + Py_DECREF(result); + return NULL; } /*[clinic input] From 8e5b52a8da07e781bda50ba0a7065b1058495a37 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 16 Apr 2017 12:04:45 +0300 Subject: [PATCH 0237/1345] bpo-30017: Allowed calling the close() method of the zip entry writer object (#1041) (#1092) multiple times. Writing to closed zip entry writer object now always produce a ValueError. (cherry picked from commit 4c0d9ea995da595e90e08813b89510de59907802) --- Lib/test/test_zipfile.py | 42 ++++++++++++++++++++++++++++++++++++++++ Lib/zipfile.py | 4 ++++ Misc/NEWS | 3 +++ 3 files changed, 49 insertions(+) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index d39f05fdb7788e..d09ad96fc8baaa 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -734,6 +734,48 @@ class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, compression = zipfile.ZIP_LZMA +class AbstractWriterTests: + + def tearDown(self): + unlink(TESTFN2) + + def test_close_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + w.close() + self.assertTrue(w.closed) + self.assertEqual(zipf.read('test'), data) + + def test_write_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + self.assertRaises(ValueError, w.write, b'') + self.assertEqual(zipf.read('test'), data) + +class StoredWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib +class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2 +class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma +class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + + class PyZipFileTests(unittest.TestCase): def assertCompiledIn(self, name, namelist): if name + 'o' not in namelist: diff --git a/Lib/zipfile.py b/Lib/zipfile.py index d7f5beba996e6e..06eedec340bc17 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -981,6 +981,8 @@ def writable(self): return True def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) @@ -991,6 +993,8 @@ def write(self, data): return nbytes def close(self): + if self.closed: + return super().close() # Flush any data from the compressor, and update header info if self._compressor: diff --git a/Misc/NEWS b/Misc/NEWS index 7628ed5cab7987..673211d72dda51 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- bpo-30017: Allowed calling the close() method of the zip entry writer object + multiple times. Writing to a closed writer now always produces a ValueError. + - bpo-30068: _io._IOBase.readlines will check if it's closed first when hint is present. From 680fea4067537a9b9c79aadd44a3a19e83cd2dbf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Apr 2017 21:22:49 +0300 Subject: [PATCH 0238/1345] bpo-30061: Check if PyObject_Size()/PySequence_Size()/PyMapping_Size() (#1096) (#1180) raised an error. (cherry picked from commit bf623ae8843dc30b28c574bec8d29fc14be59d86) --- Lib/test/test_io.py | 16 ++++++++ Misc/NEWS | 5 +++ Modules/_io/iobase.c | 13 +++++-- Modules/_winapi.c | 17 ++++++--- Modules/cjkcodecs/multibytecodec.c | 3 ++ Modules/posixmodule.c | 60 +++++++++++++++++++++--------- Objects/setobject.c | 12 ++++-- 7 files changed, 95 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e04baefef081d4..5a7617cd9f9cbf 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -543,6 +543,22 @@ def test_readline(self): with self.open(support.TESTFN, "r") as f: self.assertRaises(TypeError, f.readline, 5.3) + def test_readline_nonsizeable(self): + # Issue #30061 + # Crash when readline() returns an object without __len__ + class R(self.IOBase): + def readline(self): + return None + self.assertRaises((TypeError, StopIteration), next, R()) + + def test_next_nonsizeable(self): + # Issue #30061 + # Crash when __next__() returns an object without __len__ + class R(self.IOBase): + def __next__(self): + return None + self.assertRaises(TypeError, R().readlines, 1) + def test_raw_bytes_io(self): f = self.BytesIO() self.write_ops(f) diff --git a/Misc/NEWS b/Misc/NEWS index 673211d72dda51..44a4b873b1454b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,11 @@ Core and Builtins Library ------- +- bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when + readline() or __next__() respectively return non-sizeable object. + Fixed possible other errors caused by not checking results of PyObject_Size(), + PySequence_Size(), or PyMapping_Size(). + - bpo-30017: Allowed calling the close() method of the zip entry writer object multiple times. Writing to a closed writer now always produces a ValueError. diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index f7986d7e52f957..c8642040ae426a 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -625,7 +625,8 @@ iobase_iternext(PyObject *self) if (line == NULL) return NULL; - if (PyObject_Size(line) == 0) { + if (PyObject_Size(line) <= 0) { + /* Error or empty */ Py_DECREF(line); return NULL; } @@ -676,6 +677,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) } while (1) { + Py_ssize_t line_length; PyObject *line = PyIter_Next(it); if (line == NULL) { if (PyErr_Occurred()) { @@ -689,11 +691,14 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) Py_DECREF(line); goto error; } - length += PyObject_Size(line); + line_length = PyObject_Size(line); Py_DECREF(line); - - if (length > hint) + if (line_length < 0) { + goto error; + } + if (line_length > hint - length) break; + length += line_length; } Py_DECREF(it); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 91d4f0172c3c50..248f4582c60fb4 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -722,17 +722,22 @@ getenvironment(PyObject* environment) return NULL; } - envsize = PyMapping_Length(environment); - keys = PyMapping_Keys(environment); values = PyMapping_Values(environment); if (!keys || !values) goto error; + envsize = PySequence_Fast_GET_SIZE(keys); + if (PySequence_Fast_GET_SIZE(values) != envsize) { + PyErr_SetString(PyExc_RuntimeError, + "environment changed size during iteration"); + goto error; + } + totalsize = 1; /* trailing null character */ for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (! PyUnicode_Check(key) || ! PyUnicode_Check(value)) { PyErr_SetString(PyExc_TypeError, @@ -760,8 +765,8 @@ getenvironment(PyObject* environment) end = buffer + totalsize; for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (!PyUnicode_AsUCS4(key, p, end - p, 0)) goto error; p += PyUnicode_GET_LENGTH(key); diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index d1da189ddd3bef..d6efc77d20c883 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1670,6 +1670,9 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se if (r == -1) return NULL; } + /* PySequence_Length() can fail */ + if (PyErr_Occurred()) + return NULL; Py_RETURN_NONE; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2ea5e2def9acff..f4bbc8931b39fa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6650,7 +6650,7 @@ static PyObject * os_setgroups(PyObject *module, PyObject *groups) /*[clinic end generated code: output=3fcb32aad58c5ecd input=fa742ca3daf85a7e]*/ { - int i, len; + Py_ssize_t i, len; gid_t grouplist[MAX_GROUPS]; if (!PySequence_Check(groups)) { @@ -6658,6 +6658,9 @@ os_setgroups(PyObject *module, PyObject *groups) return NULL; } len = PySequence_Size(groups); + if (len < 0) { + return NULL; + } if (len > MAX_GROUPS) { PyErr_SetString(PyExc_ValueError, "too many groups"); return NULL; @@ -7886,9 +7889,9 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) static Py_ssize_t -iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { - int i, j; + Py_ssize_t i, j; Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); @@ -7965,8 +7968,7 @@ static Py_ssize_t os_readv_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=792da062d3fcebdb input=e679eb5dbfa0357d]*/ { - int cnt; - Py_ssize_t n; + Py_ssize_t cnt, n; int async_err = 0; struct iovec *iov; Py_buffer *buf; @@ -7978,6 +7980,8 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; @@ -8116,15 +8120,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() headers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.hdr_cnt = PySequence_Size(headers); - if (sf.hdr_cnt > 0 && - (i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(headers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); return NULL; + } + if (i > 0) { + sf.hdr_cnt = (int)i; + i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } if (trailers != NULL) { @@ -8133,15 +8146,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() trailers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.trl_cnt = PySequence_Size(trailers); - if (sf.trl_cnt > 0 && - (i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(trailers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() trailer is too large"); return NULL; + } + if (i > 0) { + sf.trl_cnt = (int)i; + i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } @@ -8411,7 +8433,7 @@ static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ { - int cnt; + Py_ssize_t cnt; Py_ssize_t result; int async_err = 0; struct iovec *iov; @@ -8423,6 +8445,8 @@ os_writev_impl(PyObject *module, int fd, PyObject *buffers) return -1; } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) { return -1; diff --git a/Objects/setobject.c b/Objects/setobject.c index 9fe28138c0012b..c1bc1e1234799f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1550,20 +1550,26 @@ set_difference(PySetObject *so, PyObject *other) PyObject *key; Py_hash_t hash; setentry *entry; - Py_ssize_t pos = 0; + Py_ssize_t pos = 0, other_size; int rv; if (PySet_GET_SIZE(so) == 0) { return set_copy(so); } - if (!PyAnySet_Check(other) && !PyDict_CheckExact(other)) { + if (PyAnySet_Check(other)) { + other_size = PySet_GET_SIZE(other); + } + else if (PyDict_CheckExact(other)) { + other_size = PyDict_Size(other); + } + else { return set_copy_and_difference(so, other); } /* If len(so) much more than len(other), it's more efficient to simply copy * so and then iterate other looking for common elements. */ - if ((PySet_GET_SIZE(so) >> 2) > PyObject_Size(other)) { + if ((PySet_GET_SIZE(so) >> 2) > other_size) { return set_copy_and_difference(so, other); } From 39dedb6e1a18069ce8f5c8e6076e9d9ef9d6d515 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Apr 2017 23:22:19 +0300 Subject: [PATCH 0239/1345] [3.6] bpo-30070: Fixed leaks and crashes in errors handling in the parser module. (GH-1131). (#1184) (cherry picked from commit a79f4c219531c05fc8f670c1e4bbf12c081935d3) --- Lib/test/test_parser.py | 81 +++++++++++++++++++++++++ Misc/NEWS | 2 + Modules/parsermodule.c | 131 ++++++++++++++++++++++++---------------- 3 files changed, 162 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index d6e6f7157782ec..70cabb28598218 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,4 +1,6 @@ +import copy import parser +import pickle import unittest import operator import struct @@ -424,6 +426,52 @@ def test_junk(self): # not even remotely valid: self.check_bad_tree((1, 2, 3), "") + def test_illegal_terminal(self): + tree = \ + (257, + (269, + (270, + (271, + (277, + (1,))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too small items in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, b'pass'))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-string second item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', '0', 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-integer third item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', 0, 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too many items in terminal node") + def test_illegal_yield_1(self): # Illegal yield statement: def f(): return 1; yield 1 tree = \ @@ -628,6 +676,24 @@ def test_missing_import_source(self): (4, ''), (0, '')) self.check_bad_tree(tree, "from import fred") + def test_illegal_encoding(self): + # Illegal encoding declaration + tree = \ + (339, + (257, (0, ''))) + self.check_bad_tree(tree, "missed encoding") + tree = \ + (339, + (257, (0, '')), + b'iso-8859-1') + self.check_bad_tree(tree, "non-string encoding") + tree = \ + (339, + (257, (0, '')), + '\udcff') + with self.assertRaises(UnicodeEncodeError): + parser.sequence2st(tree) + class CompileTestCase(unittest.TestCase): @@ -772,6 +838,21 @@ def test_comparisons(self): self.assertRaises(TypeError, operator.lt, st1, 1815) self.assertRaises(TypeError, operator.gt, b'waterloo', st2) + def test_copy_pickle(self): + sts = [ + parser.expr('2 + 3'), + parser.suite('x = 2; y = x + 3'), + parser.expr('list(x**3 for x in range(20))') + ] + for st in sts: + st_copy = copy.copy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + st_copy = copy.deepcopy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + st_copy = pickle.loads(pickle.dumps(st, proto)) + self.assertEqual(st_copy.totuple(), st.totuple()) + check_sizeof = support.check_sizeof @support.cpython_only diff --git a/Misc/NEWS b/Misc/NEWS index 44a4b873b1454b..cb9fab656ec692 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,8 @@ Core and Builtins Library ------- +- bpo-30070: Fixed leaks and crashes in errors handling in the parser module. + - bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when readline() or __next__() respectively return non-sizeable object. Fixed possible other errors caused by not checking results of PyObject_Size(), diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index b2566951d06de1..a4443350ef1cdc 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -698,7 +698,7 @@ validate_node(node *tree) short a_label = dfa_state->s_arc[arc].a_lbl; assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); if (_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) { - /* The child is acceptable; if non-terminal, validate it recursively. */ + /* The child is acceptable; if non-terminal, validate it recursively. */ if (ISNONTERMINAL(ch_type) && !validate_node(ch)) return 0; @@ -776,32 +776,35 @@ parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) */ tree = build_node_tree(tuple); if (tree != 0) { - node *validation_root = tree; + node *validation_root = NULL; int tree_type = 0; switch (TYPE(tree)) { case eval_input: /* Might be an eval form. */ tree_type = PyST_EXPR; + validation_root = tree; break; case encoding_decl: /* This looks like an encoding_decl so far. */ - if (NCH(tree) != 1) + if (NCH(tree) == 1) { + tree_type = PyST_SUITE; + validation_root = CHILD(tree, 0); + } + else { err_string("Error Parsing encoding_decl"); - validation_root = CHILD(tree, 0); - /* Fall through */ + } + break; case file_input: /* This looks like an exec form so far. */ - tree_type = PyST_SUITE; + validation_root = tree; break; default: /* This is a fragment, at best. */ - PyNode_Free(tree); err_string("parse tree does not use a valid start symbol"); - return (0); } - if (validate_node(validation_root)) + if (validation_root != NULL && validate_node(validation_root)) st = parser_newstobject(tree, tree_type); else PyNode_Free(tree); @@ -831,6 +834,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_ssize_t i; int err; + if (len < 0) { + return NULL; + } for (i = 1; i < len; ++i) { /* elem must always be a sequence, however simple */ PyObject* elem = PySequence_GetItem(tuple, i); @@ -851,7 +857,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if (type == -1 && PyErr_Occurred()) { Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } } Py_DECREF(temp); @@ -863,7 +869,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) PyErr_SetObject(parser_error, err); Py_XDECREF(err); Py_XDECREF(elem); - return (0); + return NULL; } if (ISTERMINAL(type)) { Py_ssize_t len = PyObject_Size(elem); @@ -872,11 +878,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if ((len != 2) && (len != 3)) { err_string("terminal nodes must have 2 or 3 entries"); - return 0; + Py_DECREF(elem); + return NULL; } temp = PySequence_GetItem(elem, 1); - if (temp == NULL) - return 0; + if (temp == NULL) { + Py_DECREF(elem); + return NULL; + } if (!PyUnicode_Check(temp)) { PyErr_Format(parser_error, "second item in terminal node must be a string," @@ -884,46 +893,49 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_TYPE(temp)->tp_name); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } if (len == 3) { PyObject *o = PySequence_GetItem(elem, 2); - if (o != NULL) { - if (PyLong_Check(o)) { - int num = _PyLong_AsInt(o); - if (num == -1 && PyErr_Occurred()) { - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return 0; - } - *line_num = num; - } - else { - PyErr_Format(parser_error, - "third item in terminal node must be an" - " integer, found %s", - Py_TYPE(temp)->tp_name); + if (o == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (PyLong_Check(o)) { + int num = _PyLong_AsInt(o); + if (num == -1 && PyErr_Occurred()) { Py_DECREF(o); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } + *line_num = num; + } + else { + PyErr_Format(parser_error, + "third item in terminal node must be an" + " integer, found %s", + Py_TYPE(temp)->tp_name); Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; } + Py_DECREF(o); } temp_str = PyUnicode_AsUTF8AndSize(temp, &len); if (temp_str == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); - return 0; + Py_DECREF(elem); + return NULL; } strn = (char *)PyObject_MALLOC(len + 1); if (strn == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); + Py_DECREF(elem); PyErr_NoMemory(); - return 0; + return NULL; } (void) memcpy(strn, temp_str, len + 1); Py_DECREF(temp); @@ -933,20 +945,21 @@ build_node_children(PyObject *tuple, node *root, int *line_num) * It has to be one or the other; this is an error. * Raise an exception. */ - PyObject *err = Py_BuildValue("os", elem, "unknown node type."); + PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } err = PyNode_AddChild(root, type, strn, *line_num, 0); if (err == E_NOMEM) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); - return (node *) PyErr_NoMemory(); + PyErr_NoMemory(); + return NULL; } if (err == E_OVERFLOW) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); PyErr_SetString(PyExc_ValueError, "unsupported number of child nodes"); @@ -957,14 +970,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) node* new_child = CHILD(root, i - 1); if (new_child != build_node_children(elem, new_child, line_num)) { - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } } else if (type == NEWLINE) { /* It's true: we increment the */ ++(*line_num); /* line number *after* the newline! */ } - Py_XDECREF(elem); + Py_DECREF(elem); } return root; } @@ -999,10 +1012,23 @@ build_node_tree(PyObject *tuple) if (num == encoding_decl) { encoding = PySequence_GetItem(tuple, 2); + if (encoding == NULL) { + PyErr_SetString(parser_error, "missed encoding"); + return NULL; + } + if (!PyUnicode_Check(encoding)) { + PyErr_Format(parser_error, + "encoding must be a string, found %.200s", + Py_TYPE(encoding)->tp_name); + Py_DECREF(encoding); + return NULL; + } /* tuple isn't borrowed anymore here, need to DECREF */ tuple = PySequence_GetSlice(tuple, 0, 2); - if (tuple == NULL) + if (tuple == NULL) { + Py_DECREF(encoding); return NULL; + } } res = PyNode_New(num); if (res != NULL) { @@ -1015,31 +1041,33 @@ build_node_tree(PyObject *tuple) const char *temp; temp = PyUnicode_AsUTF8AndSize(encoding, &len); if (temp == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); return NULL; } res->n_str = (char *)PyObject_MALLOC(len + 1); if (res->n_str == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); PyErr_NoMemory(); return NULL; } (void) memcpy(res->n_str, temp, len + 1); - Py_DECREF(encoding); - Py_DECREF(tuple); } } + if (encoding != NULL) { + Py_DECREF(encoding); + Py_DECREF(tuple); + } } else { /* The tuple is illegal -- if the number is neither TERMINAL nor * NONTERMINAL, we can't use it. Not sure the implementation * allows this condition, but the API doesn't preclude it. */ - PyObject *err = Py_BuildValue("os", tuple, + PyObject *err = Py_BuildValue("Os", tuple, "Illegal component tuple."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); @@ -1074,7 +1102,6 @@ parser__pickler(PyObject *self, PyObject *args) result = Py_BuildValue("O(O)", pickle_constructor, tuple); Py_DECREF(tuple); } - Py_DECREF(empty_dict); Py_DECREF(newargs); } finally: From e2546172622dd52692cf0e26c2b931f942f345b6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Apr 2017 23:59:02 +0300 Subject: [PATCH 0240/1345] [3.6] bpo-30065: Fixed arguments validation in _posixsubprocess.fork_exec(). (GH-1110) (#1186) (cherry picked from commit 66bffd1) --- Lib/multiprocessing/util.py | 2 +- Lib/subprocess.py | 3 ++- Lib/test/test_capi.py | 6 +++--- Lib/test/test_subprocess.py | 13 ++++++++++- Modules/_posixsubprocess.c | 43 ++++++++++++++++++++----------------- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 1a2c0db40b9cc6..0ce274ceca6057 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -386,7 +386,7 @@ def _close_stdin(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - passfds = sorted(passfds) + passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 0db6571f291dae..172126929160bf 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1253,7 +1253,8 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, + close_fds, tuple(sorted(map(int, fds_to_keep))), + cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c3625d4879138..08aa3ffdf6b4f1 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -98,7 +98,7 @@ class Z(object): def __len__(self): return 1 self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -106,7 +106,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -116,7 +116,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 3c871dde5c532f..d8d6d82713cded 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2431,7 +2431,7 @@ def test_fork_exec(self): with self.assertRaises(TypeError): _posixsubprocess.fork_exec( args, exe_list, - True, [], cwd, env_list, + True, (), cwd, env_list, -1, -1, -1, -1, 1, 2, 3, 4, True, True, func) @@ -2443,6 +2443,16 @@ def test_fork_exec(self): def test_fork_exec_sorted_fd_sanity_check(self): # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + gc_enabled = gc.isenabled() try: gc.enable() @@ -2453,6 +2463,7 @@ def test_fork_exec_sorted_fd_sanity_check(self): (18, 23, 42, 2**63), # Out of range. (5, 4), # Not sorted. (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), ): with self.assertRaises( ValueError, diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 5007a39bc2443d..d1434d59f818b4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -111,13 +111,17 @@ _is_fdescfs_mounted_on_dev_fd(void) static int _sanity_check_python_fd_sequence(PyObject *fd_sequence) { - Py_ssize_t seq_idx, seq_len = PySequence_Length(fd_sequence); + Py_ssize_t seq_idx; long prev_fd = -1; - for (seq_idx = 0; seq_idx < seq_len; ++seq_idx) { - PyObject* py_fd = PySequence_Fast_GET_ITEM(fd_sequence, seq_idx); - long iter_fd = PyLong_AsLong(py_fd); + for (seq_idx = 0; seq_idx < PyTuple_GET_SIZE(fd_sequence); ++seq_idx) { + PyObject* py_fd = PyTuple_GET_ITEM(fd_sequence, seq_idx); + long iter_fd; + if (!PyLong_Check(py_fd)) { + return 1; + } + iter_fd = PyLong_AsLong(py_fd); if (iter_fd < 0 || iter_fd <= prev_fd || iter_fd > INT_MAX) { - /* Negative, overflow, not a Long, unsorted, too big for a fd. */ + /* Negative, overflow, unsorted, too big for a fd. */ return 1; } prev_fd = iter_fd; @@ -132,13 +136,12 @@ _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) { /* Binary search. */ Py_ssize_t search_min = 0; - Py_ssize_t search_max = PySequence_Length(fd_sequence) - 1; + Py_ssize_t search_max = PyTuple_GET_SIZE(fd_sequence) - 1; if (search_max < 0) return 0; do { long middle = (search_min + search_max) / 2; - long middle_fd = PyLong_AsLong( - PySequence_Fast_GET_ITEM(fd_sequence, middle)); + long middle_fd = PyLong_AsLong(PyTuple_GET_ITEM(fd_sequence, middle)); if (fd == middle_fd) return 1; if (fd > middle_fd) @@ -154,9 +157,9 @@ make_inheritable(PyObject *py_fds_to_keep, int errpipe_write) { Py_ssize_t i, len; - len = PySequence_Length(py_fds_to_keep); + len = PyTuple_GET_SIZE(py_fds_to_keep); for (i = 0; i < len; ++i) { - PyObject* fdobj = PySequence_Fast_GET_ITEM(py_fds_to_keep, i); + PyObject* fdobj = PyTuple_GET_ITEM(py_fds_to_keep, i); long fd = PyLong_AsLong(fdobj); assert(!PyErr_Occurred()); assert(0 <= fd && fd <= INT_MAX); @@ -213,14 +216,13 @@ static void _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) { long end_fd = safe_get_max_fd(); - Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); Py_ssize_t keep_seq_idx; int fd_num; /* As py_fds_to_keep is sorted we can loop through the list closing * fds inbetween any in the keep list falling within our range. */ for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { - PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep, - keep_seq_idx); + PyObject* py_keep_fd = PyTuple_GET_ITEM(py_fds_to_keep, keep_seq_idx); int keep_fd = PyLong_AsLong(py_keep_fd); if (keep_fd < start_fd) continue; @@ -306,7 +308,7 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) /* Close all open file descriptors from start_fd and higher. - * Do not close any in the sorted py_fds_to_keep list. + * Do not close any in the sorted py_fds_to_keep tuple. * * This function violates the strict use of async signal safe functions. :( * It calls opendir(), readdir() and closedir(). Of these, the one most @@ -562,8 +564,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) #endif if (!PyArg_ParseTuple( - args, "OOpOOOiiiiiiiiiiO:fork_exec", - &process_args, &executable_list, &close_fds, &py_fds_to_keep, + args, "OOpO!OOiiiiiiiiiiO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, @@ -574,10 +577,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); return NULL; } - if (PySequence_Length(py_fds_to_keep) < 0) { - PyErr_SetString(PyExc_ValueError, "cannot get length of fds_to_keep"); - return NULL; - } if (_sanity_check_python_fd_sequence(py_fds_to_keep)) { PyErr_SetString(PyExc_ValueError, "bad value(s) in fds_to_keep"); return NULL; @@ -631,6 +630,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) goto cleanup; for (arg_num = 0; arg_num < num_args; ++arg_num) { PyObject *borrowed_arg, *converted_arg; + if (PySequence_Fast_GET_SIZE(fast_args) != num_args) { + PyErr_SetString(PyExc_RuntimeError, "args changed during iteration"); + goto cleanup; + } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) goto cleanup; From 5fb4bf91e78e128a22b5b58da179f09d0b307f2d Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Thu, 20 Apr 2017 12:54:06 +0900 Subject: [PATCH 0241/1345] bpo-19225: Remove duplicated description for standard warning categories (GH-1068) --- Doc/c-api/exceptions.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3516a1690384f7..c75408db770a22 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -291,16 +291,11 @@ an error value). is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. - Warning categories must be subclasses of :c:data:`Warning`; the default warning - category is :c:data:`RuntimeWarning`. The standard Python warning categories are - available as global variables whose names are ``PyExc_`` followed by the Python - exception name. These have the type :c:type:`PyObject\*`; they are all class - objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`, - :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`, - :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and - :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of - :c:data:`PyExc_Exception`; the other warning categories are subclasses of - :c:data:`PyExc_Warning`. + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standarwarningcategories`. For information about warning control, see the documentation for the :mod:`warnings` module and the :option:`-W` option in the command line @@ -963,8 +958,10 @@ Notes: Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -Standard Warnings -================= +.. _standarwarningcategories: + +Standard Warning Categories +=========================== All standard Python warning categories are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type From 596e714d79a22d1d53f26ae1a579dcd43f15865a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 20 Apr 2017 07:36:57 +0300 Subject: [PATCH 0242/1345] bpo-30078: Add an example of passing a path to unittest (GH-1178) (cherry picked from commit f7e62cf8adfb8ab6a6a870903defe8ff218a0383) --- Lib/unittest/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 09fefe11649bd3..807604f08dfd14 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -14,6 +14,7 @@ %(prog)s test_module - run tests from test_module %(prog)s module.TestClass - run tests from module.TestClass %(prog)s module.Class.test_method - run specified test method + %(prog)s path/to/test_file.py - run tests from test_file.py """ MODULE_EXAMPLES = """\ From 83a90b9aeda3a4c10e8393893ed4ec8adc861e62 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 20 Apr 2017 07:42:32 +0300 Subject: [PATCH 0243/1345] Remove redundant comma in argparse HOWTO (GH-1141) Reported by Sean Canavan on docs@p.o. (cherry picked from commit 8526fb74edf5ac9ca175b7cdcb0d82bb8780d2cf) --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 7e161a59add8ae..9d770f5232b440 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -221,7 +221,7 @@ before proceeding. Introducing Optional arguments ============================== -So far we, have been playing with positional arguments. Let us +So far we have been playing with positional arguments. Let us have a look on how to add optional ones:: import argparse From df5df13fdc3a71bcf2295acc2cba7f22cfe2d669 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 20 Apr 2017 17:26:25 +0900 Subject: [PATCH 0244/1345] [3.6] bpo-12414: Update code_sizeof() to take in account co_extra memory. (#1168) (#1198) --- Lib/test/test_sys.py | 8 +++++--- Misc/ACKS | 1 + Misc/NEWS | 4 ++++ Objects/codeobject.c | 10 +++++++--- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index df9ebd40859e21..e151f493d4e36c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -913,13 +913,15 @@ def inner(): return inner check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size('6i13P')) - check(get_cell.__code__, size('6i13P')) + def check_code_size(a, expected_size): + self.assertGreaterEqual(sys.getsizeof(a), expected_size) + check_code_size(get_cell().__code__, size('6i13P')) + check_code_size(get_cell.__code__, size('6i13P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size('6i13P') + 1) + check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n')) # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) diff --git a/Misc/ACKS b/Misc/ACKS index 72070e19f65701..ac2c4edc733576 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1060,6 +1060,7 @@ R. David Murray Matti Mäki Jörg Müller Kaushik N +Dong-hee Na Dale Nagata John Nagle Takahiro Nakayama diff --git a/Misc/NEWS b/Misc/NEWS index cb9fab656ec692..b1f79d61240281 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-12414: sys.getsizeof() on a code object now returns the sizes + which includes the code struct and sizes of objects which it references. + Patch by Dong-hee Na. + - bpo-29949: Fix memory usage regression of set and frozenset object. - bpo-29935: Fixed error messages in the index() method of tuple, list and deque diff --git a/Objects/codeobject.c b/Objects/codeobject.c index df8b9538fe9ed5..22c4f856cd83e6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -446,11 +446,15 @@ code_dealloc(PyCodeObject *co) static PyObject * code_sizeof(PyCodeObject *co, void *unused) { - Py_ssize_t res; + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; - res = _PyObject_SIZE(Py_TYPE(co)); if (co->co_cell2arg != NULL && co->co_cellvars != NULL) - res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(unsigned char); + res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(Py_ssize_t); + + if (co_extra != NULL) + res += co_extra->ce_size * sizeof(co_extra->ce_extras[0]); + return PyLong_FromSsize_t(res); } From 430020500c56259d31e07d32771f017952bad4e0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 20 Apr 2017 07:12:37 -0700 Subject: [PATCH 0245/1345] [3.6] bpo-30109: Fix reindent.py (GH-1207) (GH-1208) Skip the file if it has bad encoding. (cherry picked from commit 58f3c9dc8f5626abe09ac9738c34f6ba99ce2972) --- Lib/test/test_tools/test_reindent.py | 7 +++++++ Tools/scripts/reindent.py | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py index d7c20e1e5c7206..34df0c5d511904 100644 --- a/Lib/test/test_tools/test_reindent.py +++ b/Lib/test/test_tools/test_reindent.py @@ -7,6 +7,7 @@ import os import unittest from test.support.script_helper import assert_python_ok +from test.support import findfile from test.test_tools import scriptsdir, skip_if_missing @@ -23,6 +24,12 @@ def test_help(self): self.assertEqual(out, b'') self.assertGreater(err, b'') + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + if __name__ == '__main__': unittest.main() diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index 18424dea1437ac..f6dadaac5a5206 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -118,7 +118,11 @@ def check(file): if verbose: print("checking", file, "...", end=' ') with open(file, 'rb') as f: - encoding, _ = tokenize.detect_encoding(f.readline) + try: + encoding, _ = tokenize.detect_encoding(f.readline) + except SyntaxError as se: + errprint("%s: SyntaxError: %s" % (file, str(se))) + return try: with open(file, encoding=encoding) as f: r = Reindenter(f) From 7a113a0cbf545588d61286fcc0e89141cf211735 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 20 Apr 2017 22:55:06 +0300 Subject: [PATCH 0246/1345] bpo-29802: Fix the cleaning up issue in PyUnicode_FSDecoder(). (#1217) --- Objects/unicodeobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7871043681533d..4e0c663e338a52 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3902,6 +3902,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) PyObject *output = NULL; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); + *(PyObject**)addr = NULL; return 1; } From 483729526e06cda0befc241190bb586d472a72e8 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 20 Apr 2017 20:55:59 -0700 Subject: [PATCH 0247/1345] [3.6] Add missing .gitignore entries for VS2015 IntelliSense DB (GH-1223) (#1225) (cherry picked from commit 8e675286a92f33837cfffac5914b5175dac5d573) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index db27597aac57da..9392fe5defd6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,8 @@ PCbuild/*.suo PCbuild/*.*sdf PCbuild/*-pgi PCbuild/*-pgo +PCbuild/*.VC.db +PCbuild/*.VC.opendb PCbuild/.vs/ PCbuild/amd64/ PCbuild/obj/ From b984a05d557e78b928f38098d0b6c0e95f737b9a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Apr 2017 18:27:29 +0200 Subject: [PATCH 0248/1345] bpo-30107: don't dump core on expected test_io crash (#1235) (#1241) test_io has two unit tests which trigger a deadlock: * test_daemon_threads_shutdown_stdout_deadlock() * test_daemon_threads_shutdown_stderr_deadlock() These tests call Py_FatalError() if the expected bug is triggered which calls abort(). Use test.support.SuppressCrashReport to prevent the creation on a core dump, to fix the warning: Warning -- files was modified by test_io Before: [] After: ['python.core'] (cherry picked from commit 2a1aed04b0943636f605543522e16cca1dc23e70) --- Lib/test/test_io.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5a7617cd9f9cbf..8c91ad2ad8aa1c 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3720,6 +3720,7 @@ def check_daemon_threads_shutdown_deadlock(self, stream_name): import sys import time import threading + from test.support import SuppressCrashReport file = sys.{stream_name} @@ -3728,6 +3729,10 @@ def run(): file.write('.') file.flush() + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + thread = threading.Thread(target=run) thread.daemon = True thread.start() From 1ccbe6007e5569ab25170f8ecddd5fbbc2ef36b0 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 21 Apr 2017 11:28:46 -0700 Subject: [PATCH 0249/1345] bpo-29191: Add liblzma.vcxproj to pcbuild.sln and other missing entries (GH-1222) (GH-1244) liblzma is missing from pcbuild.sln. This causes the build of _lzma to fail when building the solution and not using build.bat. --- PCbuild/pcbuild.sln | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 0e65811ae70731..580879930a83d0 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -96,6 +96,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -728,6 +730,38 @@ Global {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 3a8f8ea2aca46f9ef1e715f2c924357b6719525f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Apr 2017 23:17:33 +0200 Subject: [PATCH 0250/1345] bpo-30125: Fix faulthandler.disable() on Windows (#1243) On Windows, faulthandler.disable() now removes the exception handler installed by faulthandler.enable(). --- Lib/test/test_faulthandler.py | 12 ++++++++++++ Modules/faulthandler.c | 13 +++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index bdd8d1a2a6163f..01cbae3f200b1a 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,18 @@ def test_raise_exception(self): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_disable_windows_exc_handler(self): + code = dedent(""" + import faulthandler + faulthandler.enable() + faulthandler.disable() + code = faulthandler._EXCEPTION_ACCESS_VIOLATION + faulthandler._raise_exception(code) + """) + output, exitcode = self.get_output(code) + self.assertEqual(output, []) + self.assertEqual(exitcode, 0xC0000005) if __name__ == "__main__": diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 2f8b624fd160d9..61fc4908b0fe9b 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -55,6 +55,9 @@ static struct { int fd; int all_threads; PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif } fatal_error = {0, NULL, -1, 0}; #ifdef FAULTHANDLER_LATER @@ -462,7 +465,8 @@ faulthandler_enable(void) } #ifdef MS_WINDOWS - AddVectoredExceptionHandler(1, faulthandler_exc_handler); + assert(fatal_error.exc_handler == NULL); + fatal_error.exc_handler = AddVectoredExceptionHandler(1, faulthandler_exc_handler); #endif return 0; } @@ -514,7 +518,12 @@ faulthandler_disable(void) faulthandler_disable_fatal_handler(handler); } } - +#ifdef MS_WINDOWS + if (fatal_error.exc_handler != NULL) { + RemoveVectoredExceptionHandler(fatal_error.exc_handler); + fatal_error.exc_handler = NULL; + } +#endif Py_CLEAR(fatal_error.file); } From 414e0283a351e7b070c3f13aa24cadc5fdd9a230 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 21 Apr 2017 19:28:29 -0700 Subject: [PATCH 0251/1345] [3.6] Correct the README link in Unix install docs (GH-1245) (GH-1248) (cherry picked from commit d1ae24e8880fe00d006eb2996af815c35fbcfb33) --- Doc/using/unix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 97f0a49ca771e5..604688ce94cc32 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -77,7 +77,7 @@ The build process consists in the usual :: make install invocations. Configuration options and caveats for specific Unix platforms are -extensively documented in the :source:`README` file in the root of the Python +extensively documented in the :source:`README.rst` file in the root of the Python source tree. .. warning:: From a3d8dda7d899bf41ab7eb2c6148459ad276fe295 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 21 Apr 2017 19:58:28 -0700 Subject: [PATCH 0252/1345] bpo-30098: Clarify that run_coroutine_threadsafe expects asyncio.Future (GH-1170) (#1247) (cherry picked from commit ae5b3260dd459845aad8a30491b76d471577785d) --- Lib/asyncio/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4d79367d5cb600..f91e70aecbad94 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -517,7 +517,8 @@ def ensure_future(coro_or_future, *, loop=None): elif compat.PY35 and inspect.isawaitable(coro_or_future): return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) else: - raise TypeError('A Future, a coroutine or an awaitable is required') + raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' + 'required') @coroutine From 9d022f169ec64987b29359ae0e904026aab07a64 Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Sat, 22 Apr 2017 19:48:53 +0800 Subject: [PATCH 0253/1345] [3.6] Fix trailing colon and newline in test.rst (GH-1250) (#1254) (cherry picked from commit 7fae81e1672d0b4110d31ea6a765b54f63a2e54b) --- Doc/library/test.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index fab3e1fe4cc6ad..9d4ff7ad8b45a3 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -570,7 +570,8 @@ The :mod:`test.support` module defines the following functions: def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()): + +.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not found on *other_api*, except for a defined list of items to be From f2ed2858eea7569c8915b3611ca5ec92ae10b17f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 23 Apr 2017 08:50:14 +0300 Subject: [PATCH 0254/1345] [3.6] bpo-15718: Document the upper bound constrain on the __len__ return value. (GH-1256) (#1259) (cherry picked from commit 85157cd) --- Doc/reference/datamodel.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 095a2380b379bc..35925a065fd495 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2011,6 +2011,14 @@ through the container; for mappings, :meth:`__iter__` should be the same as :meth:`__bool__` method and whose :meth:`__len__` method returns zero is considered to be false in a Boolean context. + .. impl-detail:: + + In CPython, the length is required to be at most :attr:`sys.maxsize`. + If the length is larger than :attr:`!sys.maxsize` some features (such as + :func:`len`) may raise :exc:`OverflowError`. To prevent raising + :exc:`!OverflowError` by truth value testing, an object must define a + :meth:`__bool__` method. + .. method:: object.__length_hint__(self) @@ -2021,6 +2029,7 @@ through the container; for mappings, :meth:`__iter__` should be the same as .. versionadded:: 3.4 + .. note:: Slicing is done exclusively with the following three methods. A call like :: From ea0efa3bc1d0b832da75519c6f85d767ae44feda Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 23 Apr 2017 21:05:01 -0700 Subject: [PATCH 0255/1345] [3.6] bpo-29751: Improve PyLong_FromString documentation (GH-915) (#1266) (cherry picked from commit 26896f2832324dde85cdd63d525571ca669f6f0b) --- Doc/c-api/long.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f592cb65c3e3a9..f50680b3d29558 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -85,13 +85,12 @@ All integers are implemented as "long" integer objects of arbitrary size. Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, the radix will be - determined based on the leading characters of *str*: if *str* starts with - ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or - ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``, - radix 2 will be used; otherwise radix 10 will be used. If *base* is not - ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are - ignored. If there are no digits, :exc:`ValueError` will be raised. + representation of the number. If *base* is ``0``, *str* is interpreted using + the :ref:`integers` definition; in this case, leading zeros in a + non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, + it must be between ``2`` and ``36``, inclusive. Leading spaces and single + underscores after a base specifier and between digits are ignored. If there + are no digits, :exc:`ValueError` will be raised. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) From 16b6f971bd8afc72b439b83324eba744460adb67 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 25 Apr 2017 18:23:05 -0700 Subject: [PATCH 0256/1345] [3.6] bpo-28851: Improve namedtuple documentation (GH-1274) (GH-1286) Clarify that a sequence of strings is the preferred value for 'field_names'. (cherry picked from commit 97bf722fcd1de1236824377e052369dc7686b644) --- Doc/library/collections.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 8e2eb4d9b0382f..2d51f0cfdc094d 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -771,9 +771,9 @@ they add the ability to access fields by name instead of position index. helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` method which lists the tuple contents in a ``name=value`` format. - The *field_names* are a single string with each fieldname separated by whitespace - and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* - can be a sequence of strings such as ``['x', 'y']``. + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. Any valid Python identifier may be used for a fieldname except for names starting with an underscore. Valid identifiers consist of letters, digits, From 9f6828119df23fca9e799f47e02baabe87adca2a Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Wed, 26 Apr 2017 16:45:31 +0800 Subject: [PATCH 0257/1345] bpo-28698: Fix c_wchar_p doc example (GH-1160) (cherry picked from commit 0d637e236d7099f7b724026c8cb7bd83d8e12e6b) --- Doc/library/ctypes.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3840935ce04eb6..51a3a360fa3e1e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -284,7 +284,7 @@ the correct type and value:: >>> c_int() c_long(0) >>> c_wchar_p("Hello, World") - c_wchar_p('Hello, World') + c_wchar_p(140018365411392) >>> c_ushort(-3) c_ushort(65533) >>> @@ -309,11 +309,15 @@ bytes objects are immutable):: >>> s = "Hello, World" >>> c_s = c_wchar_p(s) >>> print(c_s) - c_wchar_p('Hello, World') + c_wchar_p(139966785747344) + >>> print(c_s.value) + Hello World >>> c_s.value = "Hi, there" - >>> print(c_s) - c_wchar_p('Hi, there') - >>> print(s) # first object is unchanged + >>> print(c_s) # the memory location has changed + c_wchar_p(139966783348904) + >>> print(c_s.value) + Hi, there + >>> print(s) # first object is unchanged Hello, World >>> From 04f389b8b357c8507b6c9642adcb8b956489e034 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 26 Apr 2017 17:25:37 +0300 Subject: [PATCH 0258/1345] bpo-29974: Improve typing.TYPE_CHECKING example (GH-982) * Fix PEP 8 (SomeType instead of some_type) * Add a function parameter annotation * Explain, using wording from PEP 484 and PEP 526, why one annotation is in quotes and another is not. Suggested by Ivan Levkevskyi. (cherry picked from commit 87c07fe9d908d0a2143fcc8369255c6ff3241503) --- Doc/library/typing.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2cfc37f695ef2d..1780739ad1dd43 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1041,5 +1041,10 @@ The module defines the following classes, functions and decorators: if TYPE_CHECKING: import expensive_mod - def fun(): - local_var: expensive_mod.some_type = other_fun() + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + Note that the first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. From 2fc3c543fc93d6211de8dcfaeac3a0fd10a356e0 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 27 Apr 2017 01:55:55 +0900 Subject: [PATCH 0259/1345] [3.6] bpo-30142: Remove "callable" from the 2to3fixer documentation. (GH-1303) --- Doc/library/2to3.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index ace1bfaf8cb9d6..4c9a528d42e703 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -199,13 +199,6 @@ and off individually. They are described here in more detail. because the :class:`memoryview` API is similar but not exactly the same as that of :class:`buffer`. -.. 2to3fixer:: callable - - Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding - an import to :mod:`collections` if needed. Note ``callable(x)`` has returned - in Python 3.2, so if you do not intend to support Python 3.1, you can disable - this fixer. - .. 2to3fixer:: dict Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to From 33a5568f69301562d536852d12b9e03cd8dfc3a4 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 26 Apr 2017 22:18:53 -0700 Subject: [PATCH 0260/1345] [3.6] bpo-30182: Use the correct name for ISO in Unicode HOWTO. (GH-1312) (GH-1313) (cherry picked from commit 6fde770e4e940c19cd62de0b6aeb77840690843e) --- Doc/howto/unicode.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index a48ae1f5faba7e..9649b9c609c255 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -43,9 +43,9 @@ hold values ranging from 0 to 255. ASCII codes only went up to 127, so some machines assigned values between 128 and 255 to accented characters. Different machines had different codes, however, which led to problems exchanging files. Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Standards Organization, -and some were *de facto* conventions that were invented by one company or -another and managed to catch on. +Some were true standards, defined by the International Organization for +Standardization, and some were *de facto* conventions that were invented by one +company or another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian From 4dc3b9cf2a868a8378ecbc2221a15e7b59e8e944 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 27 Apr 2017 18:25:03 +0200 Subject: [PATCH 0261/1345] bpo-30175: Skip client cert tests of test_imaplib (#1320) (#1323) * bpo-30175: Skip client cert tests of test_imaplib The IMAP server cyrus.andrew.cmu.edu doesn't accept our randomly generated client x509 certificate anymore. * bpo-30188: Catch EOFError in NetworkedNNTPTests test_nntplib fails randomly with EOFError in NetworkedNNTPTests.setUpClass(). Catch EOFError to skip tests in that case. (cherry picked from commit 5bccca58b9b2b3a925b16750bedbd907695ea8d7) --- Lib/test/test_imaplib.py | 6 ++++++ Lib/test/test_nntplib.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index fac6b57081283b..9df37b1de58361 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -966,6 +966,9 @@ def test_logincapa(self): _server = self.imap_class(self.host, self.port) self.check_logincapa(_server) + @unittest.skipIf(True, + "bpo-30175: FIXME: cyrus.andrew.cmu.edu doesn't accept " + "our randomly generated client x509 certificate anymore") def test_logincapa_with_client_certfile(self): with transient_internet(self.host): with support.check_warnings(('', DeprecationWarning)): @@ -973,6 +976,9 @@ def test_logincapa_with_client_certfile(self): certfile=CERTFILE) self.check_logincapa(_server) + @unittest.skipIf(True, + "bpo-30175: FIXME: cyrus.andrew.cmu.edu doesn't accept " + "our randomly generated client x509 certificate anymore") def test_logincapa_with_client_ssl_context(self): with transient_internet(self.host): _server = self.imap_class( diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index c46ded11866589..3e84f3429ee854 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -286,7 +286,12 @@ class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase): def setUpClass(cls): support.requires("network") with support.transient_internet(cls.NNTP_HOST): - cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) + try: + cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) + except EOFError: + raise unittest.SkipTest(f"{cls} got EOF error on connecting " + f"to {cls.NNTP_HOST!r}") @classmethod def tearDownClass(cls): From c9ca57eeea4d9da748d84281506c5803512b79ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Apr 2017 03:27:51 +0200 Subject: [PATCH 0262/1345] bpo-30107: Make SuppressCrashReport quiet on macOS (#1279) (#1335) On macOS, SuppressCrashReport now redirects /usr/bin/defaults command stderr into a pipe to not pollute stderr. It fixes a test_io.test_daemon_threads_shutdown_stderr_deadlock() failure when the CrashReporter domain doesn't exists. Message logged into stderr: 2017-04-24 16:57:21.432 defaults[41046:2462851] The domain/default pair of (com.apple.CrashReporter, DialogType) does not exist (cherry picked from commit d819ad9832292d854e9710493ecdf959b69802e3) --- Lib/test/support/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15d8fc849b9a0a..1621e3de402c36 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2433,6 +2433,7 @@ def __enter__(self): (0, self.old_value[1])) except (ValueError, OSError): pass + if sys.platform == 'darwin': # Check if the 'Crash Reporter' on OSX was configured # in 'Developer' mode and warn that it will get triggered @@ -2440,10 +2441,14 @@ def __enter__(self): # # This assumes that this context manager is used in tests # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': + cmd = ['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with proc: + stdout = proc.communicate()[0] + if stdout.strip() == b'developer': print("this test triggers the Crash Reporter, " "that is intentional", end='', flush=True) From caa59c156d1e6ff3d005e0112bb27c5bed4afb71 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Apr 2017 04:13:37 +0200 Subject: [PATCH 0263/1345] [3.6] bpo-30106: Fix test_asyncore.test_quick_connect() (#1336) * Fix/optimize test_asyncore.test_quick_connect() (#1188) Don't use addCleanup() in test_quick_connect() because it keeps the Thread object alive and so @reap_threads fails on its timeout of 1 second. "./python -m test -v test_asyncore -m test_quick_connect" now takes 185 ms, instead of 11 seconds. Other minor changes: * Use "with sock:" to close the socket instead of try/finally: sock.close() * Use self.skipTest() in test_quick_connect() to remove one indentation level and notice user that the test is specific to AF_INET and AF_INET6 * bpo-30106: Fix tearDown() of test_asyncore (#1194) Call asyncore.close_all() with ignore_all=True in the tearDown() method of the test_asyncore base test case. It should prevent keeping alive sockets in asyncore.socket_map if close() fails with an unexpected error. Revert also an unwanted change of my previous commit: remove name parameter of Thread in test_quick_connect(). * bpo-30106: Fix test_asyncore.test_quick_connect() (#1234) test_quick_connect() runs a thread up to 50 seconds, whereas the socket is connected in 0.2 second and then the thread is expected to end in less than 3 second. On Linux, the thread ends quickly because select() seems to always return quickly. On FreeBSD, sometimes select() fails with timeout and so the thread runs much longer than expected. Fix the thread timeout to fix a race condition in the test. --- Lib/test/test_asyncore.py | 80 +++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index d05462b7efce35..dc2f716e0bb828 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -502,7 +502,7 @@ def handle_connect(self): class BaseTestAPI: def tearDown(self): - asyncore.close_all() + asyncore.close_all(ignore_all=True) def loop_waiting_for_flag(self, instance, timeout=5): timeout = float(timeout) / 100 @@ -755,50 +755,50 @@ def test_bind(self): def test_set_reuse_addr(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") - sock = socket.socket(self.family) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - finally: - sock.close() + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads def test_quick_connect(self): # see: http://bugs.python.org/issue10340 - if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - server = BaseServer(self.family, self.addr) - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=500)) - t.start() - def cleanup(): - t.join(timeout=TIMEOUT) - if t.is_alive(): - self.fail("join() timed out") - self.addCleanup(cleanup) - - s = socket.socket(self.family, socket.SOCK_STREAM) - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - try: - s.connect(server.address) - except OSError: - pass - finally: - s.close() + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") class TestAPI_UseIPv4Sockets(BaseTestAPI): family = socket.AF_INET From cb21f5f3d2005b891df0b76e3fc4d4676e285f21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Apr 2017 04:13:53 +0200 Subject: [PATCH 0264/1345] bpo-30131: test_logging now joins queue threads (#1298) (#1317) QueueListenerTest of test_logging now closes the multiprocessing Queue and joins its thread to prevent leaking dangling threads to following tests. Add also @support.reap_threads to detect earlier if a test leaks threads (and try to "cleanup" these threads). (cherry picked from commit 8ca2f2faefa8dba323a2e4c4b86efb633d7a53cf) --- Lib/test/test_logging.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9dedc098ba5aca..0a2584c6103c32 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3163,6 +3163,7 @@ def setup_and_log(log_queue, ident): handler.close() @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_queue_queue(self, mock_handle): for i in range(self.repeat): log_queue = queue.Queue() @@ -3172,10 +3173,13 @@ def test_handle_called_with_queue_queue(self, mock_handle): @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_mp_queue(self, mock_handle): for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + log_queue.close() + log_queue.join_thread() self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') @@ -3188,6 +3192,7 @@ def get_all_from_queue(log_queue): return [] @support.requires_multiprocessing_queue + @support.reap_threads def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3200,6 +3205,9 @@ def test_no_messages_in_queue_after_stop(self): self.setup_and_log(queue, '%s_%s' %(self.id(), i)) # time.sleep(1) items = list(self.get_all_from_queue(queue)) + queue.close() + queue.join_thread() + expected = [[], [logging.handlers.QueueListener._sentinel]] self.assertIn(items, expected, 'Found unexpected messages in queue: %s' % ( From e005dd9a6da7dfa34f47a6989a7e0a6b1ad04949 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Apr 2017 16:06:48 +0200 Subject: [PATCH 0265/1345] bpo-30125: disable faulthandler in ctypes test_SEH (#1237) (#1343) Disable faulthandler to run test_SEH() of test_ctypes to prevent the following log with a traceback: Windows fatal exception: access violation Add support.disable_faulthandler() context manager. (cherry picked from commit a36e939aeb3b5a2c56561eb24f0e339eee9f3f9d) --- Lib/ctypes/test/test_win32.py | 12 ++++++++---- Lib/test/support/__init__.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/test/test_win32.py b/Lib/ctypes/test/test_win32.py index da1624015e198c..5d85ad6200b319 100644 --- a/Lib/ctypes/test/test_win32.py +++ b/Lib/ctypes/test/test_win32.py @@ -41,15 +41,19 @@ class FunctionCallTestCase(unittest.TestCase): @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") def test_SEH(self): - # Call functions with invalid arguments, and make sure - # that access violations are trapped and raise an - # exception. - self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) + # Disable faulthandler to prevent logging the warning: + # "Windows fatal exception: access violation" + with support.disable_faulthandler(): + # Call functions with invalid arguments, and make sure + # that access violations are trapped and raise an + # exception. + self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) def test_noargs(self): # This is a special case on win32 x64 windll.user32.GetDesktopWindow() + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWintypes(unittest.TestCase): def test_HWND(self): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 1621e3de402c36..2d60adc3dc0fd1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2586,3 +2586,19 @@ def setswitchinterval(interval): if _is_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) + + +@contextlib.contextmanager +def disable_faulthandler(): + # use sys.__stderr__ instead of sys.stderr, since regrtest replaces + # sys.stderr with a StringIO which has no file descriptor when a test + # is run with -W/--verbose3. + fd = sys.__stderr__.fileno() + + is_enabled = faulthandler.is_enabled() + try: + faulthandler.disable() + yield + finally: + if is_enabled: + faulthandler.enable(file=fd, all_threads=True) From 712114b3f9f5f98f03c66a80607b6f7101c5cc90 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 28 Apr 2017 20:05:05 +0300 Subject: [PATCH 0266/1345] [3.6] bpo-30197: Enhance functions swap_attr() and swap_item() in test.support. (GH-1341) (#1345) They now work when delete replaced attribute or item inside the with statement. The old value of the attribute or item (or None if it doesn't exist) now will be assigned to the target of the "as" clause, if there is one. (cherry picked from commit d1a1def7bf221b04dcf3fc3a67aa19aa2f622f83) --- Lib/test/support/__init__.py | 16 ++++++++++++---- Lib/test/test_support.py | 29 +++++++++++++++++++++++------ Lib/test/test_tempfile.py | 5 ++--- Misc/NEWS | 9 +++++++++ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2d60adc3dc0fd1..f8d177990bfb03 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2105,12 +2105,15 @@ def swap_attr(obj, attr, new_val): restoring the old value at the end of the block. If `attr` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if hasattr(obj, attr): real_val = getattr(obj, attr) setattr(obj, attr, new_val) try: - yield + yield real_val finally: setattr(obj, attr, real_val) else: @@ -2118,7 +2121,8 @@ def swap_attr(obj, attr, new_val): try: yield finally: - delattr(obj, attr) + if hasattr(obj, attr): + delattr(obj, attr) @contextlib.contextmanager def swap_item(obj, item, new_val): @@ -2132,12 +2136,15 @@ def swap_item(obj, item, new_val): restoring the old value at the end of the block. If `item` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if item in obj: real_val = obj[item] obj[item] = new_val try: - yield + yield real_val finally: obj[item] = real_val else: @@ -2145,7 +2152,8 @@ def swap_item(obj, item, new_val): try: yield finally: - del obj[item] + if item in obj: + del obj[item] def strip_python_stderr(stderr): """Strip the stderr of a Python process from potential debug output diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e83a4d6426a8b2..ddd65092312a3b 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -282,17 +282,34 @@ def test_python_is_optimized(self): def test_swap_attr(self): class Obj: - x = 1 + pass obj = Obj() - with support.swap_attr(obj, "x", 5): + obj.x = 1 + with support.swap_attr(obj, "x", 5) as x: self.assertEqual(obj.x, 5) + self.assertEqual(x, 1) self.assertEqual(obj.x, 1) + with support.swap_attr(obj, "y", 5) as y: + self.assertEqual(obj.y, 5) + self.assertIsNone(y) + self.assertFalse(hasattr(obj, 'y')) + with support.swap_attr(obj, "y", 5): + del obj.y + self.assertFalse(hasattr(obj, 'y')) def test_swap_item(self): - D = {"item":1} - with support.swap_item(D, "item", 5): - self.assertEqual(D["item"], 5) - self.assertEqual(D["item"], 1) + D = {"x":1} + with support.swap_item(D, "x", 5) as x: + self.assertEqual(D["x"], 5) + self.assertEqual(x, 1) + self.assertEqual(D["x"], 1) + with support.swap_item(D, "y", 5) as y: + self.assertEqual(D["y"], 5) + self.assertIsNone(y) + self.assertNotIn("y", D) + with support.swap_item(D, "y", 5): + del D["y"] + self.assertNotIn("y", D) class RefClass: attribute1 = None diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 51df1ecd7d18e6..d0cf04b0cb67ca 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -273,13 +273,12 @@ def raise_OSError(*args, **kwargs): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - open = io.open def bad_writer(*args, **kwargs): - fp = open(*args, **kwargs) + fp = orig_open(*args, **kwargs) fp.write = raise_OSError return fp - with support.swap_attr(io, "open", bad_writer): + with support.swap_attr(io, "open", bad_writer) as orig_open: # test again with failing write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() diff --git a/Misc/NEWS b/Misc/NEWS index b1f79d61240281..936e2b02017f8e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -114,6 +114,15 @@ Documentation - bpo-26985: Add missing info of code object in inspect documentation. +Tests +----- + +- bpo-30197: Enhanced functions swap_attr() and swap_item() in the + test.support module. They now work when delete replaced attribute or item + inside the with statement. The old value of the attribute or item (or None + if it doesn't exist) now will be assigned to the target of the "as" clause, + if there is one. + What's New in Python 3.6.1? =========================== From 5a8dcec93936de816cef5c17522e48fca9c30b43 Mon Sep 17 00:00:00 2001 From: csabella Date: Sat, 29 Apr 2017 20:42:03 -0400 Subject: [PATCH 0267/1345] [3.6] bpo-30208: DOC: fix small typos in IDLE (#1356) (cherry picked from commit d9af73330f46d79cc0c56d369f65ebeec3cb5334) --- Doc/library/idle.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 07c2a25d6a012c..1a2dac0233c4d2 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -244,7 +244,7 @@ Go to File/Line single: stack viewer Debugger (toggle) - When actived, code entered in the Shell or run from an Editor will run + When activated, code entered in the Shell or run from an Editor will run under the debugger. In the Editor, breakpoints can be set with the context menu. This feature is still incomplete and somewhat experimental. @@ -372,7 +372,7 @@ the :kbd:`Command` key on Mac OSX. * :kbd:`C-l` center window around the insertion point - * :kbd:`C-b` go backwards one character without deleting (usually you can + * :kbd:`C-b` go backward one character without deleting (usually you can also use the cursor key for this) * :kbd:`C-f` go forward one character without deleting (usually you can @@ -394,7 +394,7 @@ After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python -Shell window one tab), number depends on Indent width. Currently tabs +Shell window one tab), number depends on Indent width. Currently, tabs are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands in the edit menu. @@ -418,7 +418,7 @@ If there is only one possible completion for the characters entered, a :kbd:`C-space` will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some +classes in the current namespaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific. If a string of characters is typed, the ACW selection will jump to the @@ -557,7 +557,7 @@ IDLE-console differences As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, +interface and operation occasionally affect visible results. For instance, ``sys.modules`` starts with more entries. IDLE also replaces ``sys.stdin``, ``sys.stdout``, and ``sys.stderr`` with @@ -583,7 +583,7 @@ If firewall software complains anyway, you can ignore it. If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch. @@ -619,14 +619,14 @@ Setting preferences The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab. Extensions ^^^^^^^^^^ -IDLE contains an extension facility. Peferences for extensions can be +IDLE contains an extension facility. Preferences for extensions can be changed with Configure Extensions. See the beginning of config-extensions.def in the idlelib directory for further information. The default extensions are currently: From 750ff5f8df8faef842470518c58dae2a251114c9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 29 Apr 2017 22:18:40 -0700 Subject: [PATCH 0268/1345] [3.6] Improve the grammar in windows.rst (GH-1330) (GH-1360) (cherry picked from commit 80a3da4d4aad0b51893e1e2f696b6252eca80e07) --- Doc/using/windows.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 3e4b70e8a17ef7..68687e9f3ec367 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -550,9 +550,9 @@ Shebang Lines If the first line of a script file starts with ``#!``, it is known as a "shebang" line. Linux and other Unix like operating systems have native -support for such lines and are commonly used on such systems to indicate how -a script should be executed. This launcher allows the same facilities to be -using with Python scripts on Windows and the examples above demonstrate their +support for such lines and they are commonly used on such systems to indicate +how a script should be executed. This launcher allows the same facilities to +be used with Python scripts on Windows and the examples above demonstrate their use. To allow shebang lines in Python scripts to be portable between Unix and From 78b23ab6827410960577c01a33df8afc8fd4496e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 1 May 2017 22:55:53 -0700 Subject: [PATCH 0269/1345] [3.6] Clean up some confusing text left by PROTOCOL_SSLv23 -> PROTOCOL_TLS transition (GH-1355) (#1371) (cherry picked from commit d4069de5112f0408801ff2479476827bb3e0e8fc) --- Doc/library/ssl.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index bbb13745b203e6..0ce73c14095480 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -610,13 +610,13 @@ Constants .. data:: PROTOCOL_TLS Selects the highest protocol version that both the client and server support. - Despite the name, this option can select "TLS" protocols as well as "SSL". + Despite the name, this option can select both "SSL" and "TLS" protocols. .. versionadded:: 3.6 .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 From 809101f14f27ddb394cd77c477470761ecf99f41 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 May 2017 10:47:37 +0200 Subject: [PATCH 0270/1345] bpo-30104: Use -fno-strict-aliasing on clang (#1376) Python/dtoa.c is not compiled correctly with clang 4.0 and optimization level -O2 or higher, because of an aliasing issue on the double/ULong[2] union. Only compile dtoa.c with -fno-strict-aliasing. LLVM bug report: https://bugs.llvm.org//show_bug.cgi?id=31928 --- Makefile.pre.in | 9 +++++++++ configure | 35 ++++++++++++++++++++++++++++------- configure.ac | 34 +++++++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8f27d7320c1d69..3eef424b303cfe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -107,6 +107,8 @@ ARFLAGS= @ARFLAGS@ CFLAGSFORSHARED=@CFLAGSFORSHARED@ # C flags used for building the interpreter object files PY_CORE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE +# Strict or non-strict aliasing flags used to compile dtoa.c, see above +CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Machine-dependent subdirectories @@ -1535,6 +1537,13 @@ config.status: $(srcdir)/configure .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +# bpo-30104: dtoa.c uses union to cast double to unsigned long[2]. clang 4.0 +# with -O2 or higher and strict aliasing miscompiles the ratio() function +# causing rounding issues. Compile dtoa.c using -fno-strict-aliasing on clang. +# https://bugs.llvm.org//show_bug.cgi?id=31928 +Python/dtoa.o: Python/dtoa.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< + # Run reindent on the library reindent: ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib diff --git a/configure b/configure index 6a1b9e4edc2c23..227d1d24b379db 100755 --- a/configure +++ b/configure @@ -668,6 +668,7 @@ OTHER_LIBTOOL_OPT UNIVERSAL_ARCH_FLAGS CFLAGS_NODIST BASECFLAGS +CFLAGS_ALIASING OPT LLVM_PROF_FOUND target_os @@ -6851,6 +6852,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line + if test "${OPT-unset}" = "unset" then case $GCC in @@ -6863,30 +6865,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; diff --git a/configure.ac b/configure.ac index 2c8e246f22f53e..6978b9090de9f6 100644 --- a/configure.ac +++ b/configure.ac @@ -1447,6 +1447,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST(OPT) +AC_SUBST(CFLAGS_ALIASING) if test "${OPT-unset}" = "unset" then case $GCC in @@ -1459,30 +1460,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; From 33a5d40de997fff5b60157c546a3bf711e083bcc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 May 2017 11:45:42 +0200 Subject: [PATCH 0271/1345] bpo-30108: Restore sys.path in test_site (#1197) (#1378) Add setUpModule() and tearDownModule() functions to test_site to save/restore sys.path at the module level to prevent warning if the user site directory is created, since site.addsitedir() modifies sys.path. (cherry picked from commit b85c136903c6d2368162f7c4a58f258c9c69ead0) --- Lib/test/test_site.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 342ec9e43d5de3..4029617aa1d9c1 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -27,14 +27,27 @@ import site -if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): - # need to add user site directory for tests - try: - os.makedirs(site.USER_SITE) - site.addsitedir(site.USER_SITE) - except PermissionError as exc: - raise unittest.SkipTest('unable to create user site directory (%r): %s' - % (site.USER_SITE, exc)) + +OLD_SYS_PATH = None + + +def setUpModule(): + global OLD_SYS_PATH + OLD_SYS_PATH = sys.path[:] + + if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): + # need to add user site directory for tests + try: + os.makedirs(site.USER_SITE) + # modify sys.path: will be restored by tearDownModule() + site.addsitedir(site.USER_SITE) + except PermissionError as exc: + raise unittest.SkipTest('unable to create user site directory (%r): %s' + % (site.USER_SITE, exc)) + + +def tearDownModule(): + sys.path[:] = OLD_SYS_PATH class HelperFunctionsTests(unittest.TestCase): From 30768958490c658fba0fe24f1cabbdad44be22ff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 May 2017 15:50:42 +0200 Subject: [PATCH 0272/1345] bpo-30132: distutils BuildExtTestCase use temp_cwd (#1387) BuildExtTestCase of test_distutils now uses support.temp_cwd() in setUp() to remove files created in the current working in all BuildExtTestCase unit tests, not only test_build_ext(). Fix the following warning: Warning -- files was modified by test_distutils Before: [] After: ['vc140.pdb'] --- Lib/distutils/tests/test_build_ext.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index be7f5f38aafda7..c6502d61d54d0e 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -37,6 +37,13 @@ def setUp(self): from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + # bpo-30132: On Windows, a .pdb file may be created in the current + # working directory. Create a temporary working directory to cleanup + # everything at the end of the test. + self.temp_cwd = support.temp_cwd() + self.temp_cwd.__enter__() + self.addCleanup(self.temp_cwd.__exit__, None, None, None) + def build_ext(self, *args, **kwargs): return build_ext(*args, **kwargs) From 3c422b92653df05c7034136b77f76376aa5aec4f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 May 2017 16:49:10 +0200 Subject: [PATCH 0273/1345] bpo-30199: test_ssl closes all asyncore channels (#1381) (#1390) AsyncoreEchoServer of test_ssl now calls asyncore.close_all(ignore_all=True) to ensure that asyncore.socket_map is cleared once the test completes, even if ConnectionHandler was not correctly unregistered. Fix the following warning: Warning -- asyncore.socket_map was modified by test_ssl Before: {} After: {6: } (cherry picked from commit 1dae7450c68bad498e57800387b24cb103c461fa) --- Lib/test/test_ssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d203cddbdfb5ee..85c59a618ce1e9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2065,7 +2065,7 @@ class AsyncoreEchoServer(threading.Thread): class EchoServer (asyncore.dispatcher): - class ConnectionHandler (asyncore.dispatcher_with_send): + class ConnectionHandler(asyncore.dispatcher_with_send): def __init__(self, conn, certfile): self.socket = test_wrap_socket(conn, server_side=True, @@ -2156,6 +2156,8 @@ def __exit__(self, *args): self.join() if support.verbose: sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) def start (self, flag=None): self.flag = flag From 4dae0d111dd7bb34ec730eea2327a3219acff211 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 May 2017 23:46:06 +0200 Subject: [PATCH 0274/1345] [3.6] bpo-30232: Support Git worktree in configure.ac (#1398) * bpo-30232: Support Git worktree in configure.ac (#1391) Don't test if .git/HEAD file exists, but only if the .git file (or directory) exists. (cherry picked from commit 5facdbb29169c2799c42f887cef4cd9d087b0167) * bpo-30232: Regenerate configure (#1396) Run autoconf. (cherry picked from commit 9ed34a89532763cf89f5e11fffb91ef7dee29fed) --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 227d1d24b379db..3c611d6afa2617 100755 --- a/configure +++ b/configure @@ -2711,7 +2711,7 @@ fi -if test -e $srcdir/.git/HEAD +if test -e $srcdir/.git then # Extract the first word of "git", so it can be a program name with args. set dummy git; ac_word=$2 diff --git a/configure.ac b/configure.ac index 6978b9090de9f6..eda075dc755844 100644 --- a/configure.ac +++ b/configure.ac @@ -29,7 +29,7 @@ AC_SUBST(GITVERSION) AC_SUBST(GITTAG) AC_SUBST(GITBRANCH) -if test -e $srcdir/.git/HEAD +if test -e $srcdir/.git then AC_CHECK_PROG(HAS_GIT, git, found, not-found) else From 0c2ff0898db2db9cd9c643dfadbff11761bacf5f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 3 May 2017 00:14:29 +0200 Subject: [PATCH 0275/1345] Backport bpo-30205 to 3.6 (#1403) --- Lib/test/test_socket.py | 4 ++++ Misc/NEWS | 2 ++ Modules/socketmodule.c | 6 +++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 2497e47c668bc3..80dfc405c71809 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -4660,6 +4660,10 @@ def bind(self, sock, path): else: raise + def testUnbound(self): + # Issue #30205 + self.assertIn(self.sock.getsockname(), ('', None)) + def testStrAddr(self): # Test binding to and retrieving a normal string pathname. path = os.path.abspath(support.TESTFN) diff --git a/Misc/NEWS b/Misc/NEWS index 936e2b02017f8e..3b615434692239 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,8 @@ Core and Builtins Library ------- +- bpo-30205: Fix getsockname() for unbound AF_UNIX sockets on Linux. + - bpo-30070: Fixed leaks and crashes in errors handling in the parser module. - bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f3654c97e76ce9..42aec59ca7f6e5 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1212,9 +1212,9 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) { struct sockaddr_un *a = (struct sockaddr_un *) addr; #ifdef __linux__ - if (a->sun_path[0] == 0) { /* Linux abstract namespace */ - addrlen -= offsetof(struct sockaddr_un, sun_path); - return PyBytes_FromStringAndSize(a->sun_path, addrlen); + size_t linuxaddrlen = addrlen - offsetof(struct sockaddr_un, sun_path); + if (linuxaddrlen > 0 && a->sun_path[0] == 0) { /* Linux abstract namespace */ + return PyBytes_FromStringAndSize(a->sun_path, linuxaddrlen); } else #endif /* linux */ From 564e89f43b8d64c165755e3c7a94e4daecfde5a3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 May 2017 02:12:22 +0200 Subject: [PATCH 0276/1345] regrtest: always show before/after of modified env (#1192) (#1406) Buildbots don't run tests with -vv and so only log "xxx was modified by test_xxx" which is not enough to debug such random issue. In many cases, I'm unable to reproduce the warning and so unable to fix it. Always logging the value before and value after should help to debug such warning on buildbots. (cherry picked from commit ec4b17239d899550be4ee6104b61751bb3c70382) --- Lib/test/libregrtest/save_env.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 96ad3af8df4cf7..8309f266bfbe9b 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -279,7 +279,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not self.quiet and not self.pgo: print(f"Warning -- {name} was modified by {self.testname}", file=sys.stderr, flush=True) - if self.verbose > 1: - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print(f" Before: {original}\n After: {current} ", + file=sys.stderr, flush=True) return False From 5bcf01d4cdc78f6b01e5017ff60da7dbfc34c36b Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 2 May 2017 21:35:43 -0700 Subject: [PATCH 0277/1345] [3.6] Fix typo in selectors.rst (GH-1383) (#1414) decriptor -> descriptor (cherry picked from commit b0d82036549074357717d130a772d1e2ebc8ea01) --- Doc/library/selectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 1624d88aaed38d..6d864a836de075 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -68,7 +68,7 @@ constants below: .. class:: SelectorKey A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file decriptor, selected event + associate a file object to its underlying file descriptor, selected event mask and attached data. It is returned by several :class:`BaseSelector` methods. From e612c28513b406779d187e5f816445c7d40d292b Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 3 May 2017 09:38:01 -0700 Subject: [PATCH 0278/1345] [3.6] bpo-28556: Routine updates to typing (GH-1366) (#1416) - Add NoReturn type - Use WrapperDescriptorType (original PR by Jim Fasarakis-Hilliard) - Minor bug-fixes (cherry picked from commit f06e0218ef6007667f5d61184b85a81a0466d3ae) --- Lib/test/test_typing.py | 51 +++++++++++++++++++++++++++++++++++++---- Lib/typing.py | 35 ++++++++++++++++++++++++---- Misc/NEWS | 4 ++++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f0070ec975791a..b3cabda394497e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6,7 +6,7 @@ from unittest import TestCase, main, skipUnless, SkipTest from copy import copy, deepcopy -from typing import Any +from typing import Any, NoReturn from typing import TypeVar, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional @@ -102,10 +102,6 @@ def test_cannot_instantiate(self): with self.assertRaises(TypeError): type(Any)() - def test_cannot_subscript(self): - with self.assertRaises(TypeError): - Any[int] - def test_any_works_with_alias(self): # These expressions must simply not fail. typing.Match[Any] @@ -113,6 +109,40 @@ def test_any_works_with_alias(self): typing.IO[Any] +class NoReturnTests(BaseTestCase): + + def test_noreturn_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, NoReturn) + + def test_noreturn_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, NoReturn) + with self.assertRaises(TypeError): + issubclass(NoReturn, Employee) + + def test_repr(self): + self.assertEqual(repr(NoReturn), 'typing.NoReturn') + + def test_not_generic(self): + with self.assertRaises(TypeError): + NoReturn[int] + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class A(NoReturn): + pass + with self.assertRaises(TypeError): + class A(type(NoReturn)): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + NoReturn() + with self.assertRaises(TypeError): + type(NoReturn)() + + class TypeVarTests(BaseTestCase): def test_basic_plain(self): @@ -2273,6 +2303,14 @@ def _fields(self): return 'no chance for this' """) + with self.assertRaises(AttributeError): + exec(""" +class XMethBad2(NamedTuple): + x: int + def _source(self): + return 'no chance for this as well' +""") + @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) @@ -2420,6 +2458,9 @@ def test_all(self): self.assertNotIn('sys', a) # Check that Text is defined. self.assertIn('Text', a) + # Check previously missing classes. + self.assertIn('SupportsBytes', a) + self.assertIn('SupportsComplex', a) if __name__ == '__main__': diff --git a/Lib/typing.py b/Lib/typing.py index 9a0f49099a3114..645bc6f8ae0edd 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -11,9 +11,9 @@ except ImportError: import collections as collections_abc # Fallback for PY3.2. try: - from types import SlotWrapperType, MethodWrapperType, MethodDescriptorType + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType except ImportError: - SlotWrapperType = type(object.__init__) + WrapperDescriptorType = type(object.__init__) MethodWrapperType = type(object().__str__) MethodDescriptorType = type(str.join) @@ -63,6 +63,8 @@ # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', 'SupportsFloat', 'SupportsInt', 'SupportsRound', @@ -420,6 +422,31 @@ def __subclasscheck__(self, cls): Any = _Any(_root=True) +class _NoReturn(_FinalTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("NoReturn cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("NoReturn cannot be used with issubclass().") + + +NoReturn = _NoReturn(_root=True) + + class TypeVar(_TypingBase, _root=True): """Type variable. @@ -1450,7 +1477,7 @@ def _get_defaults(func): _allowed_types = (types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.ModuleType, - SlotWrapperType, MethodWrapperType, MethodDescriptorType) + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) def get_type_hints(obj, globalns=None, localns=None): @@ -2051,7 +2078,7 @@ def _make_nmtuple(name, types): # attributes prohibited to set in NamedTuple class syntax _prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_field_types', - '_make', '_replace', '_asdict') + '_make', '_replace', '_asdict', '_source') _special = ('__module__', '__name__', '__qualname__', '__annotations__') diff --git a/Misc/NEWS b/Misc/NEWS index 3b615434692239..1c9026e77f2e76 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Core and Builtins Library ------- +- bpo-28556: Various updates to typing module: add typing.NoReturn type, use + WrapperDescriptorType, minor bug-fixes. Original PRs by + Jim Fasarakis-Hilliard and Ivan Levkivskyi. + - bpo-30205: Fix getsockname() for unbound AF_UNIX sockets on Linux. - bpo-30070: Fixed leaks and crashes in errors handling in the parser module. From 833dcb62594cc7a834f63ac8f1c312f2ac24d393 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 3 May 2017 11:56:44 -0500 Subject: [PATCH 0279/1345] Bump version in AppVeyor config (GH-1436) --- .github/appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/appveyor.yml b/.github/appveyor.yml index a369e52415bb17..a870e5b57d9250 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -1,4 +1,4 @@ -version: 3.6.1rc1+.{build} +version: 3.6.1+.{build} clone_depth: 5 build_script: - cmd: PCbuild\build.bat -e From af71364c3f0e9cd6cb9d83194315af0ee3c783d2 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 3 May 2017 18:41:20 -0700 Subject: [PATCH 0280/1345] [3.6] bpo-28315: Improve code examples in docs (GH-1372) (#1445) Replace File "", line 1, in ? with File "", line 1, in (cherry picked from commit 8856940cf2e82cb17db2b684cd5732fe658605ca) --- Doc/extending/newtypes.rst | 2 +- Doc/howto/functional.rst | 6 +++--- Doc/library/ctypes.rst | 30 +++++++++++++++--------------- Doc/library/doctest.rst | 6 +++--- Doc/library/fpectl.rst | 2 +- Doc/library/pdb.rst | 2 +- Doc/library/unicodedata.rst | 2 +- Doc/reference/expressions.rst | 2 +- Doc/tutorial/classes.rst | 2 +- Doc/tutorial/controlflow.rst | 2 +- Doc/tutorial/datastructures.rst | 2 +- Doc/tutorial/inputoutput.rst | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index b8ce4377877e70..003b4e505d3247 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -124,7 +124,7 @@ our objects and in some error messages, for example:: >>> "" + noddy.new_noddy() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: cannot add type "noddy.Noddy" to string Note that the name is a dotted name that includes both the module name and the diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 8ae9679894a578..a82dca7077e905 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -210,7 +210,7 @@ You can experiment with the iteration interface manually: 3 >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in StopIteration >>> @@ -474,7 +474,7 @@ Here's a sample usage of the ``generate_ints()`` generator: 2 >>> next(gen) Traceback (most recent call last): - File "stdin", line 1, in ? + File "stdin", line 1, in File "stdin", line 2, in generate_ints StopIteration @@ -577,7 +577,7 @@ And here's an example of changing the counter: 9 >>> next(it) #doctest: +SKIP Traceback (most recent call last): - File "t.py", line 15, in ? + File "t.py", line 15, in it.next() StopIteration diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 51a3a360fa3e1e..47bf89f59a5696 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -97,7 +97,7 @@ Functions are accessed as attributes of dll objects:: <_FuncPtr object at 0x...> >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 239, in __getattr__ func = _StdcallFuncPtr(name, self) AttributeError: function 'MyOwnFunction' not found @@ -135,7 +135,7 @@ functions can be accessed by indexing the dll object with the ordinal number:: <_FuncPtr object at 0x...> >>> cdll.kernel32[0] # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 310, in __getitem__ func = _StdcallFuncPtr(name, self) AttributeError: function ordinal 0 not found @@ -168,11 +168,11 @@ although an error is raised the function *has* been called:: >>> windll.kernel32.GetModuleHandleA() # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with not enough arguments (4 bytes missing) >>> windll.kernel32.GetModuleHandleA(0, 0) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with too many arguments (4 bytes in excess) >>> @@ -181,13 +181,13 @@ The same exception is raised when you call an ``stdcall`` function with the >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with not enough arguments (4 bytes missing) >>> >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with too many arguments (4 bytes in excess) >>> @@ -200,7 +200,7 @@ argument values:: >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in OSError: exception: access violation reading 0x00000020 >>> @@ -373,7 +373,7 @@ from within *IDLE* or *PythonWin*:: 19 >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 >>> @@ -436,7 +436,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: wrong type >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 @@ -486,7 +486,7 @@ single character Python bytes object into a C char:: 'def' >>> strchr(b"abcdef", b"def") Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: one character string expected >>> print(strchr(b"abcdef", b"x")) None @@ -512,7 +512,7 @@ useful to check for error return values and automatically raise an exception:: 486539264 >>> GetModuleHandle("something silly") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 3, in ValidHandle OSError: [Errno 126] The specified module could not be found. >>> @@ -583,7 +583,7 @@ Here is a simple example of a POINT structure, which contains two integers named 0 5 >>> POINT(1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: too many initializers >>> @@ -786,7 +786,7 @@ new type:: >>> PI(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: expected c_long instead of int >>> PI(c_int(42)) @@ -862,7 +862,7 @@ but not instances of other types:: >>> bar.values = (c_byte * 4)() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance >>> @@ -913,7 +913,7 @@ work:: ... ("next", POINTER(cell))] ... Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 2, in cell NameError: name 'cell' is not defined >>> diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 15b12f7aa786ea..587a0a09a94791 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -408,7 +408,7 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: list.remove(x): x not in list That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): @@ -432,7 +432,7 @@ multi-line detail:: >>> raise ValueError('multi\n line\ndetail') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: multi line detail @@ -591,7 +591,7 @@ doctest decides whether actual output matches an example's expected output: >>> (1, 2)[3] = 'moo' Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: object doesn't support item assignment passes under Python 2.3 and later Python versions with the flag specified, diff --git a/Doc/library/fpectl.rst b/Doc/library/fpectl.rst index e4b528cf0b0b6f..96607165ba4e3e 100644 --- a/Doc/library/fpectl.rst +++ b/Doc/library/fpectl.rst @@ -89,7 +89,7 @@ The following example demonstrates how to start up and test operation of the >>> import math >>> math.exp(1000) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in FloatingPointError: in math_1 diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7c37bb7d248c91..6225a3a1f07996 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,7 +76,7 @@ The typical usage to inspect a crashed program is:: >>> import mymodule >>> mymodule.test() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "./mymodule.py", line 4, in test test2() File "./mymodule.py", line 3, in test2 diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 643180953fd23d..2a9777609527d0 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -158,7 +158,7 @@ Examples: 9 >>> unicodedata.decimal('a') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: not a decimal >>> unicodedata.category('A') # 'L'etter, 'u'ppercase 'Lu' diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index c4f6c55c7ca3aa..43d2690cbba91b 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -905,7 +905,7 @@ keyword arguments (and any ``**expression`` arguments -- see below). So:: 2 1 >>> f(a=1, *(2,)) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: f() got multiple values for keyword argument 'a' >>> f(1, *(2,)) 1 2 diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index e134d5d62ea667..073444cf8b3985 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -784,7 +784,7 @@ using the :func:`next` built-in function; this example shows how it all works:: 'c' >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in next(it) StopIteration diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 6a9bb4889ff857..54171bc96f0f08 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -475,7 +475,7 @@ Here's an example that fails due to this restriction:: ... >>> function(0, a=0) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: function() got multiple values for keyword argument 'a' When a final formal parameter of the form ``**name`` is present, it receives a diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 6140ece046b975..1a73ac9d05936a 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -261,7 +261,7 @@ it must be parenthesized. :: [(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25)] >>> # the tuple must be parenthesized, otherwise an error is raised >>> [x, x**2 for x in range(6)] - File "", line 1, in ? + File "", line 1, in [x, x**2 for x in range(6)] ^ SyntaxError: invalid syntax diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index beeaac36b9bd24..bad0302b037861 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -362,7 +362,7 @@ attempts to use the file object will automatically fail. :: >>> f.close() >>> f.read() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: I/O operation on closed file It is good practice to use the :keyword:`with` keyword when dealing with file From 1bebd8a21948652a5f941c550dec06ea84d1d979 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 4 May 2017 06:50:28 +0300 Subject: [PATCH 0281/1345] [3.6] bpo-30184: Add tests for invalid use of PyArg_ParseTupleAndKeywords. (GH-1316). (#1441) (cherry picked from commit 5f161fd86dd5bb936a1a2a13391b13b7e59ec201) --- Lib/test/test_capi.py | 59 +++++++++++++++++++++++++++------------ Modules/_testcapimodule.c | 4 +-- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 08aa3ffdf6b4f1..216851c2d36c8b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -490,9 +490,8 @@ def test_skipitem(self): # test the format unit when not skipped format = c + "i" try: - # (note: the format string must be bytes!) _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, - format.encode("ascii"), keywords) + format, keywords) when_not_skipped = False except SystemError as e: s = "argument 1 (impossible)" @@ -504,7 +503,7 @@ def test_skipitem(self): optional_format = "|" + format try: _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, - optional_format.encode("ascii"), keywords) + optional_format, keywords) when_skipped = False except SystemError as e: s = "impossible: '{}'".format(format) @@ -517,40 +516,64 @@ def test_skipitem(self): self.assertIs(when_skipped, when_not_skipped, message) def test_parse_tuple_and_keywords(self): - # parse_tuple_and_keywords error handling tests + # Test handling errors in the parse_tuple_and_keywords helper itself self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, (), {}, 42, []) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', 42) + (), {}, '', 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [''] * 42) + (), {}, '', [''] * 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [42]) + (), {}, '', [42]) + + def test_bad_use(self): + # Test handling invalid format and keywords in + # PyArg_ParseTupleAndKeywords() + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '||O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1, 2), {}, '|O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$$O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$|O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|OO', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|$O', ['']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|OO', ['a', '']) def test_positional_only(self): parse = _testcapi.parse_tuple_and_keywords - parse((1, 2, 3), {}, b'OOO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OOO', ['', '', 'a']) + parse((1, 2, 3), {}, 'OOO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OOO', ['', '', 'a']) - parse((1,), {}, b'O|OO', ['', '', 'a']) + parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) + parse((1,), {}, 'O|OO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|OO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OO$O', ['', '', 'a']) + parse((), {}, 'O|OO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes exactly 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OO$O', ['', '', 'a']) - parse((1,), {}, b'O|O$O', ['', '', 'a']) + parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) + parse((1,), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|O$O', ['', '', 'a']) + parse((), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): - parse((1,), {}, b'O|$OO', ['', '', 'a']) + parse((1,), {}, 'O|$OO', ['', '', 'a']) with self.assertRaisesRegex(SystemError, 'Empty keyword'): - parse((1,), {}, b'O|OO', ['', 'a', '']) + parse((1,), {}, 'O|OO', ['', 'a', '']) @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b25f4434ffa35c..c76eefab4e80cf 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1571,7 +1571,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) { PyObject *sub_args; PyObject *sub_kwargs; - char *sub_format; + const char *sub_format; PyObject *sub_keywords; Py_ssize_t i, size; @@ -1584,7 +1584,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) double buffers[8][4]; /* double ensures alignment where necessary */ - if (!PyArg_ParseTuple(args, "OOyO:parse_tuple_and_keywords", + if (!PyArg_ParseTuple(args, "OOsO:parse_tuple_and_keywords", &sub_args, &sub_kwargs, &sub_format, &sub_keywords)) return NULL; From 31906b42fd71c6baaf80429005c720b40dfe8fae Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 4 May 2017 13:21:10 +0200 Subject: [PATCH 0282/1345] bpo-30225: Fix is_valid_fd() on macOS Tiger (#1443) (#1449) is_valid_fd() now uses fstat() instead of dup() on macOS to return 0 on a pipe when the other side of the pipe is closed. fstat() fails with EBADF in that case, whereas dup() succeed. (cherry picked from commit 1c4670ea0cc3d208121af11b9b973e6bb268e570) --- Python/pylifecycle.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index af96d6d5fcb63c..640271fd20b709 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1044,6 +1044,14 @@ initsite(void) static int is_valid_fd(int fd) { +#ifdef __APPLE__ + /* bpo-30225: On macOS Tiger, when stdout is redirected to a pipe + and the other side of the pipe is closed, dup(1) succeed, whereas + fstat(1, &st) fails with EBADF. Prefer fstat() over dup() to detect + such error. */ + struct stat st; + return (fstat(fd, &st) == 0); +#else int fd2; if (fd < 0) return 0; @@ -1056,6 +1064,7 @@ is_valid_fd(int fd) close(fd2); _Py_END_SUPPRESS_IPH return fd2 >= 0; +#endif } /* returns Py_None if the fd is not valid */ From a7e48b544becc01dbf9059ed01f38997c6d0156c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 4 May 2017 17:12:35 +0200 Subject: [PATCH 0283/1345] [3.6] bpo-30185: avoid KeyboardInterrupt tracebacks in forkserver (GH-1319) (#1454) * bpo-30185: avoid KeyboardInterrupt tracebacks in forkserver * Tweak comment. (cherry picked from commit 6dd4d734ed207ba16b017e38f8909de7ef187e29) --- Lib/multiprocessing/forkserver.py | 20 ++++++++++++++------ Misc/NEWS | 3 +++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index f2c179e4e0afaa..d5ce6257456766 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -149,8 +149,15 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): util._close_stdin() - # ignoring SIGCHLD means no need to reap zombie processes - handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) + # ignoring SIGCHLD means no need to reap zombie processes; + # letting SIGINT through avoids KeyboardInterrupt tracebacks + handlers = { + signal.SIGCHLD: signal.SIG_IGN, + signal.SIGINT: signal.SIG_DFL, + } + old_handlers = {sig: signal.signal(sig, val) + for (sig, val) in handlers.items()} + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() @@ -175,7 +182,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): code = 1 if os.fork() == 0: try: - _serve_one(s, listener, alive_r, handler) + _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() @@ -186,11 +193,12 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): if e.errno != errno.ECONNABORTED: raise -def _serve_one(s, listener, alive_r, handler): - # close unnecessary stuff and reset SIGCHLD handler +def _serve_one(s, listener, alive_r, handlers): + # close unnecessary stuff and reset signal handlers listener.close() os.close(alive_r) - signal.signal(signal.SIGCHLD, handler) + for sig, val in handlers.items(): + signal.signal(sig, val) # receive fds from parent process fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) diff --git a/Misc/NEWS b/Misc/NEWS index 1c9026e77f2e76..f0d5a3ba2073c8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Core and Builtins Library ------- +- bpo-30185: Avoid KeyboardInterrupt tracebacks in forkserver helper process + when Ctrl-C is received. + - bpo-28556: Various updates to typing module: add typing.NoReturn type, use WrapperDescriptorType, minor bug-fixes. Original PRs by Jim Fasarakis-Hilliard and Ivan Levkivskyi. From 9d02f562961efd12d3c8317a10916db7f77330cc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 5 May 2017 00:46:56 +0200 Subject: [PATCH 0284/1345] [3.6] bpo-23404: make touch becomes make regen-all (#1405) (#1461) * bpo-23404: make touch becomes make regen-all (#1405) Don't rebuild generated files based on file modification time anymore, the action is now explicit. Replace "make touch" with "make regen-all". Changes: * Remove "make touch", Tools/hg/hgtouch.py and .hgtouch * Add a new "make regen-all" command to rebuild all generated files * Add subcommands to only generate specific files: - regen-ast: Include/Python-ast.h and Python/Python-ast.c - regen-grammar: Include/graminit.h and Python/graminit.c - regen-importlib: Python/importlib_external.h and Python/importlib.h - regen-opcode: Include/opcode.h - regen-opcode-targets: Python/opcode_targets.h - regen-typeslots: Objects/typeslots.inc * Rename PYTHON_FOR_GEN to PYTHON_FOR_REGEN * pgen is now only built by by "make regen-grammar" * Add $(srcdir)/ prefix to paths to source files to handle correctly compilation outside the source directory Note: $(PYTHON_FOR_REGEN) is no more used nor needed by "make" default target building Python. (cherry picked from commit a5c62a8e9f0de6c4133825a5710984a3cd5e102b) * bpo-30273: Update sysconfig (#1464) The AST_H_DIR variable was removed from Makefile.pre.in by the commit a5c62a8e9f0de6c4133825a5710984a3cd5e102b (bpo-23404). AST_H_DIR was hardcoded to "Include", so replace the removed variable by its content. Remove also ASDLGEN variable from sysconfig example since this variable was also removed. (cherry picked from commit b109a1d3360fc4bb87b9887264e3634632d392ca) --- .hgtouch | 17 ---- Doc/library/sysconfig.rst | 1 - Lib/distutils/sysconfig.py | 2 +- Mac/BuildScript/build-installer.py | 4 - Makefile.pre.in | 149 ++++++++++++++--------------- Misc/NEWS | 6 +- Tools/hg/hgtouch.py | 129 ------------------------- configure | 32 +++---- configure.ac | 13 +-- 9 files changed, 91 insertions(+), 262 deletions(-) delete mode 100644 .hgtouch delete mode 100644 Tools/hg/hgtouch.py diff --git a/.hgtouch b/.hgtouch deleted file mode 100644 index b9be0f11fdb829..00000000000000 --- a/.hgtouch +++ /dev/null @@ -1,17 +0,0 @@ -# -*- Makefile -*- -# Define dependencies of generated files that are checked into hg. -# The syntax of this file uses make rule dependencies, without actions - -Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c - -Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py - -Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py -Python/Python-ast.c: Include/Python-ast.h - -Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py - -Objects/typeslots.inc: Include/typeslots.h Objects/typeslots.py - -Include/graminit.h: Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c -Python/graminit.c: Include/graminit.h Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 08b74a9affd24f..f066a765d00ec7 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -255,7 +255,6 @@ You can use :mod:`sysconfig` as a script with Python's *-m* option: AIX_GENUINE_CPLUSPLUS = "0" AR = "ar" ARFLAGS = "rc" - ASDLGEN = "./Parser/asdl_c.py" ... This call will print in the standard output the information returned by diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 8bf1a7016bdf42..90004acea88761 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -97,7 +97,7 @@ def get_python_inc(plat_specific=0, prefix=None): if plat_specific: return base if _sys_home: - incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR')) + incdir = os.path.join(_sys_home, 'Include') else: incdir = os.path.join(get_config_var('srcdir'), 'Include') return os.path.normpath(incdir) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index f59ae34df2549a..7b4376f42b88f3 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1142,10 +1142,6 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - # bpo-29550: avoid using make touch until it is fixed for git - # print("Running make touch") - # runCommand("make touch") - print("Running make") runCommand("make") diff --git a/Makefile.pre.in b/Makefile.pre.in index 3eef424b303cfe..6e3a82f95df06d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -229,7 +229,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ +PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -273,11 +273,6 @@ IO_OBJS= \ Modules/_io/stringio.o ########################################################################## -# Grammar -GRAMMAR_H= Include/graminit.h -GRAMMAR_C= Python/graminit.c -GRAMMAR_INPUT= $(srcdir)/Grammar/Grammar - LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ @@ -316,38 +311,9 @@ PARSER_HEADERS= \ PGENOBJS= $(POBJS) $(PGOBJS) -########################################################################## -# opcode.h generation -OPCODE_H_DIR= $(srcdir)/Include -OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py -OPCODE_H= $(OPCODE_H_DIR)/opcode.h -OPCODE_H_GEN= $(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H) - -########################################################################## -# AST -AST_H_DIR= Include -AST_H= $(AST_H_DIR)/Python-ast.h -AST_C_DIR= Python -AST_C= $(AST_C_DIR)/Python-ast.c -AST_ASDL= $(srcdir)/Parser/Python.asdl - -ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py -# Note that a build now requires Python to exist before the build starts. -# Use "hg touch" to fix up screwed up file mtimes in a checkout. -ASDLGEN= $(PYTHON_FOR_GEN) $(srcdir)/Parser/asdl_c.py - ########################################################################## # Python -OPCODETARGETS_H= \ - Python/opcode_targets.h - -OPCODETARGETGEN= \ - $(srcdir)/Python/makeopcodetargets.py - -OPCODETARGETGEN_FILES= \ - $(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py - PYTHON_OBJS= \ Python/_warnings.o \ Python/Python-ast.o \ @@ -547,7 +513,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -coverage-report: +# Force regeneration of parser and importlib +coverage-report: regen-grammar regen-importlib : # force rebuilding of parser and importlib @touch $(GRAMMAR_INPUT) @touch $(srcdir)/Lib/importlib/_bootstrap.py @@ -723,14 +690,24 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -Python/importlib_external.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib Python/marshal.c +.PHONY: regen-importlib +regen-importlib: Programs/_freeze_importlib + # Regenerate Python/importlib_external.h + # from Lib/importlib/_bootstrap_external.py using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h - -Python/importlib.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib Python/marshal.c + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Python/importlib_external.h + # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py + # using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Python/importlib.h + + +############################################################################ +# Regenerate all generated files +regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib ############################################################################ # Special rules for object files @@ -789,15 +766,18 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(IO_OBJS): $(IO_H) -$(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) - @$(MKDIR_P) Include - $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) -$(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) - touch $(GRAMMAR_C) - $(PGEN): $(PGENOBJS) $(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) +.PHONY: regen-grammar +regen-grammar: $(PGEN) + # Regenerate Include/graminit.h and Python/graminit.c + # from Grammar/Grammar using pgen + @$(MKDIR_P) Include + $(PGEN) $(srcdir)/Grammar/Grammar \ + $(srcdir)/Include/graminit.h \ + $(srcdir)/Python/graminit.c + Parser/grammar.o: $(srcdir)/Parser/grammar.c \ $(srcdir)/Include/token.h \ $(srcdir)/Include/grammar.h @@ -809,18 +789,28 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c Parser/pgenmain.o: $(srcdir)/Include/parsetok.h -$(AST_H): $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_H_DIR) - $(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL) - -$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_C_DIR) - $(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL) - -$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT) - $(OPCODE_H_GEN) - -Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H) +.PHONY=regen-ast +regen-ast: + # Regenerate Include/Python-ast.h using Parser/asdl_c.py -h + $(MKDIR_P) $(srcdir)/Include + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -h $(srcdir)/Include \ + $(srcdir)/Parser/Python.asdl + # Regenerate Python/Python-ast.c using Parser/asdl_c.py -c + $(MKDIR_P) $(srcdir)/Python + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -c $(srcdir)/Python \ + $(srcdir)/Parser/Python.asdl + +.PHONY: regen-opcode +regen-opcode: + # Regenerate Include/opcode.h from Lib/opcode.py + # using Tools/scripts/generate_opcode_h.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \ + $(srcdir)/Lib/opcode.py \ + $(srcdir)/Include/opcode.h + +Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -870,12 +860,16 @@ Objects/odictobject.o: $(srcdir)/Objects/dict-common.h Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) - $(PYTHON_FOR_GEN) $(OPCODETARGETGEN) $(OPCODETARGETS_H) +.PHONY: regen-opcode-targets +regen-opcode-targets: + # Regenerate Python/opcode_targets.h from Lib/opcode.py + # using Python/makeopcodetargets.py + $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ + $(srcdir)/Python/opcode_targets.h -Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h -Python/frozen.o: Python/importlib.h Python/importlib_external.h +Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate @@ -890,8 +884,14 @@ Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc -Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py - $(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc + +.PHONY: regen-typeslots +regen-typeslots: + # Regenerate Objects/typeslots.inc from Include/typeslotsh + # using Objects/typeslots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ + < $(srcdir)/Include/typeslots.h \ + $(srcdir)/Objects/typeslots.inc ############################################################################ # Header files @@ -944,7 +944,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/node.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ - $(OPCODE_H) \ + $(srcdir)/Include/opcode.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -987,7 +987,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/weakrefobject.h \ pyconfig.h \ $(PARSER_HEADERS) \ - $(AST_H) \ + $(srcdir)/Include/Python-ast.h \ $(DTRACE_HEADERS) $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -1554,9 +1554,12 @@ recheck: $(SHELL) config.status --recheck $(SHELL) config.status -# Rebuild the configure script from configure.ac; also rebuild pyconfig.h.in +# Regenerate configure and pyconfig.h.in +.PHONY: autoconf autoconf: + # Regenerate the configure script from configure.ac using autoconf (cd $(srcdir); autoconf -Wall) + # Regenerate pyconfig.h.in from configure.ac using autoheader (cd $(srcdir); autoheader -Wall) # Create a tags file for vi @@ -1573,14 +1576,6 @@ TAGS:: etags Include/*.h; \ for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done -# This fixes up the mtimes of checked-in generated files, assuming that they -# only *appear* to be outdated because of checkout order. -# This is run while preparing a source release tarball, and can be run manually -# to avoid bootstrap issues. -touch: - cd $(srcdir); \ - hg --config extensions.touch=Tools/hg/hgtouch.py touch -v - # Sanitation targets -- clean leaves libraries, executables and tags # files, which clobber removes as well pycremoval: @@ -1698,7 +1693,7 @@ Python/thread.o: @THREADHEADERS@ .PHONY: maninstall libinstall inclinstall libainstall sharedinstall .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools -.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean +.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean .PHONY: smelly funny patchcheck touch altmaninstall commoninstall .PHONY: gdbhooks diff --git a/Misc/NEWS b/Misc/NEWS index f0d5a3ba2073c8..6913bfc8b722f8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,7 +10,7 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- -- bpo-12414: sys.getsizeof() on a code object now returns the sizes +- bpo-12414: sys.getsizeof() on a code object now returns the sizes which includes the code struct and sizes of objects which it references. Patch by Dong-hee Na. @@ -116,6 +116,10 @@ Library Build ----- +- bpo-23404: Don't regenerate generated files based on file modification time + anymore: the action is now explicit. Replace ``make touch`` with + ``make regen-all``. + - bpo-29643: Fix ``--enable-optimization`` didn't work. Documentation diff --git a/Tools/hg/hgtouch.py b/Tools/hg/hgtouch.py deleted file mode 100644 index fbca469ba94c25..00000000000000 --- a/Tools/hg/hgtouch.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Bring time stamps of generated checked-in files into the right order - -A versioned configuration file .hgtouch specifies generated files, in the -syntax of make rules. - - output: input1 input2 - -In addition to the dependency syntax, #-comments are supported. -""" -import errno -import os -import time - -def parse_config(repo): - try: - fp = repo.wfile(".hgtouch") - except IOError, e: - if e.errno != errno.ENOENT: - raise - return {} - result = {} - with fp: - for line in fp: - # strip comments - line = line.split('#')[0].strip() - if ':' not in line: - continue - outputs, inputs = line.split(':', 1) - outputs = outputs.split() - inputs = inputs.split() - for o in outputs: - try: - result[o].extend(inputs) - except KeyError: - result[o] = inputs - return result - -def check_rule(ui, repo, modified, basedir, output, inputs): - """Verify that the output is newer than any of the inputs. - Return (status, stamp), where status is True if the update succeeded, - and stamp is the newest time stamp assigned to any file (might be in - the future). - - If basedir is nonempty, it gives a directory in which the tree is to - be checked. - """ - f_output = repo.wjoin(os.path.join(basedir, output)) - try: - o_time = os.stat(f_output).st_mtime - except OSError: - ui.warn("Generated file %s does not exist\n" % output) - return False, 0 - youngest = 0 # youngest dependency - backdate = None - backdate_source = None - for i in inputs: - f_i = repo.wjoin(os.path.join(basedir, i)) - try: - i_time = os.stat(f_i).st_mtime - except OSError: - ui.warn(".hgtouch input file %s does not exist\n" % i) - return False, 0 - if i in modified: - # input is modified. Need to backdate at least to i_time - if backdate is None or backdate > i_time: - backdate = i_time - backdate_source = i - continue - youngest = max(i_time, youngest) - if backdate is not None: - ui.warn("Input %s for file %s locally modified\n" % (backdate_source, output)) - # set to 1s before oldest modified input - backdate -= 1 - os.utime(f_output, (backdate, backdate)) - return False, 0 - if youngest >= o_time: - ui.note("Touching %s\n" % output) - youngest += 1 - os.utime(f_output, (youngest, youngest)) - return True, youngest - else: - # Nothing to update - return True, 0 - -def do_touch(ui, repo, basedir): - if basedir: - if not os.path.isdir(repo.wjoin(basedir)): - ui.warn("Abort: basedir %r does not exist\n" % basedir) - return - modified = [] - else: - modified = repo.status()[0] - dependencies = parse_config(repo) - success = True - tstamp = 0 # newest time stamp assigned - # try processing all rules in topological order - hold_back = {} - while dependencies: - output, inputs = dependencies.popitem() - # check whether any of the inputs is generated - for i in inputs: - if i in dependencies: - hold_back[output] = inputs - continue - _success, _tstamp = check_rule(ui, repo, modified, basedir, output, inputs) - success = success and _success - tstamp = max(tstamp, _tstamp) - # put back held back rules - dependencies.update(hold_back) - hold_back = {} - now = time.time() - if tstamp > now: - # wait until real time has passed the newest time stamp, to - # avoid having files dated in the future - time.sleep(tstamp-now) - if hold_back: - ui.warn("Cyclic dependency involving %s\n" % (' '.join(hold_back.keys()))) - return False - return success - -def touch(ui, repo, basedir): - "touch generated files that are older than their sources after an update." - do_touch(ui, repo, basedir) - -cmdtable = { - "touch": (touch, - [('b', 'basedir', '', 'base dir of the tree to apply touching')], - "hg touch [-b BASEDIR]") -} diff --git a/configure b/configure index 3c611d6afa2617..97d3301f8e1891 100755 --- a/configure +++ b/configure @@ -750,9 +750,8 @@ UNIVERSALSDK CONFIG_ARGS SOVERSION VERSION -GENERATED_COMMENT PYTHON_FOR_BUILD -PYTHON_FOR_GEN +PYTHON_FOR_REGEN host_os host_vendor host_cpu @@ -2881,11 +2880,11 @@ do set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_PYTHON_FOR_GEN+:} false; then : +if ${ac_cv_prog_PYTHON_FOR_REGEN+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$PYTHON_FOR_GEN"; then - ac_cv_prog_PYTHON_FOR_GEN="$PYTHON_FOR_GEN" # Let the user override the test. + if test -n "$PYTHON_FOR_REGEN"; then + ac_cv_prog_PYTHON_FOR_REGEN="$PYTHON_FOR_REGEN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2894,7 +2893,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_PYTHON_FOR_GEN="$ac_prog" + ac_cv_prog_PYTHON_FOR_REGEN="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2904,25 +2903,20 @@ IFS=$as_save_IFS fi fi -PYTHON_FOR_GEN=$ac_cv_prog_PYTHON_FOR_GEN -if test -n "$PYTHON_FOR_GEN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_GEN" >&5 -$as_echo "$PYTHON_FOR_GEN" >&6; } +PYTHON_FOR_REGEN=$ac_cv_prog_PYTHON_FOR_REGEN +if test -n "$PYTHON_FOR_REGEN"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_REGEN" >&5 +$as_echo "$PYTHON_FOR_REGEN" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi - test -n "$PYTHON_FOR_GEN" && break + test -n "$PYTHON_FOR_REGEN" && break done -test -n "$PYTHON_FOR_GEN" || PYTHON_FOR_GEN="not-found" +test -n "$PYTHON_FOR_REGEN" || PYTHON_FOR_REGEN="python3" -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi if test "$cross_compiling" = yes; then @@ -2943,18 +2937,14 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } $as_echo "$interp" >&6; } PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi - if test "$prefix" != "/"; then prefix=`echo "$prefix" | sed -e 's/\/$//g'` fi diff --git a/configure.ac b/configure.ac index eda075dc755844..8abf2f96cc45ff 100644 --- a/configure.ac +++ b/configure.ac @@ -56,13 +56,8 @@ AC_SUBST(host) # pybuilddir.txt will be created by --generate-posix-vars in the Makefile rm -f pybuilddir.txt -AC_CHECK_PROGS(PYTHON_FOR_GEN, python$PACKAGE_VERSION python3 python, not-found) -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi -AC_SUBST(PYTHON_FOR_GEN) +AC_CHECK_PROGS(PYTHON_FOR_REGEN, python$PACKAGE_VERSION python3 python, python3) +AC_SUBST(PYTHON_FOR_REGEN) if test "$cross_compiling" = yes; then AC_MSG_CHECKING([for python interpreter for cross build]) @@ -80,16 +75,12 @@ if test "$cross_compiling" = yes; then AC_MSG_RESULT($interp) PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi AC_SUBST(PYTHON_FOR_BUILD) -AC_SUBST(GENERATED_COMMENT) dnl Ensure that if prefix is specified, it does not end in a slash. If dnl it does, we get path names containing '//' which is both ugly and From 39b73dd5131ce205dcee3b9e24ba0fc28934d79c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 May 2017 10:40:30 +0300 Subject: [PATCH 0285/1345] [3.6] bpo-30243: Fixed the possibility of a crash in _json. (GH-1420) (#1469) It was possible to get a core dump by using uninitialized _json objects. Now __new__ methods create initialized objects. __init__ methods are removed.. (cherry picked from commit 76a3e51a403bc84ed536921866c86dd7d07aaa7e) --- Misc/NEWS | 4 +++ Modules/_json.c | 86 ++++++++++++------------------------------------- 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 6913bfc8b722f8..0a2e5eb3977561 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Core and Builtins Library ------- +- bpo-30243: Removed the __init__ methods of _json's scanner and encoder. + Misusing them could cause memory leaks or crashes. Now scanner and encoder + objects are completely initialized in the __new__ methods. + - bpo-30185: Avoid KeyboardInterrupt tracebacks in forkserver helper process when Ctrl-C is received. diff --git a/Modules/_json.c b/Modules/_json.c index faa213491b6387..a84b0851090779 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -89,16 +89,12 @@ static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); static void scanner_dealloc(PyObject *self); static int scanner_clear(PyObject *self); static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); static void encoder_dealloc(PyObject *self); static int @@ -1203,38 +1199,21 @@ static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->strict = NULL; - s->object_hook = NULL; - s->object_pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ PyObject *ctx; static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; + return NULL; - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s == NULL) { + return NULL; } + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + /* All of these will fail "gracefully" so we don't need to verify them */ s->strict = PyObject_GetAttrString(ctx, "strict"); if (s->strict == NULL) @@ -1255,16 +1234,11 @@ scanner_init(PyObject *self, PyObject *args, PyObject *kwds) if (s->parse_constant == NULL) goto bail; - return 0; + return (PyObject *)s; bail: - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->object_pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; + Py_DECREF(s); + return NULL; } PyDoc_STRVAR(scanner_doc, "JSON scanner object"); @@ -1306,7 +1280,7 @@ PyTypeObject PyScannerType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ + 0, /* tp_init */ 0,/* PyType_GenericAlloc, */ /* tp_alloc */ scanner_new, /* tp_new */ 0,/* PyObject_GC_Del, */ /* tp_free */ @@ -1315,25 +1289,6 @@ PyTypeObject PyScannerType = { static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; @@ -1341,22 +1296,23 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) PyObject *item_separator, *sort_keys, *skipkeys; int allow_nan; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUOOp:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) - return -1; + return NULL; if (markers != Py_None && !PyDict_Check(markers)) { PyErr_Format(PyExc_TypeError, "make_encoder() argument 1 must be dict or None, " "not %.200s", Py_TYPE(markers)->tp_name); - return -1; + return NULL; } + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s == NULL) + return NULL; + s->markers = markers; s->defaultfn = defaultfn; s->encoder = encoder; @@ -1383,7 +1339,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) Py_INCREF(s->item_separator); Py_INCREF(s->sort_keys); Py_INCREF(s->skipkeys); - return 0; + return (PyObject *)s; } static PyObject * @@ -1914,7 +1870,7 @@ PyTypeObject PyEncoderType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ + 0, /* tp_init */ 0, /* tp_alloc */ encoder_new, /* tp_new */ 0, /* tp_free */ @@ -1957,10 +1913,8 @@ PyInit__json(void) PyObject *m = PyModule_Create(&jsonmodule); if (!m) return NULL; - PyScannerType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyScannerType) < 0) goto fail; - PyEncoderType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyEncoderType) < 0) goto fail; Py_INCREF((PyObject*)&PyScannerType); From 0fe870f3f95ba883b2b06bc0d814bdab8d53df98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 5 May 2017 10:04:57 +0200 Subject: [PATCH 0286/1345] bpo-30264: ExpatParser closes the source on error (#1451) (#1474) ExpatParser.parse() of xml.sax.xmlreader now always closes the source: close the file object or the urllib object if source is a string (not an open file-like object). The change fixes a ResourceWarning on parsing error. Add test_parse_close_source() unit test. (cherry picked from commit ef9c0e732fc50aefbdd7c5a80e04e14b31684e66) --- Lib/test/test_sax.py | 24 ++++++++++++++++++------ Lib/xml/sax/expatreader.py | 33 ++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 2411895d9d12b1..2eb62905ffa882 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -4,6 +4,7 @@ from xml.sax import make_parser, ContentHandler, \ SAXException, SAXReaderNotAvailable, SAXParseException import unittest +from unittest import mock try: make_parser() except SAXReaderNotAvailable: @@ -175,12 +176,8 @@ def test_parse_bytes(self): with self.assertRaises(SAXException): self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1', None))) make_xml_file(self.data, 'iso-8859-1', None) - with support.check_warnings(('unclosed file', ResourceWarning)): - # XXX Failed parser leaks an opened file. - with self.assertRaises(SAXException): - self.check_parse(TESTFN) - # Collect leaked file. - gc.collect() + with self.assertRaises(SAXException): + self.check_parse(TESTFN) with open(TESTFN, 'rb') as f: with self.assertRaises(SAXException): self.check_parse(f) @@ -194,6 +191,21 @@ def test_parse_InputSource(self): input.setEncoding('iso-8859-1') self.check_parse(input) + def test_parse_close_source(self): + builtin_open = open + fileobj = None + + def mock_open(*args): + nonlocal fileobj + fileobj = builtin_open(*args) + return fileobj + + with mock.patch('xml.sax.saxutils.open', side_effect=mock_open): + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + self.assertTrue(fileobj.closed) + def check_parseString(self, s): from xml.sax import parseString result = StringIO() diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 98b5ca953983c7..421358fa5bc7f0 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -105,9 +105,16 @@ def parse(self, source): source = saxutils.prepare_input_source(source) self._source = source - self.reset() - self._cont_handler.setDocumentLocator(ExpatLocator(self)) - xmlreader.IncrementalParser.parse(self, source) + try: + self.reset() + self._cont_handler.setDocumentLocator(ExpatLocator(self)) + xmlreader.IncrementalParser.parse(self, source) + except: + # bpo-30264: Close the source on error to not leak resources: + # xml.sax.parse() doesn't give access to the underlying parser + # to the caller + self._close_source() + raise def prepareParser(self, source): if source.getSystemId() is not None: @@ -213,6 +220,17 @@ def feed(self, data, isFinal = 0): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def _close_source(self): + source = self._source + try: + file = source.getCharacterStream() + if file is not None: + file.close() + finally: + file = source.getByteStream() + if file is not None: + file.close() + def close(self): if (self._entity_stack or self._parser is None or isinstance(self._parser, _ClosedParser)): @@ -232,14 +250,7 @@ def close(self): parser.ErrorColumnNumber = self._parser.ErrorColumnNumber parser.ErrorLineNumber = self._parser.ErrorLineNumber self._parser = parser - try: - file = self._source.getCharacterStream() - if file is not None: - file.close() - finally: - file = self._source.getByteStream() - if file is not None: - file.close() + self._close_source() def _reset_cont_handler(self): self._parser.ProcessingInstructionHandler = \ From 70dc6a7a0b7f104d87512556fca242c2ca96a010 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 6 May 2017 15:10:50 +0300 Subject: [PATCH 0287/1345] [3.6] Revert bpo-26293 for zipfile breakage. See also bpo-29094. (GH-1484). (#1485) (cherry picked from commit 3763ea865cee5bbabcce11cd577811135e0fc747) --- Lib/zipfile.py | 27 +++++++++++++-------------- Misc/NEWS | 2 ++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 06eedec340bc17..9164f8ab086a16 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1110,7 +1110,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self._start_disk = 0 try: self.start_dir = self.fp.tell() except (AttributeError, OSError): @@ -1136,7 +1135,7 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self.start_dir = self._start_disk = self.fp.tell() + self.start_dir = self.fp.tell() else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1180,18 +1179,17 @@ def _RealGetContents(self): offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd if endrec[_ECD_SIGNATURE] == stringEndArchive64: # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: - inferred = self._start_disk + offset_cd - print("given, inferred, offset", offset_cd, inferred, self._start_disk) + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk + self.start_dir = offset_cd + concat fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) @@ -1231,7 +1229,7 @@ def _RealGetContents(self): t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk + x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1702,10 +1700,11 @@ def _write_end_record(self): file_size = zinfo.file_size compress_size = zinfo.compress_size - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset extra_data = zinfo.extra min_version = 0 @@ -1752,7 +1751,7 @@ def _write_end_record(self): # Write end-of-zip-archive record centDirCount = len(self.filelist) centDirSize = pos2 - self.start_dir - centDirOffset = self.start_dir - self._start_disk + centDirOffset = self.start_dir requires_zip64 = None if centDirCount > ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" diff --git a/Misc/NEWS b/Misc/NEWS index 0a2e5eb3977561..6d6bdee08885c6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,8 @@ Core and Builtins Library ------- +- Revert bpo-26293 for zipfile breakage. See also bpo-29094. + - bpo-30243: Removed the __init__ methods of _json's scanner and encoder. Misusing them could cause memory leaks or crashes. Now scanner and encoder objects are completely initialized in the __new__ methods. From 410d75ab71e2db0fdc8b2016600a3d3d4830f38e Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 9 May 2017 10:56:30 +0800 Subject: [PATCH 0288/1345] bpo-30289: remove Misc/python-config.sh when make distclean (#1498) (#1506) --- Makefile.pre.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6e3a82f95df06d..2e4dd3b3ceb481 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1630,7 +1630,8 @@ distclean: clobber done -rm -f core Makefile Makefile.pre config.status \ Modules/Setup Modules/Setup.local Modules/Setup.config \ - Modules/ld_so_aix Modules/python.exp Misc/python.pc + Modules/ld_so_aix Modules/python.exp Misc/python.pc \ + Misc/python-config.sh -rm -f python*-gdb.py # Issue #28258: set LC_ALL to avoid issues with Estonian locale. # Expansion is performed here by shell (spawned by make) itself before From 72e1b61da0920c5607481304879e039b63e2a3d5 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 9 May 2017 12:16:50 +0800 Subject: [PATCH 0289/1345] bpo-29990: Fix range checking in GB18030 decoder (#1495) (#1507) When decoding a 4-byte GB18030 sequence, the first and third byte cannot exceed 0xFE. --- Lib/test/test_codecencodings_cn.py | 6 ++++++ Misc/NEWS | 2 ++ Modules/cjkcodecs/_codecs_cn.c | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 3bdf7d0e14b3b7..c8a410c2e0398c 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -48,6 +48,12 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), ) has_iso10646 = True diff --git a/Misc/NEWS b/Misc/NEWS index 6d6bdee08885c6..255942d7ca47e7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,8 @@ Core and Builtins Library ------- +- bpo-29990: Fix range checking in GB18030 decoder. Original patch by Ma Lin. + - Revert bpo-26293 for zipfile breakage. See also bpo-29094. - bpo-30243: Removed the __init__ methods of _json's scanner and encoder. diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index 1a070f2f393219..bda175c55d1323 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -279,7 +279,9 @@ DECODER(gb18030) REQUIRE_INBUF(4); c3 = INBYTE3; c4 = INBYTE4; - if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) + if (c < 0x81 || c > 0xFE || + c3 < 0x81 || c3 > 0xFE || + c4 < 0x30 || c4 > 0x39) return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; From 03b8a378dfa46372b96790f82c85e9b72518f1bf Mon Sep 17 00:00:00 2001 From: torsava Date: Tue, 9 May 2017 16:05:10 +0200 Subject: [PATCH 0290/1345] [3.6] bpo-29243: Fix Makefile with respect to --enable-optimizations (GH-1478) (#1518) * bpo-29243: Fix Makefile with respect to --enable-optimizations (#1478) * bpo-29243: Fix Makefile with respect to --enable-optimizations When using the Profile Guided Optimization (./configure --enable-optimizations) Python is built not only during `make` but rebuilt again during `make test`, `make install` and others. This patch fixes the issue. Note that this fix produces no change at all in the Makefile if configure is run witout --enable-optimizations. * !squash (cherry picked from commit a1054c3b0037d4c2a5492e79fc193f36245366c7) * [3.6] bpo-29243: Fix Makefile with respect to --enable-optimizations (GH-1478) * bpo-29243: Fix Makefile with respect to --enable-optimizations When using the Profile Guided Optimization (./configure --enable-optimizations) Python is built not only during `make` but rebuilt again during `make test`, `make install` and others. This patch fixes the issue. Note that this fix produces no change at all in the Makefile if configure is run witout --enable-optimizations. * !squash. (cherry picked from commit a1054c3b0037d4c2a5492e79fc193f36245366c7) --- Makefile.pre.in | 14 +++++++------- Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2e4dd3b3ceb481..076fb446310d81 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1002,7 +1002,7 @@ TESTTIMEOUT= 1200 # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: all platform +test: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1012,7 +1012,7 @@ test: all platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: all platform +testall: @DEF_MAKE_RULE@ platform -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1021,7 +1021,7 @@ testall: all platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: all platform +testuniversal: @DEF_MAKE_RULE@ platform if [ `arch` != 'i386' ];then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1044,7 +1044,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: all platform +quicktest: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(QUICKTESTOPTS) @@ -1381,7 +1381,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: all python-config +libainstall: @DEF_MAKE_RULE@ python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -1644,7 +1644,7 @@ distclean: clobber -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) -smelly: all +smelly: @DEF_MAKE_RULE@ nm -p $(LIBRARY) | \ sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ @@ -1681,7 +1681,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: all +patchcheck: @DEF_MAKE_RULE@ $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py # Dependencies diff --git a/Misc/ACKS b/Misc/ACKS index ac2c4edc733576..b38a1fe05fa76f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1114,6 +1114,7 @@ Jason Orendorff Douglas Orr William Orr Michele Orrù +Tomáš Orsava Oleg Oshmyan Denis S. Otkidach Peter Otten diff --git a/Misc/NEWS b/Misc/NEWS index 255942d7ca47e7..64974ef2eb2260 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -124,6 +124,10 @@ Library Build ----- +- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, + ``make install`` and some other make targets when configured with + ``--enable-optimizations``. + - bpo-23404: Don't regenerate generated files based on file modification time anymore: the action is now explicit. Replace ``make touch`` with ``make regen-all``. From 418d60a52579b6847776db3e354a8660b26dc752 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 10 May 2017 06:44:02 +0300 Subject: [PATCH 0291/1345] [3.6] bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. (GH-1490) (#1525) Now allowed several subsequential inline modifiers at the start of the pattern (e.g. '(?i)(?s)...'). In verbose mode whitespaces and comments now are allowed before and between inline modifiers (e.g. '(?x) (?i) (?s)...').. (cherry picked from commit 305ccbe27ea5ba82fd2d8c32ec739f980e524330) --- Lib/sre_parse.py | 12 ++++----- Lib/test/test_re.py | 59 ++++++++++++++++++++++++++++++++++----------- Misc/NEWS | 6 +++++ 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 6aa49c3bf6f8ed..e1c749634b71fe 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -412,7 +412,7 @@ def _parse_sub(source, state, verbose, nested=True): sourcematch = source.match start = source.tell() while True: - itemsappend(_parse(source, state, verbose)) + itemsappend(_parse(source, state, verbose, not nested and not items)) if not sourcematch("|"): break @@ -466,7 +466,7 @@ def _parse_sub_cond(source, state, condgroup, verbose): subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -def _parse(source, state, verbose): +def _parse(source, state, verbose, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -730,10 +730,9 @@ def _parse(source, state, verbose): state.checklookbehindgroup(condgroup, source) elif char in FLAGS or char == "-": # flags - pos = source.pos flags = _parse_flags(source, state, char) if flags is None: # global flags - if pos != 3: # "(?x" + if not first or subpattern: import warnings warnings.warn( 'Flags not at the start of the expression %s%s' % ( @@ -742,6 +741,8 @@ def _parse(source, state, verbose): ), DeprecationWarning, stacklevel=7 ) + if (state.flags & SRE_FLAG_VERBOSE) and not verbose: + raise Verbose continue add_flags, del_flags = flags group = None @@ -795,9 +796,6 @@ def _parse_flags(source, state, char): msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": - if ((add_flags & SRE_FLAG_VERBOSE) and - not (state.flags & SRE_FLAG_VERBOSE)): - raise Verbose state.flags |= add_flags return None if add_flags & GLOBAL_FLAGS: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b945cf094e99c1..c52417ba000ebc 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1303,32 +1303,43 @@ def test_inline_flags(self): upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below lower_char = '\u1ea1' # Latin Small Letter A with Dot Below - p = re.compile(upper_char, re.I | re.U) - q = p.match(lower_char) + p = re.compile('.' + upper_char, re.I | re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile(lower_char, re.I | re.U) - q = p.match(upper_char) + p = re.compile('.' + lower_char, re.I | re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?i)' + upper_char, re.U) - q = p.match(lower_char) + p = re.compile('(?i).' + upper_char, re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?i)' + lower_char, re.U) - q = p.match(upper_char) + p = re.compile('(?i).' + lower_char, re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?iu)' + upper_char) - q = p.match(lower_char) + p = re.compile('(?is).' + upper_char) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?iu)' + lower_char) - q = p.match(upper_char) + p = re.compile('(?is).' + lower_char) + q = p.match('\n' + upper_char) self.assertTrue(q) - self.assertTrue(re.match('(?ixu) ' + upper_char, lower_char)) - self.assertTrue(re.match('(?ixu) ' + lower_char, upper_char)) + p = re.compile('(?s)(?i).' + upper_char) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?s)(?i).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + self.assertTrue(re.match('(?ix) ' + upper_char, lower_char)) + self.assertTrue(re.match('(?ix) ' + lower_char, upper_char)) + self.assertTrue(re.match(' (?i) ' + upper_char, lower_char, re.X)) + self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) p = upper_char + '(?i)' with self.assertWarns(DeprecationWarning) as warns: @@ -1346,6 +1357,26 @@ def test_inline_flags(self): 'Flags not at the start of the expression %s (truncated)' % p[:20] ) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', + lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', + lower_char)) + + def test_dollar_matches_twice(self): "$ matches the end of string, and just before the terminating \n" pattern = re.compile('$') diff --git a/Misc/NEWS b/Misc/NEWS index 64974ef2eb2260..e7090fc17840ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,12 @@ Core and Builtins Library ------- +- bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. + Now allowed several subsequential inline modifiers at the start of the + pattern (e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments + now are allowed before and between inline modifiers (e.g. + ``'(?x) (?i) (?s)...'``). + - bpo-29990: Fix range checking in GB18030 decoder. Original patch by Ma Lin. - Revert bpo-26293 for zipfile breakage. See also bpo-29094. From 81ed53784608d44a8d196ad52288b55c7bf665db Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 10 May 2017 08:47:22 +0200 Subject: [PATCH 0292/1345] bpo-30320: test_eintr now uses pthread_sigmask() (#1523) (#1524) Rewrite sigwaitinfo() and sigtimedwait() unit tests for EINTR using pthread_sigmask() to fix a race condition between the child and the parent process. Remove the pipe which was used as a weak workaround against the race condition. sigtimedwait() is now tested with a child process sending a signal instead of testing the timeout feature which is more unstable (especially regarding to clock resolution depending on the platform). (cherry picked from commit 211a392cc15f9a7b1b8ce65d8f6c9f8237d1b77f) --- Lib/test/eintrdata/eintr_tester.py | 56 ++++++++++++++---------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index c619b3d6c38a59..1dbe88efe70ffd 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -371,59 +371,55 @@ def test_sleep(self): @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +# bpo-30320: Need pthread_sigmask() to block the signal, otherwise the test +# is vulnerable to a race condition between the child and the parent processes. +@unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') class SignalEINTRTest(EINTRBaseTest): """ EINTR tests for the signal module. """ - @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), - 'need signal.sigtimedwait()') - def test_sigtimedwait(self): - t0 = time.monotonic() - signal.sigtimedwait([signal.SIGUSR1], self.sleep_time) - dt = time.monotonic() - t0 - self.assertGreaterEqual(dt, self.sleep_time) - - @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), - 'need signal.sigwaitinfo()') - def test_sigwaitinfo(self): - # Issue #25277, #25868: give a few milliseconds to the parent process - # between os.write() and signal.sigwaitinfo() to works around a race - # condition - self.sleep_time = 0.100 - + def check_sigwait(self, wait_func): signum = signal.SIGUSR1 pid = os.getpid() old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) - rpipe, wpipe = os.pipe() - code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, - 'rpipe = %r' % rpipe, - 'os.read(rpipe, 1)', - 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) + old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) + t0 = time.monotonic() - proc = self.subprocess(code, pass_fds=(rpipe,)) - os.close(rpipe) + proc = self.subprocess(code) with kill_on_error(proc): - # sync child-parent - os.write(wpipe, b'x') - os.close(wpipe) + wait_func(signum) + dt = time.monotonic() - t0 + + self.assertEqual(proc.wait(), 0) - # parent + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + def wait_func(signum): signal.sigwaitinfo([signum]) - dt = time.monotonic() - t0 - self.assertEqual(proc.wait(), 0) - self.assertGreaterEqual(dt, self.sleep_time) + self.check_sigwait(wait_func) + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigwaitinfo()') + def test_sigtimedwait(self): + def wait_func(signum): + signal.sigtimedwait([signum], 120.0) + + self.check_sigwait(wait_func) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") From 98b49a00d413972965044fd355be3b8482caa07c Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 10 May 2017 19:00:15 +0800 Subject: [PATCH 0293/1345] bpo-30281: Fix the default value for stop in PySlice_Unpack() (#1480) (#1529) --- Objects/sliceobject.c | 8 +++++--- Python/ceval.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 6a690213d14518..4e4b32d32441a8 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -197,6 +197,8 @@ PySlice_Unpack(PyObject *_r, PySliceObject *r = (PySliceObject*)_r; /* this is harder to get right than you might think */ + Py_BUILD_ASSERT(PY_SSIZE_T_MIN + 1 <= -PY_SSIZE_T_MAX); + if (r->step == Py_None) { *step = 1; } @@ -217,14 +219,14 @@ PySlice_Unpack(PyObject *_r, } if (r->start == Py_None) { - *start = *step < 0 ? PY_SSIZE_T_MAX-1 : 0;; + *start = *step < 0 ? PY_SSIZE_T_MAX : 0; } else { if (!_PyEval_SliceIndex(r->start, start)) return -1; } if (r->stop == Py_None) { - *stop = *step < 0 ? -PY_SSIZE_T_MAX : PY_SSIZE_T_MAX; + *stop = *step < 0 ? PY_SSIZE_T_MIN : PY_SSIZE_T_MAX; } else { if (!_PyEval_SliceIndex(r->stop, stop)) return -1; @@ -258,7 +260,7 @@ PySlice_AdjustIndices(Py_ssize_t length, *stop = (step < 0) ? -1 : 0; } } - else if (*stop >= length) { + else if (*stop >= length) { *stop = (step < 0) ? length - 1 : length; } diff --git a/Python/ceval.c b/Python/ceval.c index bce86ab12cf2fb..5dc0444a1acf5c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5071,7 +5071,7 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, - and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. + and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN. Return 0 on error, 1 on success. */ int From a4465a5bd08f232cf097893006118c82975c3402 Mon Sep 17 00:00:00 2001 From: stratakis Date: Thu, 11 May 2017 13:13:18 +0200 Subject: [PATCH 0294/1345] [3.6] bpo-28787: Fix out of tree --with-dtrace builds (GH-135) (#1543) * bpo-28787: Fix out of tree --with-dtrace builds * Unsilence directory creation * Add Misc/NEWS and Misc/ACKS entries. (cherry picked from commit f6eae5bf1c5d7b83e5d5bdbecfff928e478c1cfd) --- Makefile.pre.in | 1 + Misc/ACKS | 1 + Misc/NEWS | 3 +++ 3 files changed, 5 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 076fb446310d81..82e830727ef849 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -875,6 +875,7 @@ Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_externa # follow our naming conventions. dtrace(1) uses the output filename to generate # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d + $(MKDIR_P) Include $(DTRACE) $(DFLAGS) -o $@ -h -s $< : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp diff --git a/Misc/ACKS b/Misc/ACKS index b38a1fe05fa76f..486b6e32850b88 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1476,6 +1476,7 @@ Daniel Stokes Michael Stone Serhiy Storchaka Ken Stox +Charalampos Stratakis Dan Stromberg Donald Stufft Daniel Stutzbach diff --git a/Misc/NEWS b/Misc/NEWS index e7090fc17840ab..58b8d262ecfee1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -130,6 +130,9 @@ Library Build ----- +- bpo-28787: Fix out-of-tree builds of Python when configured with + ``--with--dtrace``. + - bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, ``make install`` and some other make targets when configured with ``--enable-optimizations``. From 3dc7c52a9f4fb83be3e26e31e2c7cd9dc1cb41a2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 11 May 2017 21:56:42 +0900 Subject: [PATCH 0295/1345] bpo-30048: asyncio: fix Task.cancel() was ignored. (GH-1546) when there are no more `await` or `yield (from)` before return in coroutine, cancel was ignored. example: async def coro(): asyncio.Task.current_task().cancel() return 42 ... res = await coro() # should raise CancelledError (cherry picked from commit 991adca012f5e106c2d4040ce619c696ba6f9c46) --- Lib/asyncio/tasks.py | 7 ++++++- Lib/test/test_asyncio/test_tasks.py | 18 ++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 12 ++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index f91e70aecbad94..d7867d128a8afe 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -180,7 +180,12 @@ def _step(self, exc=None): else: result = coro.throw(exc) except StopIteration as exc: - self.set_result(exc.value) + if self._must_cancel: + # Task is cancelled right before coro stops. + self._must_cancel = False + self.set_exception(futures.CancelledError()) + else: + self.set_result(exc.value) except futures.CancelledError: super().cancel() # I.e., Future.cancel(self). except Exception as exc: diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4f05319457985a..5462c80ad3013a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -588,6 +588,24 @@ def task(): self.assertFalse(t._must_cancel) # White-box test. self.assertFalse(t.cancel()) + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + def test_stop_while_run_in_complete(self): def gen(): diff --git a/Misc/NEWS b/Misc/NEWS index 58b8d262ecfee1..b6b405446f6c7e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Core and Builtins Library ------- +- bpo-30048: Fixed ``Task.cancel()`` can be ignored when the task is + running coroutine and the coroutine returned without any more ``await``. + - bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. Now allowed several subsequential inline modifiers at the start of the pattern (e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a77ff9689e4705..75327fa3014be9 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1984,6 +1984,16 @@ task_step_impl(TaskObj *task, PyObject *exc) if (_PyGen_FetchStopIterationValue(&o) == 0) { /* The error is StopIteration and that means that the underlying coroutine has resolved */ + if (task->task_must_cancel) { + // Task is cancelled right before coro stops. + Py_DECREF(o); + task->task_must_cancel = 0; + et = asyncio_CancelledError; + Py_INCREF(et); + ev = NULL; + tb = NULL; + goto set_exception; + } PyObject *res = future_set_result((FutureObj*)task, o); Py_DECREF(o); if (res == NULL) { @@ -2001,6 +2011,8 @@ task_step_impl(TaskObj *task, PyObject *exc) /* Some other exception; pop it and call Task.set_exception() */ PyErr_Fetch(&et, &ev, &tb); + +set_exception: assert(et); if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { PyErr_NormalizeException(&et, &ev, &tb); From d05f7fdf6cf77724bd3064fb5a0846ef5cfe0c88 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 13 May 2017 00:21:50 +0200 Subject: [PATCH 0296/1345] [3.6] bpo-30345: Update test_gdb.py and python-gdb.py from master (#1549) * python-gdb.py supports method-wrapper bpo-29367: python-gdb.py now supports also method-wrapper (wrapperobject) objects. (cherry picked from commit 611083331d534481ca7956a376e38fb0e9ef3854) * Update and enhance python-gdb.py bpo-29259: Detect PyCFunction is the current frame, not only in the older frame. --- Lib/test/test_gdb.py | 22 +++++++++++++-- Misc/NEWS | 6 ++++ Tools/gdb/libpython.py | 64 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 60f1d92846744d..b7554d698c9d6c 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -3,13 +3,14 @@ # The code for testing gdb was adapted from similar work in Unladen Swallow's # Lib/test/test_jit_gdb.py +import locale import os import re import subprocess import sys import sysconfig +import textwrap import unittest -import locale # Is this Python configured to support threads? try: @@ -845,7 +846,24 @@ def test_pycfunction(self): breakpoint='time_gmtime', cmds_after_breakpoint=['py-bt-full'], ) - self.assertIn('#0 ' + + def safe_tp_name(self): + try: + return self.field('self')['ob_type']['tp_name'].string() + except (NullPyObjectPtr, RuntimeError): + return '' + + def safe_self_addresss(self): + try: + address = long(self.field('self')) + return '%#x' % address + except (NullPyObjectPtr, RuntimeError): + return '' + + def proxyval(self, visited): + name = self.safe_name() + tp_name = self.safe_tp_name() + self_address = self.safe_self_addresss() + return ("" + % (name, tp_name, self_address)) + + def write_repr(self, out, visited): + proxy = self.proxyval(visited) + out.write(proxy) def int_from_int(gdbval): @@ -1364,11 +1398,13 @@ def to_string (self): def pretty_printer_lookup(gdbval): type = gdbval.type.unqualified() - if type.code == gdb.TYPE_CODE_PTR: - type = type.target().unqualified() - t = str(type) - if t in ("PyObject", "PyFrameObject", "PyUnicodeObject"): - return PyObjectPtrPrinter(gdbval) + if type.code != gdb.TYPE_CODE_PTR: + return None + + type = type.target().unqualified() + t = str(type) + if t in ("PyObject", "PyFrameObject", "PyUnicodeObject", "wrapperobject"): + return PyObjectPtrPrinter(gdbval) """ During development, I've been manually invoking the code in this way: @@ -1497,11 +1533,8 @@ def is_other_python_frame(self): return 'Garbage-collecting' # Detect invocations of PyCFunction instances: - older = self.older() - if not older: - return False - - caller = older._gdbframe.name() + frame = self._gdbframe + caller = frame.name() if not caller: return False @@ -1513,18 +1546,25 @@ def is_other_python_frame(self): # "self" is the (PyObject*) of the 'self' try: # Use the prettyprinter for the func: - func = older._gdbframe.read_var('func') + func = frame.read_var('func') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func")' elif caller == '_PyCFunction_FastCallDict': try: - func = older._gdbframe.read_var('func_obj') + func = frame.read_var('func_obj') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func_obj")' + if caller == 'wrapper_call': + try: + func = frame.read_var('wp') + return str(func) + except RuntimeError: + return '' + # This frame isn't worth reporting: return False From 3a66ab8695e9ba969ff6bdd8c91f88138c8b5187 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 13 May 2017 09:42:39 -0700 Subject: [PATCH 0297/1345] [3.6] bpo-30178: Indent methods and attributes of MimeType class (GH-1306) (#1570) (cherry picked from commit c71168090df435c1eb8c03005b11df764cd7ebd6) --- Doc/library/mimetypes.rst | 86 +++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 464248c3ea7990..67b7a7178534d3 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -186,78 +186,78 @@ than one MIME-type database; it provides an interface similar to the one of the loaded "on top" of the default database. -.. attribute:: MimeTypes.suffix_map + .. attribute:: MimeTypes.suffix_map - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + Dictionary mapping suffixes to suffixes. This is used to allow recognition of + encoded files for which the encoding and the type are indicated by the same + extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` + to allow the encoding and type to be recognized separately. This is initially a + copy of the global :data:`suffix_map` defined in the module. -.. attribute:: MimeTypes.encodings_map + .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. This is initially a + copy of the global :data:`encodings_map` defined in the module. -.. attribute:: MimeTypes.types_map + .. attribute:: MimeTypes.types_map - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + Tuple containing two dictionaries, mapping filename extensions to MIME types: + the first dictionary is for the non-standards types and the second one is for + the standard types. They are initialized by :data:`common_types` and + :data:`types_map`. -.. attribute:: MimeTypes.types_map_inv + .. attribute:: MimeTypes.types_map_inv - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + Tuple containing two dictionaries, mapping MIME types to a list of filename + extensions: the first dictionary is for the non-standards types and the + second one is for the standard types. They are initialized by + :data:`common_types` and :data:`types_map`. -.. method:: MimeTypes.guess_extension(type, strict=True) + .. method:: MimeTypes.guess_extension(type, strict=True) - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. + Similar to the :func:`guess_extension` function, using the tables stored as part + of the object. -.. method:: MimeTypes.guess_type(url, strict=True) + .. method:: MimeTypes.guess_type(url, strict=True) - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. + Similar to the :func:`guess_type` function, using the tables stored as part of + the object. -.. method:: MimeTypes.guess_all_extensions(type, strict=True) + .. method:: MimeTypes.guess_all_extensions(type, strict=True) - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. + Similar to the :func:`guess_all_extensions` function, using the tables stored + as part of the object. -.. method:: MimeTypes.read(filename, strict=True) + .. method:: MimeTypes.read(filename, strict=True) - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. + Load MIME information from a file named *filename*. This uses :meth:`readfp` to + parse the file. - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. + If *strict* is ``True``, information will be added to list of standard types, + else to the list of non-standard types. -.. method:: MimeTypes.readfp(fp, strict=True) + .. method:: MimeTypes.readfp(fp, strict=True) - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. + Load MIME type information from an open file *fp*. The file must have the format of + the standard :file:`mime.types` files. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. -.. method:: MimeTypes.read_windows_registry(strict=True) + .. method:: MimeTypes.read_windows_registry(strict=True) - Load MIME type information from the Windows registry. Availability: Windows. + Load MIME type information from the Windows registry. Availability: Windows. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. - .. versionadded:: 3.2 + .. versionadded:: 3.2 From 178756c15da0652b30b55b41f0bb939b61c66513 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 13 May 2017 13:51:24 -0700 Subject: [PATCH 0298/1345] [3.6] Move Codecov's configuration file under .github (GH-1494) (GH-1574) (cherry picked from commit cbddf58c797f850a5b06f317a4bb7ab69c6e9715) --- .codecov.yml => .github/codecov.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .codecov.yml => .github/codecov.yml (100%) diff --git a/.codecov.yml b/.github/codecov.yml similarity index 100% rename from .codecov.yml rename to .github/codecov.yml From 61b6e5cf929a80bb44500fb1a212facf66d7f45c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 14 May 2017 18:30:48 +0300 Subject: [PATCH 0299/1345] bpo-30358: Document sort argument of profile.runctx() (GH-1566) (cherry picked from commit 99776296230ddd8429ebad2d07854b8c27ea10ab) --- Doc/library/profile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index b4b1479e2ceba4..5796e3acb6a797 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -215,11 +215,11 @@ functions: and gathers profiling statistics from the execution. If no file name is present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified + instance and prints a simple profiling report. If the sort value is specified, it is passed to this :class:`~pstats.Stats` instance to control how the results are sorted. -.. function:: runctx(command, globals, locals, filename=None) +.. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the globals and locals dictionaries for the *command* string. This routine From 0ce1f7e02d4102e8a459d1855d689c8e4613a327 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 15 May 2017 13:17:01 +0800 Subject: [PATCH 0300/1345] bpo-30110: fix resource leak in test_asyncio.test_events (#1413) (#1584) --- Lib/test/test_asyncio/test_events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 802763bd11ff61..492a84a2313baf 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2194,8 +2194,10 @@ def tearDown(self): def test_get_event_loop_new_process(self): async def main(): pool = concurrent.futures.ProcessPoolExecutor() - return await self.loop.run_in_executor( + result = await self.loop.run_in_executor( pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result self.unpatch_get_running_loop() From aad1caf55f7fadb00ee097a1465bece4e128d8ef Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 15 May 2017 13:17:28 +0800 Subject: [PATCH 0301/1345] bpo-30242: resolve some undefined behaviours in struct (#1418) (#1586) --- Modules/_struct.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/_struct.c b/Modules/_struct.c index 796d1682f094f1..2635af9db69597 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -539,7 +539,7 @@ np_ubyte(char *p, PyObject *v, const formatdef *f) "ubyte format requires 0 <= number <= 255"); return -1; } - *p = (char)x; + *(unsigned char *)p = (unsigned char)x; return 0; } @@ -868,6 +868,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -880,7 +881,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffL); x >>= 8; } while (i > 0); return 0; @@ -891,6 +892,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -901,7 +903,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffUL); x >>= 8; } while (i > 0); return 0; @@ -1087,6 +1089,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -1099,7 +1102,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffL); x >>= 8; } while (--i > 0); return 0; @@ -1110,6 +1113,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -1120,7 +1124,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffUL); x >>= 8; } while (--i > 0); return 0; From 75b8a54bcad70806d9dcbbe20786f4d9092ab39c Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 15 May 2017 22:41:07 -0700 Subject: [PATCH 0302/1345] bpo-29651 - Cover edge case of square brackets in urllib docs (#1128) (#1596) (cherry picked from commit f6e863d868a621594df2a8abe072b5d4766e7137) --- Doc/library/urllib.parse.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 676321b46a2232..1cc69e62e63318 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -118,6 +118,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -236,6 +239,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. From 73fb45df0487144765808c5d25914c67232d83fe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 16 May 2017 18:16:15 +0300 Subject: [PATCH 0303/1345] [3.6] bpo-30375: Correct the stacklevel of regex compiling warnings. (GH-1595) (#1604) Warnings emitted when compile a regular expression now always point to the line in the user code. Previously they could point into inners of the re module if emitted from inside of groups or conditionals.. (cherry picked from commit c7ac7280c321b3c1679fe5f657a6be0f86adf173) --- Lib/sre_parse.py | 25 +++++++++++++------------ Lib/test/test_re.py | 17 ++++++++++++++--- Misc/NEWS | 4 ++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index e1c749634b71fe..608f9a26642f05 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -404,7 +404,7 @@ def _escape(source, escape, state): pass raise source.error("bad escape %s" % escape, len(escape)) -def _parse_sub(source, state, verbose, nested=True): +def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c items = [] @@ -412,7 +412,8 @@ def _parse_sub(source, state, verbose, nested=True): sourcematch = source.match start = source.tell() while True: - itemsappend(_parse(source, state, verbose, not nested and not items)) + itemsappend(_parse(source, state, verbose, nested + 1, + not nested and not items)) if not sourcematch("|"): break @@ -454,10 +455,10 @@ def _parse_sub(source, state, verbose, nested=True): subpattern.append((BRANCH, (None, items))) return subpattern -def _parse_sub_cond(source, state, condgroup, verbose): - item_yes = _parse(source, state, verbose) +def _parse_sub_cond(source, state, condgroup, verbose, nested): + item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): - item_no = _parse(source, state, verbose) + item_no = _parse(source, state, verbose, nested + 1) if source.next == "|": raise source.error("conditional backref with more than two branches") else: @@ -466,7 +467,7 @@ def _parse_sub_cond(source, state, condgroup, verbose): subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -def _parse(source, state, verbose, first=False): +def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -692,7 +693,7 @@ def _parse(source, state, verbose, first=False): lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups - p = _parse_sub(source, state, verbose) + p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None @@ -739,7 +740,7 @@ def _parse(source, state, verbose, first=False): source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), - DeprecationWarning, stacklevel=7 + DeprecationWarning, stacklevel=nested + 6 ) if (state.flags & SRE_FLAG_VERBOSE) and not verbose: raise Verbose @@ -757,11 +758,11 @@ def _parse(source, state, verbose, first=False): except error as err: raise source.error(err.msg, len(name) + 1) from None if condgroup: - p = _parse_sub_cond(source, state, condgroup, verbose) + p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1) else: sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) - p = _parse_sub(source, state, sub_verbose) + p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) @@ -851,7 +852,7 @@ def parse(str, flags=0, pattern=None): pattern.str = str try: - p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, False) + p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... @@ -859,7 +860,7 @@ def parse(str, flags=0, pattern=None): pattern.flags = flags | SRE_FLAG_VERBOSE pattern.str = str source.seek(0) - p = _parse_sub(source, pattern, True, False) + p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index c52417ba000ebc..e88d0b3dcf2a78 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1348,6 +1348,7 @@ def test_inline_flags(self): str(warns.warnings[0].message), 'Flags not at the start of the expression %s' % p ) + self.assertEqual(warns.warnings[0].filename, __file__) p = upper_char + '(?i)%s' % ('.?' * 100) with self.assertWarns(DeprecationWarning) as warns: @@ -1356,6 +1357,7 @@ def test_inline_flags(self): str(warns.warnings[0].message), 'Flags not at the start of the expression %s (truncated)' % p[:20] ) + self.assertEqual(warns.warnings[0].filename, __file__) with self.assertWarns(DeprecationWarning): self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) @@ -1367,14 +1369,23 @@ def test_inline_flags(self): self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) with self.assertWarns(DeprecationWarning): self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) - with self.assertWarns(DeprecationWarning): + with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) - with self.assertWarns(DeprecationWarning): + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', lower_char)) - with self.assertWarns(DeprecationWarning): + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) def test_dollar_matches_twice(self): diff --git a/Misc/NEWS b/Misc/NEWS index bfb14c13653111..567f40976e2cde 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Core and Builtins Library ------- +- bpo-30375: Warnings emitted when compile a regular expression now always + point to the line in the user code. Previously they could point into inners + of the re module if emitted from inside of groups or conditionals. + - bpo-30048: Fixed ``Task.cancel()`` can be ignored when the task is running coroutine and the coroutine returned without any more ``await``. From 8fc1ff51adb52c80f6bf78393413c72543bae3e1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 17 May 2017 00:09:23 +0300 Subject: [PATCH 0304/1345] [3.6] bpo-30380: Fix Sphinx 1.6.1 warnings. (GH-1613) (#1614) * Use explicit numbering for footnotes referred by explicit number. * Restore missed footnote reference in stdtypes.rst. * Fix literal strings formatting in howto/urllib2.rst. * Update susp-ignored.csv for zipapp.rst. * Fix suspicious mark up in Misc/NEWS.. (cherry picked from commit d97b7dc94b19063f0589d401bdc4aaadc7030762) --- Doc/howto/urllib2.rst | 10 +++++----- Doc/library/pyexpat.rst | 2 +- Doc/library/stdtypes.rst | 2 +- Doc/library/xml.dom.minidom.rst | 2 +- Doc/library/xml.etree.elementtree.rst | 2 +- Doc/tools/susp-ignored.csv | 7 +++---- Misc/NEWS | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 18b5c6556be1c9..8d383e03ee8a28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -34,8 +34,8 @@ handling common situations - like basic authentication, cookies, proxies and so on. These are provided by objects called handlers and openers. urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ":" in URL - for example "ftp" is the URL scheme of -"ftp://python.org/") using their associated network protocols (e.g. FTP, HTTP). +before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of +``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). This tutorial focuses on the most common case, HTTP. For straightforward situations *urlopen* is very easy to use. But as soon as you @@ -511,10 +511,10 @@ than the URL you pass to .add_password() will also match. :: ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) -e.g. "http://example.com/" *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. "example.com" or "example.com:8080" +e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, +optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` (the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example "joe:password@example.com" is +NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is not correct. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 075a8b5139b196..e43b9aecd86835 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -869,7 +869,7 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6729a20a9c2ecc..af0a7b7fe939b2 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -829,7 +829,7 @@ restrictions imposed by *s*. The ``in`` and ``not in`` operations have the same priorities as the comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. +operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2e9e814693df57..40470e8736e7e4 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -248,7 +248,7 @@ utility to most DOM users. .. rubric:: Footnotes -.. [#] The encoding name included in the XML output should conform to +.. [1] The encoding name included in the XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not valid in an XML document's declaration, even though Python accepts it as an encoding name. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index b54eace41188b0..7d814ad406eb1b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1192,7 +1192,7 @@ Exceptions .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 57b0cf12f53712..2dc540459020a6 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -106,7 +106,7 @@ howto/pyporting,,::,Programming Language :: Python :: 2 howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"for example ""joe:password@example.com""" +howto/urllib2,,:password,"""joe:password@example.com""" library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) @@ -316,8 +316,8 @@ library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:ac library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)" library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):" library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" -library/zipapp,,:fn,"argument should have the form ""pkg.mod:fn"", where ""pkg.mod"" is a" -library/zipapp,,:callable,"""pkg.module:callable"" and the archive will be run by importing" +library/zipapp,,:fn,"pkg.mod:fn" +library/zipapp,,:callable,"pkg.module:callable" library/stdtypes,,::,>>> m[::2].tolist() library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' @@ -326,4 +326,3 @@ whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,`,"for readability (was ""`"")." diff --git a/Misc/NEWS b/Misc/NEWS index 567f40976e2cde..ca38fe0e5cdf36 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -783,7 +783,7 @@ Library non-None value is passed to it.send(val). - Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). + for readability. - Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin a workaround to Tix library bug. From 5e2016dce92d66e6e1fb25e6ce39031d8c7d5412 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 16 May 2017 14:28:21 -0700 Subject: [PATCH 0305/1345] bpo-30380: Pin the version of Sphinx used to build the docs (GH-1612) (GH-1617) --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fee6cd7673324..fe90bad626ce83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,9 +37,11 @@ matrix: - TESTING=docs before_script: - cd Doc - - make venv + # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. + # (Updating the version is fine as long as no warnings are raised by doing so.) + - python -m pip install sphinx~=1.6.1 script: - - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" + - make check suspicious html SPHINXOPTS="-q -W" - os: linux language: c compiler: gcc From 0d267041c40a3ab655676b885c0ba35ed9685848 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 17 May 2017 00:31:39 +0300 Subject: [PATCH 0306/1345] bpo-29898: Fix incorrect env variable name (GH-1576) It should read PYTHONLEGACYWINDOWSSTDIO as stated in section "Add legacy mode" in PEP 528. (cherry picked from commit 87fa8a780e9045a26c735f085c07bba4b2d0be60) --- Doc/using/cmdline.rst | 4 ++-- Doc/whatsnew/3.6.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 08dc311d837be9..40a06b9adc06ef 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -571,7 +571,7 @@ conflict. .. versionchanged:: 3.6 On Windows, the encoding specified by this variable is ignored for interactive - console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified. + console buffers unless :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also specified. Files and pipes redirected through the standard streams are not affected. .. envvar:: PYTHONNOUSERSITE @@ -700,7 +700,7 @@ conflict. .. versionadded:: 3.6 See :pep:`529` for more details. -.. envvar:: PYTHONLEGACYWINDOWSIOENCODING +.. envvar:: PYTHONLEGACYWINDOWSSTDIO If set to a non-empty string, does not use the new console reader and writer. This means that Unicode characters will be encoded according to diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index a696af4a999c84..a701cafa610788 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -534,7 +534,7 @@ provide correctly read str objects to Python code. ``sys.stdin``, This change only applies when using an interactive console, and not when redirecting files or pipes. To revert to the previous behaviour for interactive -console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`. +console use, set :envvar:`PYTHONLEGACYWINDOWSSTDIO`. .. seealso:: From aaa053652556aa4e59cb68f3983a09ced1d1fe2a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 16 May 2017 15:29:41 -0700 Subject: [PATCH 0307/1345] bpo-30329: Catch Windows error 10022 on shutdown() (#1538) (#1620) Catch the Windows socket WSAEINVAL error (code 10022) in imaplib and poplib on shutdown(SHUT_RDWR): An invalid operation was attempted This error occurs sometimes on SSL connections. (cherry picked from commit 83a2c2879839da2e10037f5e4af1bd1dafbf1a52) --- Lib/imaplib.py | 9 ++++++--- Lib/poplib.py | 9 ++++++--- Misc/NEWS | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2fa90120e7decc..1c0b03bff8a379 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -318,9 +318,12 @@ def shutdown(self): self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: self.sock.close() diff --git a/Lib/poplib.py b/Lib/poplib.py index cae6950eb6d2d6..6bcfa5cfeba37b 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -288,9 +288,12 @@ def close(self): if sock is not None: try: sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: sock.close() diff --git a/Misc/NEWS b/Misc/NEWS index ca38fe0e5cdf36..6a50c6265c5cf1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Core and Builtins Library ------- +- bpo-30329: imaplib and poplib now catch the Windows socket WSAEINVAL error + (code 10022) on shutdown(SHUT_RDWR): An invalid operation was attempted. + This error occurs sometimes on SSL connections. + - bpo-30375: Warnings emitted when compile a regular expression now always point to the line in the user code. Previously they could point into inners of the re module if emitted from inside of groups or conditionals. From 6b5b85aecf4db796a05582202a6c875b36eae516 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 16 May 2017 17:06:14 -0700 Subject: [PATCH 0308/1345] bpo-30357: test_thread now uses threading_cleanup() (#1592) (#1622) test_thread: setUp() now uses support.threading_setup() and support.threading_cleanup() to wait until threads complete to avoid random side effects on following tests. Co-Authored-By: Grzegorz Grzywacz (cherry picked from commit 79ef7f8e88a4972c4aecf95cfc5cd934f1861e08) --- Lib/test/test_thread.py | 4 ++++ Misc/ACKS | 1 + Misc/NEWS | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index ef3059b68674b4..3909b75ccd4647 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -20,6 +20,7 @@ def verbose_print(arg): with _print_mutex: print(arg) + class BasicThreadTest(unittest.TestCase): def setUp(self): @@ -31,6 +32,9 @@ def setUp(self): self.running = 0 self.next_ident = 0 + key = support.threading_setup() + self.addCleanup(support.threading_cleanup, *key) + class ThreadRunningTests(BasicThreadTest): diff --git a/Misc/ACKS b/Misc/ACKS index 486b6e32850b88..ccc9d2af3149c0 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -551,6 +551,7 @@ Eric Groo Daniel Andrade Groppe Dag Gruneau Filip Gruszczyński +Grzegorz Grzywacz Thomas Guettler Yuyang Guo Anuj Gupta diff --git a/Misc/NEWS b/Misc/NEWS index 6a50c6265c5cf1..0459e8490048c9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -168,6 +168,11 @@ Tools/Demos Tests ----- +* bpo-30357: test_thread: setUp() now uses support.threading_setup() and + support.threading_cleanup() to wait until threads complete to avoid + random side effects on following tests. Initial patch written by Grzegorz + Grzywacz. + - bpo-30197: Enhanced functions swap_attr() and swap_item() in the test.support module. They now work when delete replaced attribute or item inside the with statement. The old value of the attribute or item (or None From 460945f22acd288e660b432b288d9d81655572bf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 16 May 2017 17:58:02 -0700 Subject: [PATCH 0309/1345] bpo-30273: update distutils.sysconfig for venv's created from Python (#1515) (#1625) compiled out-of-tree (builddir != srcdir). (see also bpo-15366) (cherry picked from commit dbdea629e2e0e4bd8845aa55041e0a0ca4172cf3) --- Lib/distutils/sysconfig.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 90004acea88761..2bcd1dd2885991 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -93,14 +93,11 @@ def get_python_inc(plat_specific=0, prefix=None): # the build directory may not be the source directory, we # must use "srcdir" from the makefile to find the "Include" # directory. - base = _sys_home or project_base if plat_specific: - return base - if _sys_home: - incdir = os.path.join(_sys_home, 'Include') + return _sys_home or project_base else: incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) + return os.path.normpath(incdir) python_dir = 'python' + get_python_version() + build_flags return os.path.join(prefix, "include", python_dir) elif os.name == "nt": From b769c91c2db304db44cd514344369cde1f8d9dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Miedzi=C5=84ski?= Date: Wed, 17 May 2017 08:17:55 +0200 Subject: [PATCH 0310/1345] Fix ModuleNotFoundError typo in import reference (#1610) (cherry picked from commit c138d84b8d27d338872587aae69a73d8f9d16641) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 4dbd9d88c301a0..9475b8d55c2dc3 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -890,7 +890,7 @@ import statements within that module. To selectively prevent import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNoFoundError` directly from +it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. From 43d4c0329e2348540a3a16ac61b3032f04eefd34 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 17 May 2017 22:03:35 +0800 Subject: [PATCH 0311/1345] bpo-30301: Fix AttributeError when using SimpleQueue.empty() (#1601) (#1628) Under *spawn* and *forkserver* start methods, SimpleQueue.empty() could raise AttributeError due to not setting _poll in __setstate__. --- Lib/multiprocessing/queues.py | 1 + Lib/test/_test_multiprocessing.py | 36 +++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index dda03ddf5425ce..a4f4ef8b7b5334 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -337,6 +337,7 @@ def __getstate__(self): def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state + self._poll = self._reader.poll def get(self): with self._rlock: diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 57ada127465d75..3eb83c59ab17d7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3958,6 +3958,42 @@ def test_semaphore_tracker(self): self.assertRegex(err, expected) self.assertRegex(err, r'semaphore_tracker: %r: \[Errno' % name1) +class TestSimpleQueue(unittest.TestCase): + + @classmethod + def _test_empty(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + # issue 30301, could fail under spawn and forkserver + try: + queue.put(queue.empty()) + queue.put(queue.empty()) + finally: + parent_can_continue.set() + + def test_empty(self): + queue = multiprocessing.SimpleQueue() + child_can_start = multiprocessing.Event() + parent_can_continue = multiprocessing.Event() + + proc = multiprocessing.Process( + target=self._test_empty, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertTrue(queue.empty()) + + child_can_start.set() + parent_can_continue.wait() + + self.assertFalse(queue.empty()) + self.assertEqual(queue.get(), True) + self.assertEqual(queue.get(), False) + self.assertTrue(queue.empty()) + + proc.join() + # # Mixins # diff --git a/Misc/NEWS b/Misc/NEWS index 0459e8490048c9..6336e67ee48edb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Core and Builtins Library ------- +- bpo-30301: Fix AttributeError when using SimpleQueue.empty() under + *spawn* and *forkserver* start methods. + - bpo-30329: imaplib and poplib now catch the Windows socket WSAEINVAL error (code 10022) on shutdown(SHUT_RDWR): An invalid operation was attempted. This error occurs sometimes on SSL connections. From 0d1727835f44492c7357b013d94539e3891adf77 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 17 May 2017 10:51:01 -0700 Subject: [PATCH 0312/1345] Remove unused variable in test_urllibnet. (#1598) (#1600) (cherry picked from commit 1bd7d299bd2a91f8267f97a413568ab8fe7fdfbb) --- Lib/test/test_urllibnet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 865a7f4f5fb172..4103b6c07505d5 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -11,6 +11,7 @@ support.requires('network') + class URLTimeoutTest(unittest.TestCase): # XXX this test doesn't seem to test anything useful. @@ -25,7 +26,7 @@ def tearDown(self): def testURLread(self): with support.transient_internet("www.example.com"): f = urllib.request.urlopen("http://www.example.com/") - x = f.read() + f.read() class urlopenNetworkTests(unittest.TestCase): @@ -188,6 +189,7 @@ def test_data_header(self): def test_reporthook(self): records = [] + def recording_reporthook(blocks, block_size, total_size): records.append((blocks, block_size, total_size)) From 69f3a5ac28041fac86897e0c90d98ad9fd6fa3f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 17 May 2017 14:45:45 -0700 Subject: [PATCH 0313/1345] tmtotuple(): use time_t for gmtoff (#1276) (#1635) timegm() return type is time_t, not int. Use time_t to prevent the following compiler warning on Windows: timemodule.c: warning C4244: '=': conversion from 'time_t' to 'int', possible loss of data (cherry picked from commit 0d659e5614cad512a1940125135b443b3eecb5d7) --- Modules/timemodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ebd44ad525bd81..328b84f3338ea3 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -279,7 +279,7 @@ static PyTypeObject StructTimeType; static PyObject * tmtotuple(struct tm *p #ifndef HAVE_STRUCT_TM_TM_ZONE - , const char *zone, int gmtoff + , const char *zone, time_t gmtoff #endif ) { @@ -305,7 +305,7 @@ tmtotuple(struct tm *p #else PyStructSequence_SET_ITEM(v, 9, PyUnicode_DecodeLocale(zone, "surrogateescape")); - SET(10, gmtoff); + PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { @@ -397,7 +397,7 @@ time_localtime(PyObject *self, PyObject *args) { struct tm local = buf; char zone[100]; - int gmtoff; + time_t gmtoff; strftime(zone, sizeof(zone), "%Z", &buf); gmtoff = timegm(&buf) - when; return tmtotuple(&local, zone, gmtoff); From 44944b602af23712d7119801183cdc6e202e4f76 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 17 May 2017 14:49:38 -0700 Subject: [PATCH 0314/1345] bpo-30387: Fix warning in test_threading (#1634) (#1636) test_is_alive_after_fork() now joins directly the thread to avoid the following warning added by bpo-30357: Warning -- threading_cleanup() failed to cleanup 0 threads after 2 sec (count: 0, dangling: 21) Use also a different exit code to catch generic exit code 1. (cherry picked from commit f8d05b3a24e745ab4a974b891ac1389e2f11ce4d) --- Lib/test/test_threading.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 6b6c4d220a3bd0..0db028864d203a 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -470,13 +470,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - self.addCleanup(t.join) pid = os.fork() if pid == 0: - os._exit(1 if t.is_alive() else 0) + os._exit(11 if t.is_alive() else 10) else: + t.join() + pid, status = os.waitpid(pid, 0) - self.assertEqual(0, status) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) def test_main_thread(self): main = threading.main_thread() From 2773add19aff873377d81e3bb6ab8aa942756f5a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 18 May 2017 13:36:51 -0700 Subject: [PATCH 0315/1345] bpo-27103: regrtest disables -W if -R is used (#1651) (#1656) Workaround for a regrtest bug. (cherry picked from commit fcdd9b6b7e73427ce5aa63cf095312f603c4edce) --- Lib/test/libregrtest/cmdline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d621f5f9f3ee1f..cdbd1b8b92553c 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -343,5 +343,10 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.huntrleaks and ns.verbose3: + ns.verbose3 = False + print("WARNING: Disable --verbose3 because it's incompatible with " + "--huntrleaks: see http://bugs.python.org/issue27103", + file=sys.stderr) return ns From 193f7e094f070cecbc6faea6dffafb80ea9e7536 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 20 May 2017 10:23:31 +0300 Subject: [PATCH 0316/1345] [3.6] bpo-25794: Fix `type.__setattr__()` for non-interned attribute names. (GH-1652) (#1673) Based on patch by Eryk Sun. (cherry picked from commit d896985bb2de49046f9b6879e906d1e4db255e23) --- Lib/test/test_class.py | 27 +++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Objects/typeobject.c | 39 ++++++++++++++++++++++++++++++++++----- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4d554a397b4a51..ecc01f277954d5 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -568,5 +568,32 @@ class B(A): a = A(hash(A.f)^(-1)) hash(a.f) + def testSetattrWrapperNameIntern(self): + # Issue #25794: __setattr__ should intern the attribute name + class A: + pass + + def add(self, other): + return 'summa' + + name = str(b'__add__', 'ascii') # shouldn't be optimized + self.assertIsNot(name, '__add__') # not interned + type.__setattr__(A, name, add) + self.assertEqual(A() + 1, 'summa') + + name2 = str(b'__add__', 'ascii') + self.assertIsNot(name2, '__add__') + self.assertIsNot(name2, name) + type.__delattr__(A, name2) + with self.assertRaises(TypeError): + A() + 1 + + def testSetattrNonStringName(self): + class A: + pass + + with self.assertRaises(TypeError): + type.__setattr__(A, b'x', None) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 6336e67ee48edb..e1ea9df116b8e6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-25794: Fixed type.__setattr__() and type.__delattr__() for + non-interned attribute names. Based on patch by Eryk Sun. + - bpo-12414: sys.getsizeof() on a code object now returns the sizes which includes the code struct and sizes of objects which it references. Patch by Dong-hee Na. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 329261b037320a..271b93575c6dcf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3075,6 +3075,7 @@ type_getattro(PyTypeObject *type, PyObject *name) static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { + int res; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format( PyExc_TypeError, @@ -3082,9 +3083,35 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) type->tp_name); return -1; } - if (PyObject_GenericSetAttr((PyObject *)type, name, value) < 0) - return -1; - return update_slot(type, name); + if (PyUnicode_Check(name)) { + if (PyUnicode_CheckExact(name)) { + if (PyUnicode_READY(name) == -1) + return -1; + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; + } + } + else { + /* Will fail in _PyObject_GenericSetAttrWithDict. */ + Py_INCREF(name); + } + res = PyObject_GenericSetAttr((PyObject *)type, name, value); + if (res == 0) { + res = update_slot(type, name); + } + Py_DECREF(name); + return res; } extern void @@ -6929,7 +6956,7 @@ init_slotdefs(void) /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyUnicode_InternFromString(p->name); - if (!p->name_strobj) + if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) Py_FatalError("Out of memory interning slotdef names"); } slotdefs_initialized = 1; @@ -6954,6 +6981,9 @@ update_slot(PyTypeObject *type, PyObject *name) slotdef **pp; int offset; + assert(PyUnicode_CheckExact(name)); + assert(PyUnicode_CHECK_INTERNED(name)); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. This could possibly be unified with the update_subclasses() recursion below, but carefully: @@ -6964,7 +6994,6 @@ update_slot(PyTypeObject *type, PyObject *name) init_slotdefs(); pp = ptrs; for (p = slotdefs; p->name; p++) { - /* XXX assume name is interned! */ if (p->name_strobj == name) *pp++ = p; } From 564398af6ccb34d0db8b6e2537830eca285689e5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 20 May 2017 13:06:26 +0300 Subject: [PATCH 0317/1345] [3.6] bpo-27945: Fixed various segfaults with dict. (GH-1657) (#1677) Based on patches by Duane Griffin and Tim Mitchell. (cherry picked from commit 753bca3934a7618a4fa96e107ad1c5c18633a683) --- Lib/test/test_dict.py | 85 +++++++++++++++++++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 4 ++ Objects/dictobject.c | 69 ++++++++++++++++++++++------------- 4 files changed, 134 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 832bb9c8e2dbcc..8013f37c88da0a 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1085,6 +1085,91 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in issue 27945 part 3. + class X(): + def __del__(self): + dict_b.clear() + + def __eq__(self, other): + dict_a.clear() + return True + + def __hash__(self): + return 13 + + dict_a = {X(): 0} + dict_b = {X(): X()} + self.assertTrue(dict_a == dict_b) + + def test_fromkeys_operator_modifying_dict_operand(self): + # test fix for seg fault reported in issue 27945 part 4a. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1): 1, X(2): 2} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_fromkeys_operator_modifying_set_operand(self): + # test fix for seg fault reported in issue 27945 part 4b. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1), X(2)} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_dictitems_contains_use_after_free(self): + class X: + def __eq__(self, other): + d.clear() + return NotImplemented + + d = {0: set()} + (0, X()) in d.items() + + def test_init_use_after_free(self): + class X: + def __hash__(self): + pair[:] = [] + return 13 + + pair = [X(), 123] + dict([pair]) + + def test_oob_indexing_dictiter_iternextitem(self): + class X(int): + def __del__(self): + d.clear() + + d = {i: X(i) for i in range(8)} + + def iter_and_mutate(): + for result in d.items(): + if result[0] == 2: + d[2] = None # free d[2] --> X(2).__del__ was called + + self.assertRaises(RuntimeError, iter_and_mutate) + class CAPITest(unittest.TestCase): diff --git a/Misc/ACKS b/Misc/ACKS index ccc9d2af3149c0..7109ccbe851384 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -542,6 +542,7 @@ Tim Graham Kim Gräsman Nathaniel Gray Eddy De Greef +Duane Griffin Grant Griffin Andrea Griffini Duncan Grisby diff --git a/Misc/NEWS b/Misc/NEWS index e1ea9df116b8e6..d3c6868c4e5103 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-27945: Fixed various segfaults with dict when input collections are + mutated during searching, inserting or comparing. Based on patches by + Duane Griffin and Tim Mitchell. + - bpo-25794: Fixed type.__setattr__() and type.__delattr__() for non-interned attribute names. Based on patch by Eryk Sun. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 566d1a5ac8d893..b0f583a067b4e1 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1115,18 +1115,18 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) PyDictKeyEntry *ep, *ep0; Py_ssize_t hashpos, ix; + Py_INCREF(key); + Py_INCREF(value); if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp) < 0) - return -1; + goto Fail; } ix = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr, &hashpos); - if (ix == DKIX_ERROR) { - return -1; - } + if (ix == DKIX_ERROR) + goto Fail; assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); - Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); /* When insertion order is different from shared key, we can't share @@ -1135,10 +1135,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (_PyDict_HasSplitTable(mp) && ((ix >= 0 && *value_addr == NULL && mp->ma_used != ix) || (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); ix = DKIX_EMPTY; } @@ -1147,16 +1145,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) /* Insert into new slot. */ if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); } ep0 = DK_ENTRIES(mp->ma_keys); ep = &ep0[mp->ma_keys->dk_nentries]; dk_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - Py_INCREF(key); ep->me_key = key; ep->me_hash = hash; if (mp->ma_values) { @@ -1184,6 +1179,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) assert(_PyDict_CheckConsistency(mp)); Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + Py_DECREF(key); return 0; } @@ -1194,7 +1190,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) mp->ma_used++; mp->ma_version_tag = DICT_NEXT_VERSION(); assert(_PyDict_CheckConsistency(mp)); + Py_DECREF(key); return 0; + +Fail: + Py_DECREF(value); + Py_DECREF(key); + return -1; } /* @@ -2432,11 +2434,18 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) /* Update/merge with this (key, value) pair. */ key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); + Py_INCREF(key); + Py_INCREF(value); if (override || PyDict_GetItem(d, key) == NULL) { int status = PyDict_SetItem(d, key, value); - if (status < 0) + if (status < 0) { + Py_DECREF(key); + Py_DECREF(value); goto Fail; + } } + Py_DECREF(key); + Py_DECREF(value); Py_DECREF(fast); Py_DECREF(item); } @@ -2737,14 +2746,15 @@ dict_equal(PyDictObject *a, PyDictObject *b) bval = NULL; else bval = *vaddr; - Py_DECREF(key); if (bval == NULL) { + Py_DECREF(key); Py_DECREF(aval); if (PyErr_Occurred()) return -1; return 0; } cmp = PyObject_RichCompareBool(aval, bval, Py_EQ); + Py_DECREF(key); Py_DECREF(aval); if (cmp <= 0) /* error or not equal */ return cmp; @@ -3633,7 +3643,7 @@ PyTypeObject PyDictIterValue_Type = { static PyObject * dictiter_iternextitem(dictiterobject *di) { - PyObject *key, *value, *result = di->di_result; + PyObject *key, *value, *result; Py_ssize_t i, n; PyDictObject *d = di->di_dict; @@ -3674,20 +3684,25 @@ dictiter_iternextitem(dictiterobject *di) } di->di_pos = i+1; di->len--; - if (result->ob_refcnt == 1) { + Py_INCREF(key); + Py_INCREF(value); + result = di->di_result; + if (Py_REFCNT(result) == 1) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ Py_INCREF(result); - Py_DECREF(PyTuple_GET_ITEM(result, 0)); - Py_DECREF(PyTuple_GET_ITEM(result, 1)); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); } else { result = PyTuple_New(2); if (result == NULL) return NULL; + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -4180,6 +4195,7 @@ dictitems_iter(_PyDictViewObject *dv) static int dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { + int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) return 0; @@ -4193,7 +4209,10 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) return -1; return 0; } - return PyObject_RichCompareBool(value, found, Py_EQ); + Py_INCREF(found); + result = PyObject_RichCompareBool(value, found, Py_EQ); + Py_DECREF(found); + return result; } static PySequenceMethods dictitems_as_sequence = { From b5bf7e85b74070973bff3e69990f948a0ed5efdb Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 20 May 2017 23:23:03 -0700 Subject: [PATCH 0318/1345] bpo-29976: urllib.parse clarify '' in scheme values. (GH-984) (GH-1692) (cherry picked from commit 906f5330b9c9a74cad1cf27fddaf77e99dff9edd) --- Lib/urllib/parse.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 32de25b374da93..888247b8e90f56 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -38,29 +38,37 @@ "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' From cf5c1be8f6e0bc90280f66c65bb49808f01bfb3a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 21 May 2017 10:35:39 +0300 Subject: [PATCH 0319/1345] [3.6] bpo-30415: Add new tests for the fnmatch module. (GH-1684) (#1694) (cherry picked from commit 8175547) --- Lib/test/test_fnmatch.py | 64 ++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index fb7424624bb041..78245c3ab1a176 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -1,18 +1,19 @@ """Test cases for the fnmatch module.""" import unittest +import os from fnmatch import fnmatch, fnmatchcase, translate, filter class FnmatchTestCase(unittest.TestCase): - def check_match(self, filename, pattern, should_match=1, fn=fnmatch): + def check_match(self, filename, pattern, should_match=True, fn=fnmatch): if should_match: self.assertTrue(fn(filename, pattern), "expected %r to match pattern %r" % (filename, pattern)) else: - self.assertTrue(not fn(filename, pattern), + self.assertFalse(fn(filename, pattern), "expected %r not to match pattern %r" % (filename, pattern)) @@ -26,15 +27,15 @@ def test_fnmatch(self): check('abc', '*') check('abc', 'ab[cd]') check('abc', 'ab[!de]') - check('abc', 'ab[de]', 0) - check('a', '??', 0) - check('a', 'b', 0) + check('abc', 'ab[de]', False) + check('a', '??', False) + check('a', 'b', False) # these test that '\' is handled correctly in character sets; # see SF bug #409651 check('\\', r'[\]') check('a', r'[!\]') - check('\\', r'[!\]', 0) + check('\\', r'[!\]', False) # test that filenames with newlines in them are handled correctly. # http://bugs.python.org/issue6665 @@ -51,14 +52,38 @@ def test_mix_bytes_str(self): def test_fnmatchcase(self): check = self.check_match - check('AbC', 'abc', 0, fnmatchcase) - check('abc', 'AbC', 0, fnmatchcase) + check('abc', 'abc', True, fnmatchcase) + check('AbC', 'abc', False, fnmatchcase) + check('abc', 'AbC', False, fnmatchcase) + check('AbC', 'AbC', True, fnmatchcase) + + check('usr/bin', 'usr/bin', True, fnmatchcase) + check('usr\\bin', 'usr/bin', False, fnmatchcase) + check('usr/bin', 'usr\\bin', False, fnmatchcase) + check('usr\\bin', 'usr\\bin', True, fnmatchcase) def test_bytes(self): self.check_match(b'test', b'te*') self.check_match(b'test\xff', b'te*\xff') self.check_match(b'foo\nbar', b'foo*') + def test_case(self): + ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') + check = self.check_match + check('abc', 'abc') + check('AbC', 'abc', ignorecase) + check('abc', 'AbC', ignorecase) + check('AbC', 'AbC') + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + check('usr/bin', 'usr/bin') + check('usr\\bin', 'usr/bin', normsep) + check('usr/bin', 'usr\\bin', normsep) + check('usr\\bin', 'usr\\bin') + + class TranslateTestCase(unittest.TestCase): def test_translate(self): @@ -75,7 +100,28 @@ def test_translate(self): class FilterTestCase(unittest.TestCase): def test_filter(self): - self.assertEqual(filter(['a', 'b'], 'a'), ['a']) + self.assertEqual(filter(['Python', 'Ruby', 'Perl', 'Tcl'], 'P*'), + ['Python', 'Perl']) + self.assertEqual(filter([b'Python', b'Ruby', b'Perl', b'Tcl'], b'P*'), + [b'Python', b'Perl']) + + def test_mix_bytes_str(self): + self.assertRaises(TypeError, filter, ['test'], b'*') + self.assertRaises(TypeError, filter, [b'test'], '*') + + def test_case(self): + ignorecase = os.path.normcase('P') == os.path.normcase('p') + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.p*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.py']) + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.P*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.PL']) + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr/*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr/bin']) + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr\\*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr\\lib']) if __name__ == "__main__": From 0702cc01fa3dc52087a5d468fc5a4984f3697446 Mon Sep 17 00:00:00 2001 From: delirious-lettuce Date: Sun, 21 May 2017 15:26:01 -0600 Subject: [PATCH 0320/1345] [3.6] Fix typos in multiple `.rst` files (GH-1668) (#1702) (cherry picked from commit 3378b2062c7fbb77a9b5e6d315d6b94160fac69a) --- Doc/c-api/exceptions.rst | 8 ++++---- Doc/howto/logging-cookbook.rst | 2 +- Doc/library/cmd.rst | 4 ++-- Doc/library/email.compat32-message.rst | 2 +- Doc/library/email.contentmanager.rst | 4 ++-- Doc/library/email.errors.rst | 4 ++-- Doc/library/email.generator.rst | 4 ++-- Doc/library/email.headerregistry.rst | 2 +- Doc/library/email.message.rst | 10 +++++----- Doc/library/email.mime.rst | 2 +- Doc/library/email.parser.rst | 2 +- Doc/library/email.policy.rst | 6 +++--- Doc/library/exceptions.rst | 2 +- Doc/library/sunau.rst | 2 +- Doc/library/turtle.rst | 2 +- Doc/library/xml.dom.pulldom.rst | 2 +- Doc/reference/expressions.rst | 2 +- Lib/email/architecture.rst | 2 +- 18 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c75408db770a22..2bc1bd876a2fe2 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -295,7 +295,7 @@ an error value). :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard Python warning categories are available as global variables whose names are - enumerated at :ref:`standarwarningcategories`. + enumerated at :ref:`standardwarningcategories`. For information about warning control, see the documentation for the :mod:`warnings` module and the :option:`-W` option in the command line @@ -958,7 +958,7 @@ Notes: Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. _standarwarningcategories: +.. _standardwarningcategories: Standard Warning Categories =========================== @@ -971,7 +971,7 @@ the variables: .. index:: single: PyExc_Warning single: PyExc_BytesWarning - single: PyExc_DepricationWarning + single: PyExc_DeprecationWarning single: PyExc_FutureWarning single: PyExc_ImportWarning single: PyExc_PendingDeprecationWarning @@ -994,7 +994,7 @@ the variables: +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | +------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDepricationWarning`| :exc:`PendingDeprecationWarning`| | +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | +------------------------------------------+---------------------------------+----------+ diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index bb79bb1748fb9c..6498ea56957719 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1683,7 +1683,7 @@ Implementing structured logging ------------------------------- Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be cirumstances where you want to output +readily machine-parseable, there might be circumstances where you want to output messages in a structured format which *is* capable of being parsed by a program (without needing complex regular expressions to parse the log message). This is straightforward to achieve using the logging package. There are a number of diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index f40cfdfd592163..3b4a8ff440e7ae 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -266,10 +266,10 @@ immediate playback:: 'Draw circle with given radius an options extent and steps: CIRCLE 50' circle(*parse(arg)) def do_position(self, arg): - 'Print the current turle position: POSITION' + 'Print the current turtle position: POSITION' print('Current position is %d %d\n' % position()) def do_heading(self, arg): - 'Print the current turle heading in degrees: HEADING' + 'Print the current turtle heading in degrees: HEADING' print('Current heading is %d\n' % (heading(),)) def do_color(self, arg): 'Set the color: COLOR BLUE' diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index b0707646e64ac6..7e11782face074 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -67,7 +67,7 @@ Here are the methods of the :class:`Message` class: Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compabitility reasons, + *unixfrom* defaults to ``False``. For backward compatibility reasons, *maxheaderlen* defaults to ``0``, so if you want a different value you must override it explicitly (the value specified for *max_line_length* in the policy will be ignored by this method). The *policy* argument may be diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 57743d5a29f1dd..f56836ae2d2781 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -157,7 +157,7 @@ Currently the email package provides only one concrete content manager, MIME charset name, use the standard charset instead. If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to + encoding, and set the :mailheader:`Content-Transfer-Encoding` header to that value. Possible values for *cte* are ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be encoded in the specified encoding (for example, specifying a *cte* of @@ -203,5 +203,5 @@ Currently the email package provides only one concrete content manager, .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module ` diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 2d0d1923cd25c4..5838767b18f742 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -102,9 +102,9 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. return false even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - enocded bytes, the padding was not correct. Enough padding is added to + encoded bytes, the padding was not correct. Enough padding is added to perform the decode, but the resulting decoded bytes may be invalid. * :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - enocded bytes, characters outside the base64 alphebet were encountered. + encoded bytes, characters outside the base64 alphabet were encountered. The characters are ignored, but the resulting decoded bytes may be invalid. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index ab0fbc29d1ec85..1e64e1066c7da5 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -88,8 +88,8 @@ over channels that are not "8 bit clean". If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, transform parts with non-ASCII - :mailheader:`Cotnent-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatibile + :mailheader:`Content-Transfer-Encoding` + (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus rendering them RFC-compliant. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 2c830cfd81fb47..ce283c6b596cf2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -451,5 +451,5 @@ construct structured values to assign to specific headers. .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional module ` diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index d36e769f763cd8..261d0d62cfe618 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -52,7 +52,7 @@ message objects. .. class:: EmailMessage(policy=default) - If *policy* is specified use the rules it specifies to udpate and serialize + If *policy* is specified use the rules it specifies to update and serialize the representation of the message. If *policy* is not set, use the :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses @@ -63,7 +63,7 @@ message objects. Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compabitility + string. *unixfrom* defaults to ``False``. For backward compatibility with the base :class:`~email.message.Message` class *maxheaderlen* is accepted, but defaults to ``None``, which means that by default the line length is controlled by the @@ -213,7 +213,7 @@ message objects. del msg['subject'] msg['subject'] = 'Python roolz!' - If the :mod:`policy` defines certain haders to be unique (as the standard + If the :mod:`policy` defines certain headers to be unique (as the standard policies do), this method may raise a :exc:`ValueError` when an attempt is made to assign a value to such a header when one already exists. This behavior is intentional for consistency's sake, but do not depend on it @@ -558,7 +558,7 @@ message objects. the part a candidate match if the value of the header is ``inline``. If none of the candidates matches any of the preferences in - *preferneclist*, return ``None``. + *preferencelist*, return ``None``. Notes: (1) For most applications the only *preferencelist* combinations that really make sense are ``('plain',)``, ``('html', 'plain')``, and the @@ -746,6 +746,6 @@ message objects. .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d9dae9f0b993c4..f37f6aa28dec7d 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -242,7 +242,7 @@ Here are the classes: Unless the *_charset* argument is explicitly set to ``None``, the MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Endcoding` + with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` header. This means that a subsequent ``set_payload`` call will not result in an encoded payload, even if a charset is passed in the ``set_payload`` command. You can "reset" this behavior by deleting the diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index c323ebc6401b99..dea409d223da4c 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -139,7 +139,7 @@ message body, instead setting the payload to the raw body. .. class:: BytesParser(_class=None, *, policy=policy.compat32) Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and sematnics as the *_factory* + arguments have the same meaning and semantics as the *_factory* and *policy* arguments of :class:`BytesFeedParser`. Note: **The policy keyword should always be specified**; The default will diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8a418778b8e0fc..8e7076259810f5 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -276,7 +276,7 @@ added matters. To illustrate:: Called when a header is added to an :class:`~email.message.EmailMessage` or :class:`~email.message.Message` object. If the returned value is not ``0`` or ``None``, and there are already a number of headers with the - name *name* greather than or equal to the value returned, a + name *name* greater than or equal to the value returned, a :exc:`ValueError` is raised. Because the default behavior of ``Message.__setitem__`` is to append the @@ -533,7 +533,7 @@ more closely to the RFCs relevant to their domains. The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP trasmission if the + words in the headers. Should only be used for SMTP transmission if the sender or recipient addresses have non-ASCII characters (the :meth:`smtplib.SMTP.send_message` method handles this automatically). @@ -647,5 +647,5 @@ The header objects and their attributes are described in .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional feature `. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a428f5165fc8d2..a6b20a5ac95b1b 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -243,7 +243,7 @@ The following exceptions are the exceptions that are usually raised. .. note:: - It should not be used to indicate that an operater or method is not + It should not be used to indicate that an operator or method is not meant to be supported at all -- in that case either leave the operator / method undefined or, if a subclass, set it to :data:`None`. diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 1ecc7a7cf92bd0..c8357e4fcc85e2 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -118,7 +118,7 @@ AU_read objects, as returned by :func:`.open` above, have the following methods: .. method:: AU_read.getnchannels() - Returns number of audio channels (1 for mone, 2 for stereo). + Returns number of audio channels (1 for mono, 2 for stereo). .. method:: AU_read.getsampwidth() diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 1986972549c503..31761be02b78ee 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1797,7 +1797,7 @@ Input methods :param prompt: string Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, propmt is a text mostly describing + the title of the dialog window, prompt is a text mostly describing what information to input. Return the string input. If the dialog is canceled, return ``None``. :: diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index b50255434de945..5c0f469ad7a5cf 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -108,7 +108,7 @@ DOMEventStream Objects :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals :data:`CHARACTERS`. - The current node does not contain informations about its children, unless + The current node does not contain information about its children, unless :func:`expandNode` is called. .. method:: expandNode(node) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 43d2690cbba91b..d92be975aacef3 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -636,7 +636,7 @@ which are used to control the execution of a generator function. without yielding another value, an :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitalbe is run that exception + raises a different exception, then when the awaitable is run that exception propagates to the caller of the awaitable. .. index:: exception: GeneratorExit diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst index 78572ae63b4d2b..fcd10bde1325bb 100644 --- a/Lib/email/architecture.rst +++ b/Lib/email/architecture.rst @@ -66,7 +66,7 @@ data payloads. Message Lifecycle ----------------- -The general lifecyle of a message is: +The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be From 54af41d42eebbe4c6afe6b34ebb0fb550de1e7ba Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 23 May 2017 01:03:00 +0800 Subject: [PATCH 0321/1345] bpo-30003: Fix handling escape characters in HZ codec (#1556) (#1719) --- Lib/test/test_codecencodings_cn.py | 4 ++++ Misc/NEWS | 3 +++ Modules/cjkcodecs/_codecs_cn.c | 25 ++++++++++++------------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index c8a410c2e0398c..2a450718d5aae5 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -86,6 +86,10 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode ) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index d3c6868c4e5103..ae15eefcd46ad4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -43,6 +43,9 @@ Core and Builtins Library ------- +- bpo-30003: Fix handling escape characters in HZ codec. Based on patch + by Ma Lin. + - bpo-30301: Fix AttributeError when using SimpleQueue.empty() under *spawn* and *forkserver* start methods. diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index bda175c55d1323..1fcc220b8db0f4 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -350,15 +350,17 @@ ENCODER(hz) DBCHAR code; if (c < 0x80) { - if (state->i == 0) { - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); - } - else { - WRITEBYTE3('~', '}', (unsigned char)c); - NEXT(1, 3); + if (state->i) { + WRITEBYTE2('~', '}'); + NEXT_OUT(2); state->i = 0; } + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); + if (c == '~') { + WRITEBYTE1('~'); + NEXT_OUT(1); + } continue; } @@ -409,17 +411,14 @@ DECODER(hz) unsigned char c2 = INBYTE2; REQUIRE_INBUF(2); - if (c2 == '~') { + if (c2 == '~' && state->i == 0) OUTCHAR('~'); - NEXT_IN(2); - continue; - } else if (c2 == '{' && state->i == 0) state->i = 1; /* set GB */ + else if (c2 == '\n' && state->i == 0) + ; /* line-continuation */ else if (c2 == '}' && state->i == 1) state->i = 0; /* set ASCII */ - else if (c2 == '\n') - ; /* line-continuation */ else return 1; NEXT_IN(2); From 3d3b189f8ef321ccfb144a46d50892939d6004aa Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 22 May 2017 14:36:19 -0700 Subject: [PATCH 0322/1345] [3.6] bpo-30372: Clarify that '__builtins__' is a CPython Implementation detail (GH-1725) (GH-1727) (cherry picked from commit 43c8a9e39bb2d4f25ed7ae17a55f34e375d1868b) --- Doc/reference/executionmodel.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index f0dbbd1edf6321..d08abdf3343fe4 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -164,15 +164,6 @@ Builtins and restricted execution .. index:: pair: restricted; execution -The builtins namespace associated with the execution of a code block is actually -found by looking up the name ``__builtins__`` in its global namespace; this -should be a dictionary or a module (in the latter case the module's dictionary -is used). By default, when in the :mod:`__main__` module, ``__builtins__`` is -the built-in module :mod:`builtins`; when in any other module, -``__builtins__`` is an alias for the dictionary of the :mod:`builtins` module -itself. ``__builtins__`` can be set to a user-created dictionary to create a -weak form of restricted execution. - .. impl-detail:: Users should not touch ``__builtins__``; it is strictly an implementation @@ -180,6 +171,15 @@ weak form of restricted execution. :keyword:`import` the :mod:`builtins` module and modify its attributes appropriately. +The builtins namespace associated with the execution of a code block +is actually found by looking up the name ``__builtins__`` in its +global namespace; this should be a dictionary or a module (in the +latter case the module's dictionary is used). By default, when in the +:mod:`__main__` module, ``__builtins__`` is the built-in module +:mod:`builtins`; when in any other module, ``__builtins__`` is an +alias for the dictionary of the :mod:`builtins` module itself. + + .. _dynamic-features: Interaction with dynamic features From 1398b1bc7d80df5bde17041e7ec0a3bdbf54b19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 22 May 2017 16:35:15 -0700 Subject: [PATCH 0323/1345] [3.6] Make rb'' strings work in lib2to3 (GH-1724) (#1730) This partially solves bpo-23894. (cherry picked from commit 0c4aca54dcf0c54f299c78aa71fe8f48ff04f9d9) --- Lib/lib2to3/pgen2/tokenize.py | 23 ++++++++++++++++------- Lib/lib2to3/tests/test_parser.py | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index fba0fa2c00dccd..9a0cc1ea4086c7 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -74,10 +74,11 @@ def maybe(*choices): return group(*choices) + '?' Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""') +_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?" +Triple = group(_litprefix + "'''", _litprefix + '"""') # Single-line ' or " string. -String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get @@ -95,9 +96,9 @@ def maybe(*choices): return group(*choices) + '?' Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) @@ -111,6 +112,7 @@ def maybe(*choices): return group(*choices) + '?' "b'''": single3prog, 'b"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, + "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog, @@ -120,6 +122,9 @@ def maybe(*choices): return group(*choices) + '?' "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "rB'''": single3prog, 'rB"""': double3prog, + "Rb'''": single3prog, 'Rb"""': double3prog, + "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, 'b': None, 'B': None} @@ -132,7 +137,9 @@ def maybe(*choices): return group(*choices) + '?' "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""',): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "rb'''", 'rb"""', "Rb'''", 'Rb"""', + "rB'''", 'rB"""', "RB'''", 'RB"""',): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', @@ -142,7 +149,9 @@ def maybe(*choices): return group(*choices) + '?' "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', ): + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',): single_quoted[t] = t tabsize = 8 diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 9a969e8ec04741..c79611db56522a 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -320,6 +320,7 @@ def test_5(self): def test_6(self): self.validate("lst: List[int] = []") + class TestExcept(GrammarTest): def test_new(self): s = """ @@ -338,6 +339,26 @@ def test_old(self): self.validate(s) +class TestStringLiterals(GrammarTest): + prefixes = ("'", '"', + "r'", 'r"', "R'", 'R"', + "u'", 'u"', "U'", 'U"', + "b'", 'b"', "B'", 'B"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',) + + def test_lit(self): + for pre in self.prefixes: + single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) + self.validate(single) + triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) + self.validate(triple) + + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms class TestSetLiteral(GrammarTest): def test_1(self): From d29feccec3ce3dcd9ab3100f8956171c70ec3027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 22 May 2017 22:23:05 -0700 Subject: [PATCH 0324/1345] [3.6] bpo-30395 _PyGILState_Reinit deadlock fix (GH-1734) (#1740) head_lock could be held by another thread when fork happened. We should reset it to avoid deadlock. (cherry picked from commit f82c951d1c5416f3550d544e50ff5662d3836e73) --- Misc/ACKS | 1 + Python/pystate.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Misc/ACKS b/Misc/ACKS index 7109ccbe851384..b8bac248d98867 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -552,6 +552,7 @@ Eric Groo Daniel Andrade Groppe Dag Gruneau Filip Gruszczyński +Andrii Grynenko Grzegorz Grzywacz Thomas Guettler Yuyang Guo diff --git a/Python/pystate.c b/Python/pystate.c index 65c244e6f73617..ccb0092c42b3a4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -743,6 +743,10 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { +#ifdef WITH_THREAD + head_mutex = NULL; + HEAD_INIT(); +#endif PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_delete_key(autoTLSkey); if ((autoTLSkey = PyThread_create_key()) == -1) From e8412e684ed741df246e8430f4911b31b0d8be1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 22 May 2017 22:23:29 -0700 Subject: [PATCH 0325/1345] [3.6] bpo-23894: make lib2to3 recognize f-strings (GH-1733) (#1737) Note: this doesn't unpack f-strings into the underlying JoinedStr AST. Ideally we'd fully implement JoinedStr here but given its additional complexity, I think this is worth bandaiding as is. This unblocks tools like https://github.com/google/yapf to format 3.6 syntax using f-strings. (cherry picked from commit 1b9530c536664276ce866ae602ce04adce0810e1) --- Lib/lib2to3/pgen2/tokenize.py | 7 ++++++- Lib/lib2to3/tests/test_parser.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 9a0cc1ea4086c7..45afc5f4e53fcf 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -74,7 +74,7 @@ def maybe(*choices): return group(*choices) + '?' Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?" +_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?" Triple = group(_litprefix + "'''", _litprefix + '"""') # Single-line ' or " string. String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", @@ -110,12 +110,14 @@ def maybe(*choices): return group(*choices) + '?' "r'''": single3prog, 'r"""': double3prog, "u'''": single3prog, 'u"""': double3prog, "b'''": single3prog, 'b"""': double3prog, + "f'''": single3prog, 'f"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog, + "F'''": single3prog, 'F"""': double3prog, "uR'''": single3prog, 'uR"""': double3prog, "Ur'''": single3prog, 'Ur"""': double3prog, "UR'''": single3prog, 'UR"""': double3prog, @@ -127,6 +129,7 @@ def maybe(*choices): return group(*choices) + '?' "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, + 'f': None, 'F': None, 'b': None, 'B': None} triple_quoted = {} @@ -134,6 +137,7 @@ def maybe(*choices): return group(*choices) + '?' "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""', + "f'''", 'f"""', "F'''", 'F"""', "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', @@ -146,6 +150,7 @@ def maybe(*choices): return group(*choices) + '?' "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index c79611db56522a..3f7ab9714e38f9 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -344,6 +344,7 @@ class TestStringLiterals(GrammarTest): "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', From 07a40a16238995e2b6b9ab1d5fc3a30bcd63d955 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 23 May 2017 07:25:17 +0200 Subject: [PATCH 0326/1345] [3.6] Add --with-assertions configure flag to enable C assertions(GH-1731) (#1739) Defaults to 'no', but as before assertions are implied by --with-pydebug.. (cherry picked from commit ddbfa2c35b298a56f0b813656745bca9cb7334f1) --- Misc/NEWS | 4 ++++ configure | 31 ++++++++++++++++++++++++++++++- configure.ac | 23 ++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index ae15eefcd46ad4..0dda4fa5122948 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -154,6 +154,10 @@ Library Build ----- +- bpo-29941: Add ``--with-assertions`` configure flag to explicitly enable + C ``assert()`` checks. Defaults to off. ``--with-pydebug`` implies + ``--with-assertions``. + - bpo-28787: Fix out-of-tree builds of Python when configured with ``--with--dtrace``. diff --git a/configure b/configure index 97d3301f8e1891..09a94624c1b2b4 100755 --- a/configure +++ b/configure @@ -818,6 +818,7 @@ with_suffix enable_shared enable_profiling with_pydebug +with_assertions enable_optimizations with_lto with_hash_algorithm @@ -1513,6 +1514,7 @@ Optional Packages: compiler --with-suffix=.exe set executable suffix --with-pydebug build with Py_DEBUG defined + --with-assertions build with C assertions enabled --with-lto Enable Link Time Optimization in PGO builds. Disabled by default. --with-hash-algorithm=[fnv|siphash24] @@ -6515,6 +6517,33 @@ $as_echo "no" >&6; } fi +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-assertions" >&5 +$as_echo_n "checking for --with-assertions... " >&6; } + +# Check whether --with-assertions was given. +if test "${with_assertions+set}" = set; then : + withval=$with_assertions; +if test "$withval" != no +then + assertions='true' +fi +fi + +if test "$assertions" = 'true'; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + { $as_echo "$as_me:${as_lineno-$LINENO}: result: implied by --with-pydebug" >&5 +$as_echo "implied by --with-pydebug" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + # Enable optimization flags @@ -7424,7 +7453,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" diff --git a/configure.ac b/configure.ac index 8abf2f96cc45ff..146ae22a632a59 100644 --- a/configure.ac +++ b/configure.ac @@ -1258,6 +1258,27 @@ else AC_MSG_RESULT(no); Py_DEBUG='false' fi], [AC_MSG_RESULT(no)]) +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +AC_MSG_CHECKING(for --with-assertions) +AC_ARG_WITH(assertions, + AC_HELP_STRING([--with-assertions], [build with C assertions enabled]), +[ +if test "$withval" != no +then + assertions='true' +fi], +[]) +if test "$assertions" = 'true'; then + AC_MSG_RESULT(yes) +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + AC_MSG_RESULT(implied by --with-pydebug) +else + AC_MSG_RESULT(no) +fi + # Enable optimization flags AC_SUBST(DEF_MAKE_ALL_RULE) AC_SUBST(DEF_MAKE_RULE) @@ -1855,7 +1876,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" From 6ef0882303cff24c58785bb1082250088266886d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 22 May 2017 22:27:02 -0700 Subject: [PATCH 0327/1345] [3.6] bpo-21056: Document return type of next method of csv reader (GH-146) (#1749) (cherry picked from commit d618c8c6d31b9b288f8a070417683974eb98e3ba) --- Doc/library/csv.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 52a8a310ec9d62..43714f7479283d 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -401,8 +401,10 @@ Reader objects (:class:`DictReader` instances and objects returned by the .. method:: csvreader.__next__() - Return the next row of the reader's iterable object as a list, parsed according - to the current dialect. Usually you should call this as ``next(reader)``. + Return the next row of the reader's iterable object as a list (if the object + was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` + instance), parsed according to the current dialect. Usually you should call + this as ``next(reader)``. Reader objects have the following public attributes: From 762add7877258b62b38f0cff87159179faa471dc Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Tue, 23 May 2017 11:44:02 +0530 Subject: [PATCH 0328/1345] Fix spelling mistakes in tkinter.py (#1751) Ran the docstrings through spell checker, and fixed spelling issues. --- Lib/tkinter/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index ee2415da72a01e..deea791831ed66 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1004,7 +1004,7 @@ def winfo_ismapped(self): return self.tk.getint( self.tk.call('winfo', 'ismapped', self._w)) def winfo_manager(self): - """Return the window mananger name for this widget.""" + """Return the window manager name for this widget.""" return self.tk.call('winfo', 'manager', self._w) def winfo_name(self): """Return the name of this widget.""" @@ -1679,7 +1679,7 @@ def image_names(self): return self.tk.splitlist(self.tk.call('image', 'names')) def image_types(self): - """Return a list of all available image types (e.g. phote bitmap).""" + """Return a list of all available image types (e.g. photo bitmap).""" return self.tk.splitlist(self.tk.call('image', 'types')) @@ -1818,7 +1818,7 @@ def wm_focusmodel(self, model=None): return self.tk.call('wm', 'focusmodel', self._w, model) focusmodel = wm_focusmodel def wm_forget(self, window): # new in Tk 8.5 - """The window will be unmappend from the screen and will no longer + """The window will be unmapped from the screen and will no longer be managed by wm. toplevel windows will be treated like frame windows once they are no longer managed by wm, however, the menu option configuration will be remembered and the menus will return @@ -2527,7 +2527,7 @@ def find_closest(self, x, y, halo=None, start=None): """Return item which is closest to pixel at X, Y. If several match take the top-most. All items closer than HALO are considered overlapping (all are - closests). If START is specified the next below this tag is taken.""" + closest). If START is specified the next below this tag is taken.""" return self.find('closest', x, y, halo, start) def find_enclosed(self, x1, y1, x2, y2): """Return all items in rectangle defined @@ -2587,7 +2587,7 @@ def postscript(self, cnf={}, **kw): """Print the contents of the canvas to a postscript file. Valid options: colormap, colormode, file, fontmap, height, pageanchor, pageheight, pagewidth, pagex, pagey, - rotate, witdh, x, y.""" + rotate, width, x, y.""" return self.tk.call((self._w, 'postscript') + self._options(cnf, kw)) def tag_raise(self, *args): @@ -3522,7 +3522,7 @@ def height(self): return self.tk.getint( self.tk.call('image', 'height', self.name)) def type(self): - """Return the type of the imgage, e.g. "photo" or "bitmap".""" + """Return the type of the image, e.g. "photo" or "bitmap".""" return self.tk.call('image', 'type', self.name) def width(self): """Return the width of the image.""" From 70705e3cdfff6a2c1211456afce99afc3d9ca59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 23 May 2017 00:36:23 -0700 Subject: [PATCH 0329/1345] [3.6] call remove_done_callback in finally section (GH-1688) (#1755) (cherry picked from commit 21b3e04c13212b29e8c35ffc36eed8603fde08f4) --- Lib/asyncio/base_events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 0df58c5f873d88..a4967b854c5e68 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -459,7 +459,8 @@ def run_until_complete(self, future): # local task. future.exception() raise - future.remove_done_callback(_run_until_complete_cb) + finally: + future.remove_done_callback(_run_until_complete_cb) if not future.done(): raise RuntimeError('Event loop stopped before Future completed.') From 9bcf81da1436468c3c5e500ae36678a2ad675e32 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Tue, 23 May 2017 09:57:04 -0700 Subject: [PATCH 0330/1345] [3.6] bpo-30376: Update outdated WindowObject references (GH-1630) (#1746) (cherry picked from commit 93fc20b73eea3da0b6305aaee951e5dd22d5c408) --- Doc/library/curses.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d51085506c2a96..3442e4e75a3e45 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -290,8 +290,8 @@ The module :mod:`curses` defines the following functions: .. function:: initscr() - Initialize the library. Return a :class:`WindowObject` which represents the - whole screen. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. .. note:: @@ -383,8 +383,8 @@ The module :mod:`curses` defines the following functions: .. function:: newwin(nlines, ncols) newwin(nlines, ncols, begin_y, begin_x) - Return a new window, whose left-upper corner is at ``(begin_y, begin_x)``, and - whose height/width is *nlines*/*ncols*. + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. By default, the window will extend from the specified position to the lower right corner of the screen. @@ -1679,10 +1679,10 @@ You can instantiate a :class:`Textbox` object as follows: .. class:: Textbox(win) Return a textbox widget object. The *win* argument should be a curses - :class:`WindowObject` in which the textbox is to be contained. The edit cursor - of the textbox is initially located at the upper left hand corner of the - containing window, with coordinates ``(0, 0)``. The instance's - :attr:`stripspaces` flag is initially on. + :ref:`window ` object in which the textbox is to + be contained. The edit cursor of the textbox is initially located at the + upper left hand corner of the containing window, with coordinates ``(0, 0)``. + The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: From c3454f0e79b35fb81b0426cfac4b801d4495b8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Wed, 24 May 2017 20:59:06 +0200 Subject: [PATCH 0331/1345] bpo-30394: Fix a socket leak in smtplib.SMTP.__init__() (#1700) (#1790) --- Lib/smtplib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index f7c2c77ab42075..5e422b704ad4dc 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -250,6 +250,7 @@ def __init__(self, host='', port=0, local_hostname=None, if host: (code, msg) = self.connect(host, port) if code != 220: + self.close() raise SMTPConnectError(code, msg) if local_hostname is not None: self.local_hostname = local_hostname From a2a9984a278d2ee945e91ce13e000e571794fc24 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 24 May 2017 16:11:01 -0500 Subject: [PATCH 0332/1345] [3.6] bpo-30160: Clarify intended usage of wfile (gh-1300) (GH-1793) The library does not enforce compliance with the HTTP protocol, so violations are not technically disallowed. Extend the stream's description to avoid suggesting that intentional protocol violations are not supported. (cherry picked from commit a083c8e) --- Doc/library/http.server.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fb5c1df611d8f2..b29020bc7ca5ca 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -105,7 +105,8 @@ of which this module provides three different variants: Contains the output stream for writing a response back to the client. Proper adherence to the HTTP protocol must be used when writing to - this stream. + this stream in order to achieve successful interoperation with HTTP + clients. .. versionchanged:: 3.6 This is an :class:`io.BufferedIOBase` stream. From 89a310264000a613b7f6abd5916946aef09ae7d2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 25 May 2017 14:18:55 +0300 Subject: [PATCH 0333/1345] [3.6] bpo-29104: Fixed parsing backslashes in f-strings. (GH-490) (#1812) (cherry picked from commit 0cd7a3f) --- Lib/test/test_fstring.py | 14 +++++++++++ Misc/NEWS | 2 ++ Python/ast.c | 53 ++++++++++++++++++++++++---------------- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 708ed25b526b74..25730029ae76f1 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -361,6 +361,20 @@ def test_backslashes_in_string_part(self): self.assertEqual(f'2\x203', '2 3') self.assertEqual(f'\x203', ' 3') + with self.assertWarns(DeprecationWarning): # invalid escape sequence + value = eval(r"f'\{6*7}'") + self.assertEqual(value, '\\42') + self.assertEqual(f'\\{6*7}', '\\42') + self.assertEqual(fr'\{6*7}', '\\42') + + AMPERSAND = 'spam' + # Get the right unicode character (&), or pick up local variable + # depending on the number of backslashes. + self.assertEqual(f'\N{AMPERSAND}', '&') + self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam') + self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') + self.assertEqual(f'\\\N{AMPERSAND}', '\\&') + def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. diff --git a/Misc/NEWS b/Misc/NEWS index 0dda4fa5122948..09eb4a76df032f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-29104: Fixed parsing backslashes in f-strings. + - bpo-27945: Fixed various segfaults with dict when input collections are mutated during searching, inserting or comparing. Based on patches by Duane Griffin and Tim Mitchell. diff --git a/Python/ast.c b/Python/ast.c index 217ea14bf310be..ed05a1e53bdb5e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4170,9 +4170,11 @@ decode_unicode_with_escapes(struct compiling *c, const node *n, const char *s, while (s < end) { if (*s == '\\') { *p++ = *s++; - if (*s & 0x80) { + if (s >= end || *s & 0x80) { strcpy(p, "u005c"); p += 5; + if (s >= end) + break; } } if (*s & 0x80) { /* XXX inefficient */ @@ -4325,30 +4327,37 @@ fstring_find_literal(const char **str, const char *end, int raw, brace (which isn't part of a unicode name escape such as "\N{EULER CONSTANT}"), or the end of the string. */ - const char *literal_start = *str; - const char *literal_end; - int in_named_escape = 0; + const char *s = *str; + const char *literal_start = s; int result = 0; assert(*literal == NULL); - for (; *str < end; (*str)++) { - char ch = **str; - if (!in_named_escape && ch == '{' && (*str)-literal_start >= 2 && - *(*str-2) == '\\' && *(*str-1) == 'N') { - in_named_escape = 1; - } else if (in_named_escape && ch == '}') { - in_named_escape = 0; - } else if (ch == '{' || ch == '}') { + while (s < end) { + char ch = *s++; + if (!raw && ch == '\\' && s < end) { + ch = *s++; + if (ch == 'N') { + if (s < end && *s++ == '{') { + while (s < end && *s++ != '}') { + } + continue; + } + break; + } + if (ch == '{' && warn_invalid_escape_sequence(c, n, ch) < 0) { + return -1; + } + } + if (ch == '{' || ch == '}') { /* Check for doubled braces, but only at the top level. If we checked at every level, then f'{0:{3}}' would fail with the two closing braces. */ if (recurse_lvl == 0) { - if (*str+1 < end && *(*str+1) == ch) { + if (s < end && *s == ch) { /* We're going to tell the caller that the literal ends here, but that they should continue scanning. But also skip over the second brace when we resume scanning. */ - literal_end = *str+1; - *str += 2; + *str = s + 1; result = 1; goto done; } @@ -4356,6 +4365,7 @@ fstring_find_literal(const char **str, const char *end, int raw, /* Where a single '{' is the start of a new expression, a single '}' is not allowed. */ if (ch == '}') { + *str = s - 1; ast_error(c, n, "f-string: single '}' is not allowed"); return -1; } @@ -4363,21 +4373,22 @@ fstring_find_literal(const char **str, const char *end, int raw, /* We're either at a '{', which means we're starting another expression; or a '}', which means we're at the end of this f-string (for a nested format_spec). */ + s--; break; } } - literal_end = *str; - assert(*str <= end); - assert(*str == end || **str == '{' || **str == '}'); + *str = s; + assert(s <= end); + assert(s == end || *s == '{' || *s == '}'); done: - if (literal_start != literal_end) { + if (literal_start != s) { if (raw) *literal = PyUnicode_DecodeUTF8Stateful(literal_start, - literal_end-literal_start, + s - literal_start, NULL, NULL); else *literal = decode_unicode_with_escapes(c, n, literal_start, - literal_end-literal_start); + s - literal_start); if (!*literal) return -1; } From f43b293f2fee91578e28c7aa566510a0cd6e33cb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 25 May 2017 15:32:08 +0300 Subject: [PATCH 0334/1345] [3.6] bpo-29943: Do not replace the function PySlice_GetIndicesEx() with a macro (GH-1049) (#1813) if Py_LIMITED_API is not defined. (cherry picked from commit 49a9059) --- Include/sliceobject.h | 2 ++ Misc/NEWS | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 36263544607096..579ac073d0f241 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,11 +45,13 @@ PyAPI_FUNC(int) PySlice_GetIndicesEx(PyObject *r, Py_ssize_t length, Py_ssize_t *step, Py_ssize_t *slicelength); #if !defined(Py_LIMITED_API) || (Py_LIMITED_API+0 >= 0x03050400 && Py_LIMITED_API+0 < 0x03060000) || Py_LIMITED_API+0 >= 0x03060100 +#ifdef Py_LIMITED_API #define PySlice_GetIndicesEx(slice, length, start, stop, step, slicelen) ( \ PySlice_Unpack((slice), (start), (stop), (step)) < 0 ? \ ((*(slicelen) = 0), -1) : \ ((*(slicelen) = PySlice_AdjustIndices((length), (start), (stop), *(step))), \ 0)) +#endif PyAPI_FUNC(int) PySlice_Unpack(PyObject *slice, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, diff --git a/Misc/NEWS b/Misc/NEWS index 09eb4a76df032f..58a2ee0c3e53a7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -153,6 +153,13 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() no longer replaced with a macro + if Py_LIMITED_API is not set. + + Build ----- From 2783cc42629b9445ea848ce36bbf213ef7789271 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 25 May 2017 16:57:46 +0200 Subject: [PATCH 0335/1345] [3.6] bpo-30414: multiprocessing.Queue._feed do not break from main loop on exc (GH-1683) (#1815) * bpo-30414: multiprocesing.Queue._feed do not break from main loop on exc Queue background running thread was not handling exceptions correctly. Any exception occurred inside thread (putting unpickable object) cause feeder to finish running. After that every message put into queue is silently ignored. * bpo-30414: multiprocesing.Queue._feed do not break from main loop on exc Queue background running thread was not handling exceptions correctly. Any exception occurred inside thread (putting unpickable object) cause feeder to finish running. After that every message put into queue is silently ignored. (cherry picked from commit bc50f03db4f58c869b78e98468e374d7e61f1227) --- Lib/multiprocessing/queues.py | 22 ++++++++++------------ Lib/test/_test_multiprocessing.py | 14 ++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index a4f4ef8b7b5334..7f77837a74acb6 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -221,8 +221,8 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): else: wacquire = None - try: - while 1: + while 1: + try: nacquire() try: if not buffer: @@ -249,21 +249,19 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): wrelease() except IndexError: pass - except Exception as e: - if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - return - # Since this runs in a daemon thread the resources it uses - # may be become unusable while the process is cleaning up. - # We ignore errors which happen after the process has - # started to cleanup. - try: + except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return + # Since this runs in a daemon thread the resources it uses + # may be become unusable while the process is cleaning up. + # We ignore errors which happen after the process has + # started to cleanup. if is_exiting(): info('error in queue thread: %s', e) + return else: import traceback traceback.print_exc() - except Exception: - pass _sentinel = object() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 3eb83c59ab17d7..f1f93674935e7a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -752,6 +752,20 @@ def test_timeout(self): # Windows (usually 15.6 ms) self.assertGreaterEqual(delta, 0.170) + def test_queue_feeder_donot_stop_onexc(self): + # bpo-30414: verify feeder handles exceptions correctly + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + def __reduce__(self): + raise AttributeError + with test.support.captured_stderr(): + q = self.Queue() + q.put(NotSerializable()) + q.put(True) + self.assertTrue(q.get(timeout=0.1)) + # # # diff --git a/Misc/NEWS b/Misc/NEWS index 58a2ee0c3e53a7..bdaf7aaf70a0bf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-30414: multiprocessing.Queue._feed background running + thread do not break from main loop on exception. + - bpo-30003: Fix handling escape characters in HZ codec. Based on patch by Ma Lin. From 584ef2aecfdaaf753e3959a8ceb0ea745189ddeb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 27 May 2017 16:37:11 +0300 Subject: [PATCH 0336/1345] [3.6] Fix the signature of JSONDecodeError (no end parameter). (GH-1827) (#1828) Unlikely to the simplejson module, json.JSONDecodeError doesn't accept the end argument. (cherry picked from commit 5becf38) --- Doc/library/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8103c614aaf450..fdbdcb169fab09 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -503,7 +503,7 @@ Encoders and Decoders Exceptions ---------- -.. exception:: JSONDecodeError(msg, doc, pos, end=None) +.. exception:: JSONDecodeError(msg, doc, pos) Subclass of :exc:`ValueError` with the following additional attributes: From b52c68a5a3f546cbbe1589f8bb3e51bfd55a6c15 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 27 May 2017 16:37:40 +0300 Subject: [PATCH 0337/1345] [3.6] bpo-30398: Add a docstring for re.error. (GH-1647) (#1830) Also document that some attributes may be None. (cherry picked from commit 12d6b5d) --- Doc/library/re.rst | 6 +++--- Lib/sre_constants.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 944c6fb10d6c25..5ae304d25e1399 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -828,15 +828,15 @@ form. .. attribute:: pos - The index of *pattern* where compilation failed. + The index in *pattern* where compilation failed (may be ``None``). .. attribute:: lineno - The line corresponding to *pos*. + The line corresponding to *pos* (may be ``None``). .. attribute:: colno - The column corresponding to *pos*. + The column corresponding to *pos* (may be ``None``). .. versionchanged:: 3.5 Added additional attributes. diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index fc684ae96fd30a..a6e8a1f08d68a8 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -21,6 +21,17 @@ # should this really be here? class error(Exception): + """Exception raised for invalid regular expressions. + + Attributes: + + msg: The unformatted error message + pattern: The regular expression pattern + pos: The index in the pattern where compilation failed (may be None) + lineno: The line corresponding to pos (may be None) + colno: The column corresponding to pos (may be None) + """ + def __init__(self, msg, pattern=None, pos=None): self.msg = msg self.pattern = pattern From 94d8261d1ca3f0c551d9d43a4db342d806af7a6a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 27 May 2017 07:20:24 -0700 Subject: [PATCH 0338/1345] =?UTF-8?q?[3.6]=20bpo-29960=20=5Frandom.Random?= =?UTF-8?q?=20corrupted=20on=20exception=20in=20setstate().=20=E2=80=A6=20?= =?UTF-8?q?(#1287)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 9616a82e7802241a4b74cf7ae38d43c37bf66e48) --- Lib/test/test_random.py | 5 +++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_randommodule.c | 5 ++++- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5b6a4f06baaf41..03adc0a72294ab 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -427,6 +427,7 @@ def test_setstate_first_arg(self): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) def test_setstate_middle_arg(self): + start_state = self.gen.getstate() # Wrong type, s/b tuple self.assertRaises(TypeError, self.gen.setstate, (2, None, None)) # Wrong length, s/b 625 @@ -440,6 +441,10 @@ def test_setstate_middle_arg(self): self.gen.setstate((2, (1,)*624+(625,), None)) with self.assertRaises((ValueError, OverflowError)): self.gen.setstate((2, (1,)*624+(-1,), None)) + # Failed calls to setstate() should not have changed the state. + bits100 = self.gen.getrandbits(100) + self.gen.setstate(start_state) + self.assertEqual(self.gen.getrandbits(100), bits100) # Little trick to make "tuple(x % (2**32) for x in internalstate)" # raise ValueError. I cannot think of a simple way to achieve this, so diff --git a/Misc/ACKS b/Misc/ACKS index b8bac248d98867..d03fe472c0076c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1109,6 +1109,7 @@ Milan Oberkirch Pascal Oberndoerfer Jeffrey Ollie Adam Olsen +Bryan Olson Grant Olson Koray Oner Piet van Oostrum diff --git a/Misc/NEWS b/Misc/NEWS index bdaf7aaf70a0bf..600da2d507cf63 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-29960: Preserve generator state when _random.Random.setstate() + raises an exception. Patch by Bryan Olson. + - bpo-30414: multiprocessing.Queue._feed background running thread do not break from main loop on exception. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index e5dd2c96166384..d006aebf927c38 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -348,6 +348,7 @@ random_setstate(RandomObject *self, PyObject *state) int i; unsigned long element; long index; + uint32_t new_state[N]; if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, @@ -364,7 +365,7 @@ random_setstate(RandomObject *self, PyObject *state) element = PyLong_AsUnsignedLong(PyTuple_GET_ITEM(state, i)); if (element == (unsigned long)-1 && PyErr_Occurred()) return NULL; - self->state[i] = (uint32_t)element; + new_state[i] = (uint32_t)element; } index = PyLong_AsLong(PyTuple_GET_ITEM(state, i)); @@ -375,6 +376,8 @@ random_setstate(RandomObject *self, PyObject *state) return NULL; } self->index = (int)index; + for (i = 0; i < N; i++) + self->state[i] = new_state[i]; Py_INCREF(Py_None); return Py_None; From 2ee91c8ad825079cdfc0d98f3f439f3b1c8ca471 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 27 May 2017 07:58:25 -0700 Subject: [PATCH 0339/1345] [3.6] bpo-30470: Deprecate invalid ctypes call protection on Windows. (GH-1810) (GH-1833) Calling Ctypes functions is deprecated in 3.6.2 and will be removed in 3.7. (cherry picked from commit f931fd1c2ad969db72460d3ab41e3d1a4a62c371) --- Doc/library/ctypes.rst | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 47bf89f59a5696..cdcbefa4e8084a 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -161,22 +161,14 @@ as the NULL pointer):: 0x1d000000 >>> -:mod:`ctypes` tries to protect you from calling functions with the wrong number -of arguments or the wrong calling convention. Unfortunately this only works on -Windows. It does this by examining the stack after the function returns, so -although an error is raised the function *has* been called:: +.. note:: - >>> windll.kernel32.GetModuleHandleA() # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> windll.kernel32.GetModuleHandleA(0, 0) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> + :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if + it detects that an invalid number of arguments were passed. This behavior + should not be relied upon. It is deprecated in 3.6.2, and will be removed + in 3.7. -The same exception is raised when you call an ``stdcall`` function with the +:exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS From d52f47a8f7794ee09151973a00d29c8612672e7a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 29 May 2017 15:19:31 -0700 Subject: [PATCH 0340/1345] [3.6] bpo-30361: Use better example for mixed-type operands (GH-1701) (#1856) (cherry picked from commit e405d4b8dfb8b497e1c3d1f0f8e28030040c165e) --- Doc/tutorial/introduction.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 52120a0b16e75f..8956aa5a261350 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -100,10 +100,8 @@ give you an error:: There is full support for floating point; operators with mixed type operands convert the integer operand to floating point:: - >>> 3 * 3.75 / 1.5 - 7.5 - >>> 7.0 / 2 - 3.5 + >>> 4 * 3.75 - 1 + 14.0 In interactive mode, the last printed expression is assigned to the variable ``_``. This means that when you are using Python as a desk calculator, it is From 03c7600982027f4fbe5770c3caea59e08716374f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 30 May 2017 09:29:06 -0700 Subject: [PATCH 0341/1345] Improve grammar in The Import System documentation (GH-1862) (GH-1863) Replace `an ModuleNotFoundError` with `a ModuleNotFoundError`. Replace `an path` with `a path`. (cherry picked from commit 9c972b578002cc3bbbf2ebf2856c69de892bc9e2) --- Doc/reference/import.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 9475b8d55c2dc3..d504f37387422e 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -36,7 +36,7 @@ implement import semantics. When a module is first imported, Python searches for the module and if found, it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, an :exc:`ModuleNotFoundError` is raised. Python implements various +cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various strategies to search for the named module when the import machinery is invoked. These strategies can be modified and extended by using various hooks described in the sections below. @@ -167,7 +167,7 @@ arguments to the :keyword:`import` statement, or from the parameters to the This name will be used in various phases of the import search, and it may be the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, an :exc:`ModuleNotFoundError` is raised. +If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. The module cache @@ -185,7 +185,7 @@ object. During import, the module name is looked up in :data:`sys.modules` and if present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then an +process completes. However, if the value is ``None``, then a :exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will continue searching for the module. @@ -194,7 +194,7 @@ associated module (as other modules may hold references to it), but it will invalidate the cache entry for the named module, causing Python to search anew for the named module upon its next import. The key can also be assigned to ``None``, forcing the next import -of the module to result in an :exc:`ModuleNotFoundError`. +of the module to result in a :exc:`ModuleNotFoundError`. Beware though, as if you keep a reference to the module object, invalidate its cache entry in :data:`sys.modules`, and then re-import the @@ -298,7 +298,7 @@ The second argument is the path entries to use for the module search. For top-level modules, the second argument is ``None``, but for submodules or subpackages, the second argument is the value of the parent package's ``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, an :exc:`ModuleNotFoundError` is raised. The third argument +be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument is an existing module object that will be the target of loading later. The import system passes in a target module only during reload. @@ -616,7 +616,7 @@ the module. module.__path__ --------------- -By definition, if a module has an ``__path__`` attribute, it is a package, +By definition, if a module has a ``__path__`` attribute, it is a package, regardless of its value. A package's ``__path__`` attribute is used during imports of its subpackages. From 00b381b9a7b6b15350d1dcf7c688bf808cbf0ebb Mon Sep 17 00:00:00 2001 From: csabella Date: Tue, 30 May 2017 16:27:39 -0400 Subject: [PATCH 0342/1345] bpo-17188: DOC: Document 'from None' in raise statement (GH-1671) (GH-1867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original patch by Dennis Mårtensson. (cherry picked from commit 763557eac06ba60d7c5133e4f80df8870d8f917e) --- Doc/reference/simple_stmts.rst | 22 ++++++++++++++++++++-- Misc/ACKS | 1 + 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index e152b16ee32d4f..4355f185aae8c8 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -587,7 +587,7 @@ printed:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: @@ -606,7 +606,7 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: @@ -614,9 +614,27 @@ attached as the new exception's :attr:`__context__` attribute:: File "", line 4, in RuntimeError: Something bad happened +Exception chaining can be explicitly suppressed by specifying :const:`None` in +the ``from`` clause:: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + Additional information on exceptions can be found in section :ref:`exceptions`, and information about handling exceptions is in section :ref:`try`. +.. versionchanged:: 3.3 + :const:`None` is now permitted as ``Y`` in ``raise X from Y`` + +.. versionadded:: 3.3 + The ``__suppress_context__`` attribute to suppress automatic display of the + exception context .. _break: diff --git a/Misc/ACKS b/Misc/ACKS index d03fe472c0076c..7fc5fecc8a4605 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -963,6 +963,7 @@ David Marek Doug Marien Sven Marnach Alex Martelli +Dennis Mårtensson Anthony Martin Owen Martin Sidney San Martín From ad13d54646d189cea8e6f7a9bc4a87e20df33ac4 Mon Sep 17 00:00:00 2001 From: csabella Date: Tue, 30 May 2017 16:48:22 -0400 Subject: [PATCH 0343/1345] bpo-30354: Update data model documentation for super() (GH-1561) (GH-1868) The data model section of the language reference was written well before the zero-argument form of super() was added. To avoid giving the impression that they're doing something unusual, this updates the description of `__new__` and `__init__` to use the zero-argument form. Patch by Cheryl Sabella. (cherry picked from commit 12b1c180986fc744331b8f30d3d2f49a0fdb43dd) --- Doc/reference/datamodel.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 35925a065fd495..0e0a9e24f0d18d 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1119,9 +1119,9 @@ Basic customization (usually an instance of *cls*). Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super(currentclass, - cls).__new__(cls[, ...])`` with appropriate arguments and then modifying the - newly-created instance as necessary before returning it. + superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` + with appropriate arguments and then modifying the newly-created instance + as necessary before returning it. If :meth:`__new__` returns an instance of *cls*, then the new instance's :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where @@ -1145,7 +1145,7 @@ Basic customization class constructor expression. If a base class has an :meth:`__init__` method, the derived class's :meth:`__init__` method, if any, must explicitly call it to ensure proper initialization of the base class part of the - instance; for example: ``BaseClass.__init__(self, [args...])``. + instance; for example: ``super().__init__([args...])``. Because :meth:`__new__` and :meth:`__init__` work together in constructing objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), @@ -1578,8 +1578,8 @@ Class Binding ``A.__dict__['x'].__get__(None, A)``. Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, - obj).m()`` searches ``obj.__class__.__mro__`` for the base class ``A`` + If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` + searches ``obj.__class__.__mro__`` for the base class ``A`` immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. From 89e0a189f8d6054e3e2a7345ac779c1052dc2a49 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 30 May 2017 15:29:24 -0700 Subject: [PATCH 0344/1345] [3.6] Build docs in parallel (GH-1842) (GH-1872) (cherry picked from commit eba68e2c42e149acecb15bbeb692786e2540157d) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe90bad626ce83..cf524e5549289b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ matrix: # (Updating the version is fine as long as no warnings are raised by doing so.) - python -m pip install sphinx~=1.6.1 script: - - make check suspicious html SPHINXOPTS="-q -W" + - make check suspicious html SPHINXOPTS="-q -W -j4" - os: linux language: c compiler: gcc From 60b257b01ca497d7a556e3f526704aa1b923d1ac Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 30 May 2017 15:37:43 -0700 Subject: [PATCH 0345/1345] bpo-17188: add missing periods at the end of sentences (GH-1875) (GH-1876) (cherry picked from commit 9efad1e5aef32a6d8ca288793e8ee6df8782f615) --- Doc/reference/simple_stmts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 4355f185aae8c8..e1cec8794cc0e8 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -630,11 +630,11 @@ Additional information on exceptions can be found in section :ref:`exceptions`, and information about handling exceptions is in section :ref:`try`. .. versionchanged:: 3.3 - :const:`None` is now permitted as ``Y`` in ``raise X from Y`` + :const:`None` is now permitted as ``Y`` in ``raise X from Y``. .. versionadded:: 3.3 The ``__suppress_context__`` attribute to suppress automatic display of the - exception context + exception context. .. _break: From e49c5e92778cbd8f5b72da185eacd50b3a9896f7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 30 May 2017 22:08:04 -0700 Subject: [PATCH 0346/1345] [3.6] Fix a trivial typo in global section (GH-1497) (GH-1880) (cherry picked from commit f34c6850203a2406c4950af7a9c8a134145df4ea) --- Doc/reference/simple_stmts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index e1cec8794cc0e8..8786d73f68a369 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -940,7 +940,7 @@ annotation. builtin: eval builtin: compile -**Programmer's note:** the :keyword:`global` is a directive to the parser. It +**Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`global` statement. In particular, a :keyword:`global` statement contained in a string or code object supplied to the built-in :func:`exec` function does not affect the code From 767b6d7d2da6823f71d10e6eeae24451db115c61 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 31 May 2017 15:52:39 -0700 Subject: [PATCH 0347/1345] [3.6] Fix Travis config to reinstate test build (GH-1879) (GH-1889) (cherry picked from commit a5aa72ac789052411797c6f079ffff0a9fdbf82c) --- .travis.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf524e5549289b..e4e7b3d9b0246a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,25 +11,19 @@ branches: - master - /^\d\.\d$/ -os: - - linux - # macOS builds are disabled as the machines are under-provisioned on Travis, - # adding up to an extra hour completing a full CI run. - -compiler: - - clang - # gcc also works, but to keep the # of concurrent builds down, we use one C - # compiler here and the other to run the coverage build. - -env: - - TESTING=cpython - matrix: fast_finish: true allow_failures: - env: - TESTING=coverage include: + - os: linux + language: c + compiler: clang + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. + env: + - TESTING=cpython - os: linux language: python python: 3.6 From 9abd0bf68f73b3beb56954e22740f0efcf04bc79 Mon Sep 17 00:00:00 2001 From: csabella Date: Wed, 31 May 2017 21:53:24 -0400 Subject: [PATCH 0348/1345] bpo-27618: Clarify that threading.Lock is a factory function (GH-1307) (GH-1894) (cherry picked from commit 56ddfd2eea4d98456a9a99bf2a718f21ee6b2be2) --- Doc/library/threading.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 2792dfdce04c6c..cda859fe4cbd3a 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -371,8 +371,9 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - .. versionchanged:: 3.3 - Changed from a factory function to a class. + Note that ``Lock`` is actually a factory function which returns an instance + of the most efficient version of the concrete Lock class that is supported + by the platform. .. method:: acquire(blocking=True, timeout=-1) From 9522159bc41f2be0f4aafd0fa182669876aca47f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 31 May 2017 19:49:01 -0700 Subject: [PATCH 0349/1345] bpo-22702: Clarify documentation of str.join & bytes.join (GH-156) (GH-1897) The "iterable iterable" phrasing created confusion between the term reference and the parameter name. This simplifies the phrasing to just use the parameter name without linking directly to the term definition. (cherry picked from commit 08e2f355d04d3cbea5751ce1275306ee3f569b32) --- Doc/library/stdtypes.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index af0a7b7fe939b2..4272b96fecfa21 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1719,10 +1719,10 @@ expression support in the :mod:`re` module). .. method:: str.join(iterable) - Return a string which is the concatenation of the strings in the - :term:`iterable` *iterable*. A :exc:`TypeError` will be raised if there are - any non-string values in *iterable*, including :class:`bytes` objects. The - separator between elements is the string providing this method. + Return a string which is the concatenation of the strings in *iterable*. + A :exc:`TypeError` will be raised if there are any non-string values in + *iterable*, including :class:`bytes` objects. The separator between + elements is the string providing this method. .. method:: str.ljust(width[, fillchar]) @@ -2539,11 +2539,11 @@ arbitrary binary data. bytearray.join(iterable) Return a bytes or bytearray object which is the concatenation of the - binary data sequences in the :term:`iterable` *iterable*. A - :exc:`TypeError` will be raised if there are any values in *iterable* - that are not :term:`bytes-like objects `, including - :class:`str` objects. The separator between elements is the contents - of the bytes or bytearray object providing this method. + binary data sequences in *iterable*. A :exc:`TypeError` will be raised + if there are any values in *iterable* that are not :term:`bytes-like + objects `, including :class:`str` objects. The + separator between elements is the contents of the bytes or + bytearray object providing this method. .. staticmethod:: bytes.maketrans(from, to) From 9d752aa5e695d682f3cfc40655580900b132e52a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 1 Jun 2017 07:12:01 -0700 Subject: [PATCH 0350/1345] bpo-30499: Remove a deprecated note about sets. (GH-1848) (GH-1905) (cherry picked from commit 0737ee20671d60802a7ef40f653fe449f1921de0) --- Doc/library/stdtypes.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4272b96fecfa21..81146f41784976 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3982,9 +3982,7 @@ The constructors for both classes work the same: Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, the *elem* set is temporarily mutated during the search and then - restored. During the search, the *elem* set should not be read or mutated - since it does not have a meaningful value. + frozenset, a temporary one is created from *elem*. .. _typesmapping: From 95b4da2be4aeb74ee46ddece71f2075b32915c60 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Thu, 1 Jun 2017 22:20:27 +0800 Subject: [PATCH 0351/1345] bpo-30378: Fix the problem that SysLogHandler can't handle IPv6 addresses (#1676) (#1903) --- Lib/logging/handlers.py | 25 ++++++++++++++++++++----- Lib/test/test_logging.py | 22 ++++++++++++++++++++-- Misc/NEWS | 3 +++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 2356f8d3d2d8ce..2f934b33071563 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -827,11 +827,26 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self.unixsocket = False if socktype is None: socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_INET, socktype) - if socktype == socket.SOCK_STREAM: - self.socket.connect(address) + host, port = address + ress = socket.getaddrinfo(host, port, 0, socktype) + if not ress: + raise OSError("getaddrinfo returns an empty list") + for res in ress: + af, socktype, proto, _, sa = res + err = sock = None + try: + sock = socket.socket(af, socktype, proto) + if socktype == socket.SOCK_STREAM: + sock.connect(sa) + break + except OSError as exc: + err = exc + if sock is not None: + sock.close() + if err is not None: + raise err + self.socket = sock self.socktype = socktype - self.formatter = None def _connect_unixsocket(self, address): use_socktype = self.socktype @@ -870,7 +885,7 @@ def encodePriority(self, facility, priority): priority = self.priority_names[priority] return (facility << 3) | priority - def close (self): + def close(self): """ Closes the socket. """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 0a2584c6103c32..474affdaebcd35 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1663,7 +1663,7 @@ def setUp(self): server.ready.wait() hcls = logging.handlers.SysLogHandler if isinstance(server.server_address, tuple): - self.sl_hdlr = hcls(('localhost', server.port)) + self.sl_hdlr = hcls((server.server_address[0], server.port)) else: self.sl_hdlr = hcls(server.server_address) self.log_output = '' @@ -1723,6 +1723,24 @@ def tearDown(self): SysLogHandlerTest.tearDown(self) support.unlink(self.address) +@unittest.skipUnless(support.IPV6_ENABLED, + 'IPv6 support required for this test.') +@unittest.skipUnless(threading, 'Threading required for this test.') +class IPv6SysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with IPv6 host.""" + + server_class = TestUDPServer + address = ('::1', 0) + + def setUp(self): + self.server_class.address_family = socket.AF_INET6 + super(IPv6SysLogHandlerTest, self).setUp() + + def tearDown(self): + self.server_class.address_family = socket.AF_INET + super(IPv6SysLogHandlerTest, self).tearDown() + @unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -4378,7 +4396,7 @@ def test_main(): QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, BasicConfigTest, LoggerAdapterTest, LoggerTest, SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, LastResortTest, LogRecordTest, - ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, + ExceptionTest, SysLogHandlerTest, IPv6SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, MiscTestCase diff --git a/Misc/NEWS b/Misc/NEWS index 600da2d507cf63..86dd1b4543819c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-30378: Fix the problem that logging.handlers.SysLogHandler cannot + handle IPv6 addresses. + - bpo-29960: Preserve generator state when _random.Random.setstate() raises an exception. Patch by Bryan Olson. From e417d12d728103445b8516a35f643536e401f120 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 1 Jun 2017 20:26:15 -0700 Subject: [PATCH 0352/1345] bpo-29660: traceback: Document that etype is ignored in some places. (GH-344) (GH-1913) (cherry picked from commit cdb89cd) --- Doc/library/traceback.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 066ee96fc004ab..55d331c996a187 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -45,9 +45,9 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` * it prints the exception *etype* and *value* after the stack trace - * if *etype* is :exc:`SyntaxError` and *value* has the appropriate format, it - prints the line where the syntax error occurred with a caret indicating the - approximate position of the error. + * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate + format, it prints the line where the syntax error occurred with a caret + indicating the approximate position of the error. The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the @@ -55,6 +55,9 @@ The module defines the following functions: printed as well, like the interpreter itself does when printing an unhandled exception. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -131,6 +134,9 @@ The module defines the following functions: containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does :func:`print_exception`. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: format_exc(limit=None, chain=True) @@ -372,6 +378,7 @@ exception and traceback: print("*** print_tb:") traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) print("*** print_exception:") + # exc_type below is ignored on 3.5 and later traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout) print("*** print_exc:") @@ -381,6 +388,7 @@ exception and traceback: print(formatted_lines[0]) print(formatted_lines[-1]) print("*** format_exception:") + # exc_type below is ignored on 3.5 and later print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) print("*** extract_tb:") From 1c92c0edca413c6f8ce8ec326a67451ae751ead9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 1 Jun 2017 21:56:24 -0700 Subject: [PATCH 0353/1345] bpo-30052: Link `bytes` & `bytearray` to stdtypes not functions (GH-1271) (GH-1915) Builtin container types have two potential link targets in the docs: - their entry in the list of builtin callables - their type documentation This change brings `bytes` and `bytearray` into line with other container types by having cross-references default to linking to their type documentation, rather than their builtin callable entry.. (cherry picked from commit c6db4811f9ea3aeff0e1fafe1c60a22835ef359e) --- Doc/library/functions.rst | 9 ++- Doc/library/stdtypes.rst | 150 +++++++++++++++++++----------------- Doc/reference/datamodel.rst | 16 ++-- 3 files changed, 92 insertions(+), 83 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 138f7d027cb86f..6621f4aabf7058 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -16,8 +16,8 @@ are always available. They are listed here in alphabetical order. :func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` :func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ :func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -:func:`bytearray` :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -:func:`bytes` :func:`float` :func:`iter` :func:`print` |func-tuple|_ +|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` +|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ :func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` :func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` :func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` @@ -37,7 +37,8 @@ are always available. They are listed here in alphabetical order. .. |func-str| replace:: ``str()`` .. |func-tuple| replace:: ``tuple()`` .. |func-range| replace:: ``range()`` - +.. |func-bytearray| replace:: ``bytearray()`` +.. |func-bytes| replace:: ``bytes()`` .. function:: abs(x) @@ -99,6 +100,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray([source[, encoding[, errors]]]) + :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable sequence of integers in the range 0 <= x < 256. It has most of the usual @@ -128,6 +130,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes([source[, encoding[, errors]]]) + :noindex: Return a new "bytes" object, which is an immutable sequence of integers in the range ``0 <= x < 256``. :class:`bytes` is an immutable version of diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 81146f41784976..ca3404aa21e555 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2264,8 +2264,8 @@ The :mod:`array` module supports efficient storage of basic data types like .. _typebytes: -Bytes ------ +Bytes Objects +------------- .. index:: object: bytes @@ -2274,65 +2274,67 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -Firstly, the syntax for bytes literals is largely the same as that for string -literals, except that a ``b`` prefix is added: +.. class:: bytes([source[, encoding[, errors]]]) -* Single quotes: ``b'still allows embedded "double" quotes'`` -* Double quotes: ``b"still allows embedded 'single' quotes"``. -* Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` + Firstly, the syntax for bytes literals is largely the same as that for string + literals, except that a ``b`` prefix is added: -Only ASCII characters are permitted in bytes literals (regardless of the -declared source code encoding). Any binary values over 127 must be entered -into bytes literals using the appropriate escape sequence. + * Single quotes: ``b'still allows embedded "double" quotes'`` + * Double quotes: ``b"still allows embedded 'single' quotes"``. + * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` -As with string literals, bytes literals may also use a ``r`` prefix to disable -processing of escape sequences. See :ref:`strings` for more about the various -forms of bytes literal, including supported escape sequences. + Only ASCII characters are permitted in bytes literals (regardless of the + declared source code encoding). Any binary values over 127 must be entered + into bytes literals using the appropriate escape sequence. -While bytes literals and representations are based on ASCII text, bytes -objects actually behave like immutable sequences of integers, with each -value in the sequence restricted such that ``0 <= x < 256`` (attempts to -violate this restriction will trigger :exc:`ValueError`. This is done -deliberately to emphasise that while many binary formats include ASCII based -elements and can be usefully manipulated with some text-oriented algorithms, -this is not generally the case for arbitrary binary data (blindly applying -text processing algorithms to binary data formats that are not ASCII -compatible will usually lead to data corruption). + As with string literals, bytes literals may also use a ``r`` prefix to disable + processing of escape sequences. See :ref:`strings` for more about the various + forms of bytes literal, including supported escape sequences. -In addition to the literal forms, bytes objects can be created in a number of -other ways: + While bytes literals and representations are based on ASCII text, bytes + objects actually behave like immutable sequences of integers, with each + value in the sequence restricted such that ``0 <= x < 256`` (attempts to + violate this restriction will trigger :exc:`ValueError`. This is done + deliberately to emphasise that while many binary formats include ASCII based + elements and can be usefully manipulated with some text-oriented algorithms, + this is not generally the case for arbitrary binary data (blindly applying + text processing algorithms to binary data formats that are not ASCII + compatible will usually lead to data corruption). -* A zero-filled bytes object of a specified length: ``bytes(10)`` -* From an iterable of integers: ``bytes(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytes(obj)`` + In addition to the literal forms, bytes objects can be created in a number of + other ways: -Also see the :ref:`bytes ` built-in. + * A zero-filled bytes object of a specified length: ``bytes(10)`` + * From an iterable of integers: ``bytes(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytes(obj)`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytes type has an additional class method to read data in that format: + Also see the :ref:`bytes ` built-in. -.. classmethod:: bytes.fromhex(string) + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytes type has an additional class method to read data in that format: - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII spaces being ignored. + .. classmethod:: fromhex(string) - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' + This :class:`bytes` class method returns a bytes object, decoding the + given string object. The string must contain two hexadecimal digits per + byte, with ASCII whitespace being ignored. -A reverse conversion function exists to transform a bytes object into its -hexadecimal representation. + >>> bytes.fromhex('2Ef0 F1f2 ') + b'.\xf0\xf1\xf2' -.. method:: bytes.hex() + A reverse conversion function exists to transform a bytes object into its + hexadecimal representation. - Return a string object containing two hexadecimal digits for each - byte in the instance. + .. method:: hex() - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> b'\xf0\xf1\xf2'.hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes @@ -2362,45 +2364,49 @@ Bytearray Objects .. index:: object: bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. There is no dedicated literal syntax for bytearray objects, instead -they are always created by calling the constructor: +objects. -* Creating an empty instance: ``bytearray()`` -* Creating a zero-filled instance with a given length: ``bytearray(10)`` -* From an iterable of integers: ``bytearray(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` +.. class:: bytearray([source[, encoding[, errors]]]) -As bytearray objects are mutable, they support the -:ref:`mutable ` sequence operations in addition to the -common bytes and bytearray operations described in :ref:`bytes-methods`. + There is no dedicated literal syntax for bytearray objects, instead + they are always created by calling the constructor: -Also see the :ref:`bytearray ` built-in. + * Creating an empty instance: ``bytearray()`` + * Creating a zero-filled instance with a given length: ``bytearray(10)`` + * From an iterable of integers: ``bytearray(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytearray type has an additional class method to read data in that format: + As bytearray objects are mutable, they support the + :ref:`mutable ` sequence operations in addition to the + common bytes and bytearray operations described in :ref:`bytes-methods`. -.. classmethod:: bytearray.fromhex(string) + Also see the :ref:`bytearray ` built-in. - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII spaces being ignored. + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytearray type has an additional class method to read data in that format: - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') + .. classmethod:: fromhex(string) -A reverse conversion function exists to transform a bytearray object into its -hexadecimal representation. + This :class:`bytearray` class method returns bytearray object, decoding + the given string object. The string must contain two hexadecimal digits + per byte, with ASCII whitespace being ignored. -.. method:: bytearray.hex() + >>> bytearray.fromhex('2Ef0 F1f2 ') + bytearray(b'.\xf0\xf1\xf2') - Return a string object containing two hexadecimal digits for each - byte in the instance. + A reverse conversion function exists to transform a bytearray object into its + hexadecimal representation. - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' + .. method:: hex() + + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> bytearray(b'\xf0\xf1\xf2').hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0e0a9e24f0d18d..7c140a3bc86cb8 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -320,9 +320,9 @@ Sequences A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in function :func:`bytes` can be used to - construct bytes objects. Also, bytes objects can be decoded to strings - via the :meth:`~bytes.decode` method. + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. Mutable sequences .. index:: @@ -349,9 +349,9 @@ Sequences .. index:: bytearray A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable (and hence - unhashable), byte arrays otherwise provide the same interface and - functionality as immutable bytes objects. + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. .. index:: module: array @@ -1253,8 +1253,8 @@ Basic customization .. index:: builtin: bytes - Called by :func:`bytes` to compute a byte-string representation of an - object. This should return a ``bytes`` object. + Called by :ref:`bytes ` to compute a byte-string representation + of an object. This should return a :class:`bytes` object. .. index:: single: string; __format__() (object method) From c63ae1122f84d4188ffadfd1454902093eb10be1 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 2 Jun 2017 14:39:05 -0700 Subject: [PATCH 0354/1345] bpo-30544: _io._WindowsConsoleIO.write raises the wrong error when WriteConsoleW fails (#1912) (#1925) * bpo-30544: _io._WindowsConsoleIO.write raises the wrong error when WriteConsoleW fails * bpo-30544: _io._WindowsConsoleIO.write raises the wrong error when WriteConsoleW fails --- Modules/_io/winconsoleio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 1d169e2764ed27..cbe21422bb4653 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1017,7 +1017,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); - if (n < wlen) { + if (res && n < wlen) { /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the * characters that were written. As this could potentially From ba6bfcdfb43f06f025613348bb6c71ff9b597b35 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 3 Jun 2017 10:33:53 -0700 Subject: [PATCH 0355/1345] [3.6] Turn on macOS builds for Travis (GH-1846) (#1929) Initially the macOS builds are allowed to fail until such time that they can be determined to be stable and not add an unacceptable amount of time to the overall Travis-passing process. (cherry picked from commit 21c2dd7cf8414c903f0e83cf1d6b7f02f645f422) --- .travis.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4e7b3d9b0246a..4dfefb72700dbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,21 +14,24 @@ branches: matrix: fast_finish: true allow_failures: - - env: - - TESTING=coverage + - env: OPTIONAL=true include: - os: linux language: c compiler: clang # gcc also works, but to keep the # of concurrent builds down, we use one C - # compiler here and the other to run the coverage build. - env: - - TESTING=cpython + # compiler here and the other to run the coverage build. Clang is preferred + # in this instance for its better error messages. + env: TESTING=cpython + - os: osx + language: c + compiler: clang + # Testing under macOS is optional until testing stability has been demonstrated. + env: OPTIONAL=true - os: linux language: python python: 3.6 - env: - - TESTING=docs + env: TESTING=docs before_script: - cd Doc # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. @@ -39,8 +42,7 @@ matrix: - os: linux language: c compiler: gcc - env: - - TESTING=coverage + env: OPTIONAL=true before_script: - | if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' From 7e6d999b6c22de2594df8bca56aedf60ba4714b0 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sun, 4 Jun 2017 17:14:03 -0500 Subject: [PATCH 0356/1345] [3.6] Fix skipping test_UNC_path on AppVeyor due to a different error being raised (GH-1920) We get `ERROR_BAD_NETPATH` (53) on AppVeyor which is translated to ENOENT (2). (cherry picked from commit 7a99625e0d95cd88ed8842d8677b5beea1fde5ae) --- Lib/test/test_import/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 760908efe671fe..4aca9e597cf9f9 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -567,7 +567,7 @@ def test_UNC_path(self): try: os.listdir(unc) except OSError as e: - if e.errno in (errno.EPERM, errno.EACCES): + if e.errno in (errno.EPERM, errno.EACCES, errno.ENOENT): # See issue #15338 self.skipTest("cannot access administrative share %r" % (unc,)) raise From 47779d720a0fa0eec818065f41b17d0f2d77d3ff Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sun, 4 Jun 2017 19:50:37 -0500 Subject: [PATCH 0357/1345] [3.6] Only run AppVeyor on long-lived branches (GH-1941) Also on the short-lived `buildbot-custom` branch. (cherry picked from commit d3bedf356aca04ed9135a84b08) --- .github/appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/appveyor.yml b/.github/appveyor.yml index a870e5b57d9250..d8bfb9adf9378d 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -1,5 +1,10 @@ version: 3.6.1+.{build} clone_depth: 5 +branches: + only: + - master + - /\d\.\d/ + - buildbot-custom build_script: - cmd: PCbuild\build.bat -e test_script: From 86eb93fae6fc5bf121de815b82697f2bc5bc126c Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 4 Jun 2017 20:06:48 -0700 Subject: [PATCH 0358/1345] bpo-30530: Update Descriptor How To Documentation (GH-1845) (GH-1953) Update the code example in Functions and Methods section Remove objtype argument in MethodType (cherry picked from commit 1bced56567335745f91676192fc39c06aab30da9) --- Doc/howto/descriptor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index c2bf473e1ff9ea..2dd6c34e2ced02 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -282,7 +282,7 @@ this:: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" - return types.MethodType(self, obj, objtype) + return types.MethodType(self, obj) Running the interpreter shows how the function descriptor works in practice:: From ce40550acd3a8928bb6fef4d4e73642a8a69745d Mon Sep 17 00:00:00 2001 From: csabella Date: Sun, 4 Jun 2017 23:12:23 -0400 Subject: [PATCH 0359/1345] bpo-30538: Update count() in Functional Programming HOWTO (GH-1919) (GH-1943) * bpo-30538: Update count() in Functional HOWTO * bpo-30538: Update enumerate() arguments in Functional HOWTO (cherry picked from commit 9be4ff359daa67cde6246494f643ed7cd2825d46) --- Doc/howto/functional.rst | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index a82dca7077e905..40601812a77cb5 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -653,8 +653,9 @@ This can also be written as a list comprehension: [0, 2, 4, 6, 8] -:func:`enumerate(iter) ` counts off the elements in the iterable, -returning 2-tuples containing the count and each element. :: +:func:`enumerate(iter, start=0) ` counts off the elements in the +iterable returning 2-tuples containing the count (from *start*) and +each element. :: >>> for item in enumerate(['subject', 'verb', 'object']): ... print(item) @@ -747,14 +748,16 @@ The module's functions fall into a few broad classes: Creating new iterators ---------------------- -:func:`itertools.count(n) ` returns an infinite stream of -integers, increasing by 1 each time. You can optionally supply the starting -number, which defaults to 0:: +:func:`itertools.count(start, step) ` returns an infinite +stream of evenly spaced values. You can optionally supply the starting number, +which defaults to 0, and the interval between numbers, which defaults to 1:: itertools.count() => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... itertools.count(10) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... + itertools.count(10, 5) => + 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... :func:`itertools.cycle(iter) ` saves a copy of the contents of a provided iterable and returns a new iterator that returns its elements from @@ -1060,10 +1063,10 @@ write the obvious :keyword:`for` loop:: for i in [1,2,3]: product *= i -A related function is `itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of +returning only the final result, :func:`accumulate` returns an iterator that +also yields each partial result:: itertools.accumulate([1,2,3,4,5]) => 1, 3, 6, 10, 15 @@ -1235,6 +1238,8 @@ Python documentation Documentation for the :mod:`itertools` module. +Documentation for the :mod:`functools` module. + Documentation for the :mod:`operator` module. :pep:`289`: "Generator Expressions" From d125738e93a35125d0edbe8070dd47160bcda829 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 5 Jun 2017 19:10:21 -0500 Subject: [PATCH 0360/1345] [3.6] bpo-30417: Disable 'cpu' and 'tzdata' resources on Travis (GH-1928) Also weakens the 'should this be run?' regex to allow all builds when .travis.yml changes. (cherry picked from commit c53b13b270767948fddb58b287149c499f9a03c4) --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dfefb72700dbd..cb33e9b61e8631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ matrix: env: OPTIONAL=true before_script: - | - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' then echo "Only docs were updated, stopping build process." exit @@ -57,16 +57,16 @@ matrix: ./venv/bin/python -m pip install -U coverage script: # Skip tests that re-run the entire test suite. - - ./venv/bin/python -m coverage run --pylib -m test -uall -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu,-tzdata -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn after_script: # Probably should be after_success once test suite updated to run under coverage.py. # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate - bash <(curl -s https://codecov.io/bash) -# Travis provides only 2 cores, so don't overdue the parallelism and waste memory. +# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - | - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' then echo "Only docs were updated, stopping build process." exit @@ -76,7 +76,7 @@ before_script: script: # `-r -w` implicitly provided through `make buildbottest`. - - make buildbottest TESTOPTS="-j4" + - make buildbottest TESTOPTS="-j4 -uall,-cpu,-tzdata" notifications: email: false From 798cfb212383a6f5a31b610825d1af1d376804b2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 6 Jun 2017 12:19:37 +1000 Subject: [PATCH 0361/1345] bpo-30052: Always regenerate cross-references (GH-1339) (GH-1921) The patch for bpo-30052 changed the preferred link target for :func:`bytes` and :func`bytearray` references to be the respective type definitions rather than the corresponding builtin function entries. This patch changes the daily documentation builds to disable the output caching in Sphinx, in order to ensure that cross-reference changes like this one are reliably picked up and applied automatically after merging. (cherry picked from commit 7a82f9c2b94d31c8f4cc8bb8e3151765d8b148d7) --- Doc/Makefile | 9 +++++++-- Misc/NEWS | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 94697f95671894..ae59f3294f1c06 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -159,15 +159,20 @@ serve: ../Tools/scripts/serve.py build/html # Targets for daily automated doc build +# By default, Sphinx only rebuilds pages where the page content has changed. +# This means it doesn't always pick up changes to preferred link targets, etc +# To ensure such changes are picked up, we build the published docs with +# `-E` (to ignore the cached environment) and `-a` (to ignore already existing +# output files) # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Misc/NEWS b/Misc/NEWS index 86dd1b4543819c..1656e07f6eb1df 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -192,6 +192,15 @@ Build Documentation ------------- +- Issue #30052: the link targets for :func:`bytes` and + :func:`bytearray` are now their respective type definitions, rather + than the corresponding builtin function entries. Use :ref:`bytes ` + and :ref:`bytearray ` to reference the latter. + + In order to ensure this and future cross-reference updates are applied + automatically, the daily documentation builds now disable the default + output caching features in Sphinx. + - bpo-26985: Add missing info of code object in inspect documentation. Tools/Demos From 01ea561f29aec4fa82cabc3896409f918d6ef470 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 5 Jun 2017 22:41:17 -0500 Subject: [PATCH 0362/1345] [3.6] Install dependencies in Travis OSX build (GH-1952) (cherry picked from commit 167e0fc211c06df) --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index cb33e9b61e8631..2b4556fc6fc462 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,10 @@ matrix: compiler: clang # Testing under macOS is optional until testing stability has been demonstrated. env: OPTIONAL=true + before_install: + - brew install openssl xz + - export CPPFLAGS="-I$(brew --prefix openssl)/include" + - export LDFLAGS="-L$(brew --prefix openssl)/lib" - os: linux language: python python: 3.6 From e1a60d903276da10d1ac098e59103f4a72bef08e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 6 Jun 2017 08:56:59 -0700 Subject: [PATCH 0363/1345] [3.6] bpo-30303: IDLE: Add _utest argument to textview (GH-1499) (#1916) (cherry picked from commit ba365da9cee8901fad08ab0b61c7489c110e5c97) --- Lib/idlelib/idle_test/test_textview.py | 42 +++++++++++++++++++++++++- Lib/idlelib/textview.py | 26 ++++++++++------ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index f018f5ef1975ba..67fc0b7c4f1c6d 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -13,7 +13,7 @@ import unittest import os -from tkinter import Tk +from tkinter import Tk, Button from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func @@ -96,5 +96,45 @@ def test_view_file(self): self.assertIsNone(view) +class ButtonClickTextViewTest(unittest.TestCase): + + def setUp(self): + self.view = None + self.called = False + + def tearDown(self): + if self.view: + self.view.destroy() + + def test_view_text_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_text(root, 'TITLE_TEXT', 'COMMAND', _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_TEXT') + self.assertEqual(self.view.textView.get('1.0', '1.end'), 'COMMAND') + + def test_view_file_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_file(root, 'TITLE_FILE', __file__, _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_FILE') + with open(__file__) as f: + self.assertEqual(self.view.textView.get('1.0', '1.end'), + f.readline().strip()) + f.readline() + self.assertEqual(self.view.textView.get('3.0', '3.end'), + f.readline().strip()) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index adee326e1deb91..f33ac32a33a3bb 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -9,14 +9,15 @@ class TextViewer(Toplevel): "A simple text viewer dialog for IDLE." - def __init__(self, parent, title, text, modal=True, _htest=False): - """Show the given text in a scrollable window with a 'close' button + def __init__(self, parent, title, text, modal=True, + _htest=False, _utest=False): + """Show the given text in a scrollable window with a 'close' button. - If modal option set to False, user can interact with other windows, - otherwise they will be unable to interact with other windows until - the textview window is closed. + If modal is left True, users cannot interact with other windows + until the textview window is closed. _htest - bool; change box location when running htest. + _utest - bool; don't wait_window when running unittest. """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -42,9 +43,11 @@ def __init__(self, parent, title, text, modal=True, _htest=False): if modal: self.transient(parent) self.grab_set() - self.wait_window() + if not _utest: + self.wait_window() def CreateWidgets(self): + "Create Frame with Text (with vertical Scrollbar) and Button." frameText = Frame(self, relief=SUNKEN, height=700) frameButtons = Frame(self) self.buttonOk = Button(frameButtons, text='Close', @@ -65,10 +68,12 @@ def Ok(self, event=None): self.destroy() -def view_text(parent, title, text, modal=True): - return TextViewer(parent, title, text, modal) +def view_text(parent, title, text, modal=True, _utest=False): + "Display text in a TextViewer." + return TextViewer(parent, title, text, modal, _utest=_utest) -def view_file(parent, title, filename, encoding=None, modal=True): +def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): + "Display file in a TextViever or show error message." try: with open(filename, 'r', encoding=encoding) as file: contents = file.read() @@ -81,7 +86,8 @@ def view_file(parent, title, filename, encoding=None, modal=True): message=str(err), parent=parent) else: - return view_text(parent, title, contents, modal) + return view_text(parent, title, contents, modal, _utest=_utest) + if __name__ == '__main__': import unittest From 68e5af89d46db6250a0fa7fe4249c66f866faff8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 6 Jun 2017 19:23:02 +0300 Subject: [PATCH 0364/1345] [3.6] bpo-30567: Fix refleak in sys.getwindowsversion (GH-1940) (#1970) (cherry picked from commit 48fb766) --- Python/sysmodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 99e6b5ea018ea1..b09268b0e53854 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -985,10 +985,10 @@ sys_getwindowsversion(PyObject *self) } PyMem_RawFree(verblock); } - PyStructSequence_SET_ITEM(version, pos++, PyTuple_Pack(3, - PyLong_FromLong(realMajor), - PyLong_FromLong(realMinor), - PyLong_FromLong(realBuild) + PyStructSequence_SET_ITEM(version, pos++, Py_BuildValue("(kkk)", + realMajor, + realMinor, + realBuild )); if (PyErr_Occurred()) { From b25b7254d909697bb7f3aa893b740cb650547cb6 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Tue, 6 Jun 2017 14:32:14 -0500 Subject: [PATCH 0365/1345] bpo-30583: Fix typo in datetime dateutil documentation (GH-1972) (GH-1974) Replace `datetuil` into `dateutil` (cherry picked from commit 53f2af16551eb3a080da313257603c31ef8b93b4) --- Doc/library/datetime.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c9318557f5c826..98fe86eba8ac77 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1849,11 +1849,11 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `datetuil.tz `_ + `dateutil.tz `_ The standard library has :class:`timezone` class for handling arbitrary fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. - *datetuil.tz* library brings the *IANA timezone database* (also known as the + *dateutil.tz* library brings the *IANA timezone database* (also known as the Olson database) to Python and its usage is recommended. `IANA timezone database `_ From 2bafc0dccac2390a52670ba289878318b6ea0293 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 6 Jun 2017 13:47:14 -0700 Subject: [PATCH 0366/1345] =?UTF-8?q?[3.6]=20bpo-30557:=20faulthandler=20n?= =?UTF-8?q?ow=20correctly=20filters=20and=20displays=20exception=20?= =?UTF-8?q?=E2=80=A6=20(#1960)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bpo-30557: faulthandler now correctly filters and displays exception codes on Windows (#1924) * bpo-30557: faulthandler now correctly filters and displays exception codes on Windows * Adds test for non-fatal exceptions. * Adds bpo number to comment. * bpo-30557: Fix test_faulthandler (#1969) On Windows 8, 8.1 and 10 at least, the exit code is the exception code (no bit is cleared). --- Lib/test/test_faulthandler.py | 28 ++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/faulthandler.c | 8 ++++---- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 01cbae3f200b1a..20fe1e2655946f 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,34 @@ def test_raise_exception(self): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_nonfatal_exception(self): + # These exceptions are not strictly errors. Letting + # faulthandler display the traceback when they are + # raised is likely to result in noise. However, they + # may still terminate the process if there is no + # handler installed for them (which there typically + # is, e.g. for debug messages). + for exc in ( + 0x00000000, + 0x34567890, + 0x40000000, + 0x40001000, + 0x70000000, + 0x7FFFFFFF, + ): + output, exitcode = self.get_output(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(0x{exc:x}) + """ + ) + self.assertEqual(output, []) + # On Windows older than 7 SP1, the actual exception code has + # bit 29 cleared. + self.assertIn(exitcode, + (exc, exc & ~0x10000000)) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') def test_disable_windows_exc_handler(self): code = dedent(""" diff --git a/Misc/NEWS b/Misc/NEWS index 1656e07f6eb1df..6177ee40ab6577 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-30557: faulthandler now correctly filters and displays exception codes + on Windows + - bpo-30378: Fix the problem that logging.handlers.SysLogHandler cannot handle IPv6 addresses. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 61fc4908b0fe9b..2efa998ff89aac 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -373,8 +373,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) DWORD code = exc_info->ExceptionRecord->ExceptionCode; DWORD flags = exc_info->ExceptionRecord->ExceptionFlags; - /* only log fatal exceptions */ - if (flags & EXCEPTION_NONCONTINUABLE) { + /* bpo-30557: only log fatal exceptions */ + if (!(code & 0x80000000)) { /* call the next exception handler */ return EXCEPTION_CONTINUE_SEARCH; } @@ -391,8 +391,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: - PUTS(fd, "code "); - _Py_DumpDecimal(fd, code); + PUTS(fd, "code 0x"); + _Py_DumpHexadecimal(fd, code, 8); } PUTS(fd, "\n\n"); From 063f0b358331322d84f85008e6c01f80ff321fd6 Mon Sep 17 00:00:00 2001 From: gfyoung Date: Tue, 6 Jun 2017 17:17:18 -0400 Subject: [PATCH 0367/1345] bpo-29596: Improve clinic howto documentation (GH-1710) (GH-1976) Clarify that `two-pass` buffer can only be dumped once, and it prints out all text sent to it during all processing, even from Clinic blocks *after* the dumping point. --- Doc/howto/clinic.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index eaab20ad975548..083a299cb32a56 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1407,8 +1407,8 @@ Let's start with defining some terminology: ``two-pass`` A buffer like ``buffer``. However, a two-pass buffer can only - be written once, and it prints out all text sent to it during - all of processing, even from Clinic blocks *after* the + be dumped once, and it prints out all text sent to it during + all processing, even from Clinic blocks *after* the dumping point. ``suppress`` The text is suppressed—thrown away. @@ -1471,7 +1471,7 @@ preset configurations, as follows: The default filename is ``"{dirname}/clinic/{basename}.h"``. ``buffer`` - Save up all most of the output from Clinic, to be written into + Save up most of the output from Clinic, to be written into your file near the end. For Python files implementing modules or builtin types, it's recommended that you dump the buffer just above the static structures for your module or From 6fb12b5c43945f61f3da82e33eafb4ddae2296ee Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 6 Jun 2017 17:31:03 -0700 Subject: [PATCH 0368/1345] bpo-29581: bpo-29581: Make ABCMeta.__new__ pass **kwargs to type.__new__ (GH-527) (GH-1282) Many metaclasses in the standard library don't play nice with __init_subclass__. This bug makes ABCMeta in particular with __init_subclass__, which is an 80/20 solution for me personally. AFAICT, a general solution to this problem requires updating all metaclasses in the standard library to make sure they pass **kwargs to type.__new__, whereas this PR only fixes ABCMeta. For context, see https://bugs.python.org/issue29581. * added a test combining ABCMeta and __init_subclass__ * Added NEWS item (cherry picked from commit bd583ef9857d99f9145ad0bb2c4424cc0baa63fc) * [3.6] bpo-29581: Make ABCMeta.__new__ pass **kwargs to type.__new__ (GH-527) Many metaclasses in the standard library don't play nice with __init_subclass__. This bug makes ABCMeta in particular with __init_subclass__, which is an 80/20 solution for me personally. AFAICT, a general solution to this problem requires updating all metaclasses in the standard library to make sure they pass **kwargs to type.__new__, whereas this PR only fixes ABCMeta. For context, see https://bugs.python.org/issue29581. * added a test combining ABCMeta and __init_subclass__ * Added NEWS item. (cherry picked from commit bd583ef9857d99f9145ad0bb2c4424cc0baa63fc) * **kwargs -> ``kwargs`` in attempts to fix the Travis build. * Quote the **kwargs --- Lib/abc.py | 4 ++-- Lib/test/test_abc.py | 12 ++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Lib/abc.py b/Lib/abc.py index 1cbf96a61f238f..43a34a0bbded78 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -129,8 +129,8 @@ class ABCMeta(type): # external code. _abc_invalidation_counter = 0 - def __new__(mcls, name, bases, namespace): - cls = super().__new__(mcls, name, bases, namespace) + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index e1765f0d5a54cb..4bc838200413af 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -404,5 +404,17 @@ class C(A, B): self.assertEqual(B.counter, 1) +class TestABCWithInitSubclass(unittest.TestCase): + def test_works_with_init_subclass(self): + saved_kwargs = {} + class ReceivesClassKwargs: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__() + saved_kwargs.update(kwargs) + class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): + pass + self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 6177ee40ab6577..515960daf7930f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-29581: ABCMeta.__new__ now accepts ``**kwargs``, allowing abstract base + classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. + - bpo-30557: faulthandler now correctly filters and displays exception codes on Windows From 09b6c0c71ea944f7e8b46998f3ebaf5b9fbe15f6 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 6 Jun 2017 21:21:34 -0700 Subject: [PATCH 0369/1345] [3.6] bpo-29822: make inspect.isabstract() work during __init_subclass__ (#1979) At the time when an abstract base class' __init_subclass__ runs, ABCMeta.__new__ has not yet finished running, so in the presence of __init_subclass__, inspect.isabstract() can no longer depend only on TPFLAGS_IS_ABSTRACT. (cherry picked from commit fcfe80ec2592fed8b3941c79056a8737abef7d3b) --- Lib/inspect.py | 23 ++++++++++++++++++++++- Lib/test/test_inspect.py | 24 ++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index a2dcb888a0c609..2894672f501e8b 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -31,6 +31,7 @@ __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') +import abc import ast import dis import collections.abc @@ -291,7 +292,27 @@ def isroutine(object): def isabstract(object): """Return true if the object is an abstract base class (ABC).""" - return bool(isinstance(object, type) and object.__flags__ & TPFLAGS_IS_ABSTRACT) + if not isinstance(object, type): + return False + if object.__flags__ & TPFLAGS_IS_ABSTRACT: + return True + if not issubclass(type(object), abc.ABCMeta): + return False + if hasattr(object, '__abstractmethods__'): + # It looks like ABCMeta.__new__ has finished running; + # TPFLAGS_IS_ABSTRACT should have been accurate. + return False + # It looks like ABCMeta.__new__ has not finished running yet; we're + # probably in __init_subclass__. We'll look for abstractmethods manually. + for name, value in object.__dict__.items(): + if getattr(value, "__isabstractmethod__", False): + return True + for base in object.__bases__: + for name in getattr(base, "__abstractmethods__", ()): + value = getattr(object, name, None) + if getattr(value, "__isabstractmethod__", False): + return True + return False def getmembers(object, predicate=None): """Return all members of an object as (name, value) pairs sorted by name. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index cfea281c70db11..37e451188a97be 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -232,6 +232,30 @@ def foo(self): self.assertFalse(inspect.isabstract(int)) self.assertFalse(inspect.isabstract(5)) + def test_isabstract_during_init_subclass(self): + from abc import ABCMeta, abstractmethod + isabstract_checks = [] + class AbstractChecker(metaclass=ABCMeta): + def __init_subclass__(cls): + isabstract_checks.append(inspect.isabstract(cls)) + class AbstractClassExample(AbstractChecker): + @abstractmethod + def foo(self): + pass + class ClassExample(AbstractClassExample): + def foo(self): + pass + self.assertEqual(isabstract_checks, [True, False]) + + isabstract_checks.clear() + class AbstractChild(AbstractClassExample): + pass + class AbstractGrandchild(AbstractChild): + pass + class ConcreteGrandchild(ClassExample): + pass + self.assertEqual(isabstract_checks, [True, True, False]) + class TestInterpreterStack(IsTestBase): def __init__(self, *args, **kwargs): diff --git a/Misc/NEWS b/Misc/NEWS index 515960daf7930f..b819c6db73af2f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-29822: inspect.isabstract() now works during __init_subclass__. Patch + by Nate Soares. + - bpo-29581: ABCMeta.__new__ now accepts ``**kwargs``, allowing abstract base classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. From ceabf9acf03f9bbe660d856bff90ecab475ab543 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Wed, 7 Jun 2017 19:18:56 +0200 Subject: [PATCH 0370/1345] bpo-30177: pathlib: include the full path in resolve(strict=False) (#1893) (#1985) --- Lib/pathlib.py | 19 ++++++++----------- Lib/test/test_pathlib.py | 21 ++++++++++++--------- Misc/ACKS | 1 + 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 70f5cba76ffa8a..48b566d9295f8f 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -187,19 +187,18 @@ def resolve(self, path, strict=False): if strict: return self._ext_to_normal(_getfinalpathname(s)) else: + tail_parts = [] # End of the path after the first one not found while True: try: s = self._ext_to_normal(_getfinalpathname(s)) except FileNotFoundError: previous_s = s - s = os.path.dirname(s) + s, tail = os.path.split(s) + tail_parts.append(tail) if previous_s == s: return path else: - if previous_s is None: - return s - else: - return s + os.path.sep + os.path.basename(previous_s) + return os.path.join(s, *reversed(tail_parts)) # Means fallback on absolute return None @@ -330,12 +329,10 @@ def _resolve(path, rest): try: target = accessor.readlink(newpath) except OSError as e: - if e.errno != EINVAL: - if strict: - raise - else: - return newpath - # Not a symlink + if e.errno != EINVAL and strict: + raise + # Not a symlink, or non-strict mode. We just leave the path + # untouched. path = newpath else: seen[newpath] = None # not resolved symlink diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 846f721e8d9abb..6502413a73d061 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1507,10 +1507,10 @@ def test_resolve_common(self): os.path.join(BASE, 'foo')) p = P(BASE, 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) + os.path.join(BASE, 'foo', 'in', 'spam')) p = P(BASE, '..', 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo'))) + os.path.abspath(os.path.join('foo', 'in', 'spam'))) # These are all relative symlinks p = P(BASE, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -1522,16 +1522,18 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) # Now create absolute symlinks d = tempfile.mkdtemp(suffix='-dirD') self.addCleanup(support.rmtree, d) @@ -1541,16 +1543,17 @@ def test_resolve_common(self): self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + False) p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo'), False) + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) @with_symlinks def test_resolve_dot(self): @@ -1564,7 +1567,7 @@ def test_resolve_dot(self): r = q / '3' / '4' self.assertRaises(FileNotFoundError, r.resolve, strict=True) # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3') + self.assertEqual(r.resolve(strict=False), p / '3' / '4') def test_with(self): p = self.cls(BASE) diff --git a/Misc/ACKS b/Misc/ACKS index 7fc5fecc8a4605..bbdc69d9c9fd64 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1190,6 +1190,7 @@ Steve Piercy Jim St. Pierre Dan Pierson Martijn Pieters +Antoine Pietri Anand B. Pillai François Pinard Tom Pinckney From 2c7f927369922562adecd5c25ad55b4bf733f716 Mon Sep 17 00:00:00 2001 From: Matthias Klose Date: Wed, 7 Jun 2017 18:07:41 -0700 Subject: [PATCH 0371/1345] [3.6] trivial: update config.{guess,sub} from gnu.org. (GH-1987) (#1990) * Update config.{guess,sub} from gnu.org. (cherry picked from commit 6f46683a6257f22f25d136ed080d58d0c060a43b) --- config.guess | 197 ++++++++++++++++++++++++++++++++------------------- config.sub | 90 ++++++++++++++++------- 2 files changed, 191 insertions(+), 96 deletions(-) diff --git a/config.guess b/config.guess index 1f5c50c0d1529d..2193702b12af30 100755 --- a/config.guess +++ b/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-03-23' +timestamp='2017-05-27' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ timestamp='2014-03-23' # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # -# Originally written by Per Bothner. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . me=`echo "$0" | sed -e 's,.*/,,'` @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -168,19 +168,29 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-unknown + ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. + # to ELF recently (or will in the future) and ABI. case "${UNAME_MACHINE_ARCH}" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ @@ -197,6 +207,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need @@ -207,13 +224,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -223,6 +240,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} exit ;; @@ -235,6 +256,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-unknown-sortix + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -251,42 +275,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -359,16 +383,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build - SUN_ARCH="i386" + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` @@ -393,7 +417,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos${UNAME_RELEASE} @@ -579,8 +603,9 @@ EOF else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi @@ -617,13 +642,13 @@ EOF sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi @@ -662,11 +687,11 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ ${HP_ARCH} = hppa2.0w ] then eval $set_cc_for_build @@ -679,12 +704,12 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} @@ -789,14 +814,14 @@ EOF echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -812,10 +837,11 @@ EOF UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin @@ -878,7 +904,7 @@ EOF exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix @@ -901,7 +927,7 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arc:Linux:*:* | arceb:Linux:*:*) @@ -932,6 +958,9 @@ EOF crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -944,6 +973,9 @@ EOF ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -969,6 +1001,9 @@ EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; + mips64el:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; openrisc*:Linux:*:*) echo or1k-unknown-linux-${LIBC} exit ;; @@ -1001,6 +1036,9 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-${LIBC} exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux-${LIBC} exit ;; @@ -1020,7 +1058,7 @@ EOF echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} @@ -1099,7 +1137,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1248,6 +1286,9 @@ EOF SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux${UNAME_RELEASE} exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} exit ;; @@ -1261,16 +1302,23 @@ EOF UNAME_PROCESSOR=powerpc fi if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi fi elif test "$UNAME_PROCESSOR" = i386 ; then # Avoid executing cc on OS X 10.9, as it ships with a stub @@ -1285,7 +1333,7 @@ EOF exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi @@ -1294,15 +1342,18 @@ EOF *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) + NEO-*:NONSTOP_KERNEL:*:*) echo neo-tandem-nsk${UNAME_RELEASE} exit ;; NSE-*:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk${UNAME_RELEASE} exit ;; - NSR-?:NONSTOP_KERNEL:*:*) + NSR-*:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; @@ -1316,7 +1367,7 @@ EOF # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" @@ -1358,7 +1409,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` exit ;; i*86:rdos:*:*) echo ${UNAME_MACHINE}-pc-rdos @@ -1369,23 +1420,25 @@ EOF x86_64:VMkernel:*:*) echo ${UNAME_MACHINE}-unknown-esx exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs + exit ;; esac cat >&2 < in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp diff --git a/config.sub b/config.sub index d654d03cdcd222..40ea5dfe1152f0 100755 --- a/config.sub +++ b/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-05-01' +timestamp='2017-04-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ timestamp='2014-05-01' # of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -33,7 +33,7 @@ timestamp='2014-05-01' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -53,8 +53,7 @@ timestamp='2014-05-01' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. @@ -68,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -117,8 +116,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ + kopensolaris*-gnu* | cloudabi*-eabi* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` @@ -255,15 +254,16 @@ case $basic_machine in | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ + | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ @@ -301,10 +301,12 @@ case $basic_machine in | open8 | or1k | or1knd | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ | pyramid \ + | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -312,6 +314,8 @@ case $basic_machine in | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ + | wasm32 \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -326,6 +330,9 @@ case $basic_machine in c6x) basic_machine=tic6x-unknown ;; + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) basic_machine=$basic_machine-unknown os=-none @@ -371,17 +378,18 @@ case $basic_machine in | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ + | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ @@ -422,13 +430,15 @@ case $basic_machine in | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ | pyramid-* \ + | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ @@ -436,6 +446,8 @@ case $basic_machine in | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ + | visium-* \ + | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -512,6 +524,9 @@ case $basic_machine in basic_machine=i386-pc os=-aros ;; + asmjs) + basic_machine=asmjs-unknown + ;; aux) basic_machine=m68k-apple os=-aux @@ -632,6 +647,14 @@ case $basic_machine in basic_machine=m68k-bull os=-sysv3 ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + os=$os"spe" + ;; ebmon29k) basic_machine=a29k-amd os=-ebmon @@ -773,6 +796,9 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` + ;; m68knommu) basic_machine=m68k-unknown os=-linux @@ -828,6 +854,10 @@ case $basic_machine in basic_machine=powerpc-unknown os=-morphos ;; + moxiebox) + basic_machine=moxie-unknown + os=-moxiebox + ;; msdos) basic_machine=i386-pc os=-msdos @@ -920,6 +950,9 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf @@ -1004,7 +1037,7 @@ case $basic_machine in ppc-* | ppcbe-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppcle | powerpclittle | ppc-le | powerpc-little) + ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) @@ -1014,7 +1047,7 @@ case $basic_machine in ;; ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) + ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) @@ -1215,6 +1248,9 @@ case $basic_machine in basic_machine=a29k-wrs os=-vxworks ;; + wasm32) + basic_machine=wasm32-unknown + ;; w65*) basic_machine=w65-wdc os=-none @@ -1360,27 +1396,28 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ + | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ + | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ + | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*) + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ + | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1512,6 +1549,8 @@ case $os in ;; -nacl*) ;; + -ios) + ;; -none) ;; *) @@ -1607,6 +1646,9 @@ case $basic_machine in sparc-* | *-sun) os=-sunos4.1.1 ;; + pru-*) + os=-elf + ;; *-be) os=-beos ;; From 854f7ba1d5cbb6a42511beae66c8dbe34f2cbcd3 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 8 Jun 2017 04:14:40 -0700 Subject: [PATCH 0372/1345] [3.6] bpo-30594: Fixed refcounting in newPySSLSocket (GH-1992) (#1994) If pass a server_hostname= that fails IDNA decoding to SSLContext.wrap_socket or SSLContext.wrap_bio, then the SSLContext object had a spurious Py_DECREF called on it, eventually leading to segfaults. (cherry picked from commit 65ece7ca2366308fa91a39a8dfa255e6bdce3cca) --- Modules/_ssl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2a2c18fe2f7309..dbfbd44399299b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -599,6 +599,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->ssl = NULL; self->Socket = NULL; self->ctx = sslctx; + Py_INCREF(sslctx); self->shutdown_seen_zero = 0; self->handshake_done = 0; self->owner = NULL; @@ -613,8 +614,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->server_hostname = hostname; } - Py_INCREF(sslctx); - /* Make sure the SSL error state is initialized */ (void) ERR_get_state(); ERR_clear_error(); From ca1b66fd059758bbd6e2cc62c8f2adecefe8f942 Mon Sep 17 00:00:00 2001 From: Denis Osipov Date: Thu, 8 Jun 2017 17:02:05 +0500 Subject: [PATCH 0373/1345] [3.6] bpo-30584: Fix test_os fails on non-English Windows (GH-1980) (#1999) * Fix bpo-30584 * Adding a comment mentionning the bpo and explaining what is the identifier * Add Denis Osipov to Misc/ACKS (cherry picked from commit 897bba75632dfce87c355e3cd4700468357715a7) --- Lib/test/test_os.py | 4 +++- Misc/ACKS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e9fdb0719f4bf4..46ad2099a9f524 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -473,7 +473,9 @@ def test_access_denied(self): # force CreateFile to fail with ERROR_ACCESS_DENIED. DETACHED_PROCESS = 8 subprocess.check_call( - ['icacls.exe', fname, '/deny', 'Users:(S)'], + # bpo-30584: Use security identifier *S-1-5-32-545 instead + # of localized "Users" to not depend on the locale. + ['icacls.exe', fname, '/deny', '*S-1-5-32-545:(S)'], creationflags=DETACHED_PROCESS ) result = os.stat(fname) diff --git a/Misc/ACKS b/Misc/ACKS index bbdc69d9c9fd64..c2ebf7faa40a9e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1121,6 +1121,7 @@ William Orr Michele Orrù Tomáš Orsava Oleg Oshmyan +Denis Osipov Denis S. Otkidach Peter Otten Michael Otteneder From 31b950ab86f5c99a18c16dbf1900b854b94e0659 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Jun 2017 23:13:12 +0200 Subject: [PATCH 0374/1345] bpo-30601: Fix a refleak in WindowsConsoleIO (#2003) (#2008) Fix a reference leak in _io._WindowsConsoleIO: PyUnicode_FSDecoder() always initialize decodedname when it succeed and it doesn't clear input decodedname object. (cherry picked from commit 29adc13bd797d9c9e7fcb893a7c49ce7f7ad388c) --- Modules/_io/winconsoleio.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index cbe21422bb4653..94dfe3e4c63146 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -101,7 +101,7 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { DWORD length; wchar_t name_buf[MAX_PATH], *pname_buf = name_buf; - + length = GetFullPathNameW(decoded_wstr, MAX_PATH, pname_buf, NULL); if (length > MAX_PATH) { pname_buf = PyMem_New(wchar_t, length); @@ -305,8 +305,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = fd; if (fd < 0) { - PyObject *decodedname = Py_None; - Py_INCREF(decodedname); + PyObject *decodedname; int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname); if (!d) From b319d09ee4427aac1ee8f298692127d34ef57dc0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Jun 2017 23:14:07 +0200 Subject: [PATCH 0375/1345] bpo-30418: Popen.communicate() always ignore EINVAL (#2002) (#2004) On Windows, subprocess.Popen.communicate() now also ignore EINVAL on stdin.write() if the child process is still running but closed the pipe. (cherry picked from commit d52aa31378ae43e044a300edfe8285954c167216) --- Lib/subprocess.py | 14 ++++++++------ Misc/NEWS | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 172126929160bf..e626a8afdbb7f0 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -776,19 +776,21 @@ def _stdin_write(self, input): self.stdin.write(input) except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: - # Issue #19612: On Windows, stdin.write() fails with EINVAL - # if the process already exited before the write + except OSError as exc: + if exc.errno == errno.EINVAL: + # bpo-19612, bpo-30418: On Windows, stdin.write() fails + # with EINVAL if the child process exited or if the child + # process is still running but closed the pipe. pass else: raise + try: self.stdin.close() except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: + except OSError as exc: + if exc.errno == errno.EINVAL: pass else: raise diff --git a/Misc/NEWS b/Misc/NEWS index b819c6db73af2f..d10e7a371f489d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,9 @@ Core and Builtins Library ------- +- bpo-30418: On Windows, subprocess.Popen.communicate() now also ignore EINVAL + on stdin.write() if the child process is still running but closed the pipe. + - bpo-29822: inspect.isabstract() now works during __init_subclass__. Patch by Nate Soares. From 570b1c971c31cd08dbf060f4e21636c40aa47786 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 9 Jun 2017 00:38:06 +0300 Subject: [PATCH 0376/1345] [3.6] bpo-30529: Fix errors for invalid whitespaces in f-string subexpressions. (GH-1888) (#2013) 'invalid character in identifier' now is raised instead of 'f-string: empty expression not allowed' if a subexpression contains only whitespaces and they are not accepted by Python parser. (cherry picked from commit 2e9cd58) --- Lib/test/test_fstring.py | 10 ++++++++++ Python/ast.c | 31 +++++++------------------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 25730029ae76f1..b39870457af492 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -280,6 +280,10 @@ def test_missing_expression(self): "f'{10:{ }}'", "f' { } '", + # The Python parser ignores also the following + # whitespace characters in additional to a space. + "f'''{\t\f\r\n}'''", + # Catch the empty expression before the # invalid conversion. "f'{!x}'", @@ -300,6 +304,12 @@ def test_missing_expression(self): "f'{:x'", ]) + # Different error message is raised for other whitespace characters. + self.assertAllRaise(SyntaxError, 'invalid character in identifier', + ["f'''{\xa0}'''", + "\xa0", + ]) + def test_parens_in_expressions(self): self.assertEqual(f'{3,}', '(3,)') diff --git a/Python/ast.c b/Python/ast.c index ed05a1e53bdb5e..c61ca4bbcd5528 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4247,49 +4247,32 @@ fstring_compile_expr(const char *expr_start, const char *expr_end, struct compiling *c, const node *n) { - int all_whitespace = 1; - int kind; - void *data; PyCompilerFlags cf; mod_ty mod; char *str; - PyObject *o; Py_ssize_t len; - Py_ssize_t i; + const char *s; assert(expr_end >= expr_start); assert(*(expr_start-1) == '{'); assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':'); - /* We know there are no escapes here, because backslashes are not allowed, - and we know it's utf-8 encoded (per PEP 263). But, in order to check - that each char is not whitespace, we need to decode it to unicode. - Which is unfortunate, but such is life. */ - /* If the substring is all whitespace, it's an error. We need to catch this here, and not when we call PyParser_ASTFromString, because turning the expression '' in to '()' would go from being invalid to valid. */ - /* Note that this code says an empty string is all whitespace. That's - important. There's a test for it: f'{}'. */ - o = PyUnicode_DecodeUTF8(expr_start, expr_end-expr_start, NULL); - if (o == NULL) - return NULL; - len = PyUnicode_GET_LENGTH(o); - kind = PyUnicode_KIND(o); - data = PyUnicode_DATA(o); - for (i = 0; i < len; i++) { - if (!Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, i))) { - all_whitespace = 0; + for (s = expr_start; s != expr_end; s++) { + char c = *s; + /* The Python parser ignores only the following whitespace + characters (\r already is converted to \n). */ + if (!(c == ' ' || c == '\t' || c == '\n' || c == '\f')) { break; } } - Py_DECREF(o); - if (all_whitespace) { + if (s == expr_end) { ast_error(c, n, "f-string: empty expression not allowed"); return NULL; } - /* Reuse len to be the length of the utf-8 input string. */ len = expr_end - expr_start; /* Allocate 3 extra bytes: open paren, close paren, null byte. */ str = PyMem_RawMalloc(len + 3); From f0ff849adc6b4a01f9d1f08d9ad0f1511ff84541 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Jun 2017 13:24:53 +0200 Subject: [PATCH 0377/1345] bpo-30524: Fix _PyStack_UnpackDict() (#1886) * bpo-29259: Remove unused func parameter of _PyStack_UnpackDict() * bpo-29286: Change _PyStack_UnpackDict() prototype to be able to notify of failure when args is NULL. _PyStack_UnpackDict() now returns -1 on error. --- Include/abstract.h | 16 +++++++++------- Objects/abstract.c | 19 ++++++++++--------- Objects/methodobject.c | 3 +-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Include/abstract.h b/Include/abstract.h index 7d137a22bfb7a8..6bb76ec95f1b2e 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -290,21 +290,23 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ PyObject **values, PyObject *kwnames); - /* Convert (args, nargs, kwargs) into a (stack, nargs, kwnames). + /* Convert (args, nargs, kwargs: dict) into (stack, nargs, kwnames: tuple). - Return a new stack which should be released by PyMem_Free(), or return - args unchanged if kwargs is NULL or an empty dictionary. + Return 0 on success, raise an exception and return -1 on error. + + Write the new stack into *p_stack. If *p_stack is differen than args, it + must be released by PyMem_Free(). The stack uses borrowed references. The type of keyword keys is not checked, these checks should be done - later (ex: _PyArg_ParseStack). */ - PyAPI_FUNC(PyObject **) _PyStack_UnpackDict( + later (ex: _PyArg_ParseStackAndKeywords). */ + PyAPI_FUNC(int) _PyStack_UnpackDict( PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **kwnames, - PyObject *func); + PyObject ***p_stack, + PyObject **p_kwnames); /* Call the callable object func with the "fast call" calling convention: args is a C array for positional arguments (nargs is the number of diff --git a/Objects/abstract.c b/Objects/abstract.c index d838856d452872..3585b34ca4d8bd 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2389,9 +2389,9 @@ _PyStack_AsDict(PyObject **values, PyObject *kwnames) return kwdict; } -PyObject ** +int _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **p_kwnames, PyObject *func) + PyObject ***p_stack, PyObject **p_kwnames) { PyObject **stack, **kwstack; Py_ssize_t nkwargs; @@ -2402,27 +2402,27 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, assert(nargs >= 0); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - nkwargs = (kwargs != NULL) ? PyDict_Size(kwargs) : 0; - if (!nkwargs) { + if (kwargs == NULL || (nkwargs = PyDict_Size(kwargs)) == 0) { + *p_stack = args; *p_kwnames = NULL; - return args; + return 0; } if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { PyErr_NoMemory(); - return NULL; + return -1; } stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); if (stack == NULL) { PyErr_NoMemory(); - return NULL; + return -1; } kwnames = PyTuple_New(nkwargs); if (kwnames == NULL) { PyMem_Free(stack); - return NULL; + return -1; } /* Copy position arguments (borrowed references) */ @@ -2441,8 +2441,9 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, i++; } + *p_stack = stack; *p_kwnames = kwnames; - return stack; + return 0; } PyObject * diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c2001f0169b030..fe52545667f2ed 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -243,8 +243,7 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs, PyObject *kwnames; _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - stack = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames, func_obj); - if (stack == NULL) { + if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { return NULL; } From c422959daca0598fabbefa5de4417ebce3b1da06 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 9 Jun 2017 22:37:53 +1000 Subject: [PATCH 0378/1345] [3.6] bpo-19180: Updated references for RFC 1750, RFC 3280 & RFC 4366 * RFC 1750 has been been obsoleted by RFC 4086. * RFC 3280 has been obsoleted by RFC 5280. * RFC 4366 has been obsoleted by RFC 6066. (cherry picked from commit 63c2c8ac17750ba2be2cfc4e339cae1f4edee54f) --- Doc/library/ssl.rst | 19 ++++++++----------- Modules/_ssl.c | 2 +- Modules/clinic/_ssl.c.h | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 0ce73c14095480..16756e1d73e767 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -374,9 +374,9 @@ Certificate handling Verify that *cert* (in decoded format as returned by :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818` and :rfc:`6125`. In addition to HTTPS, this function - should be suitable for checking the identity of servers in various - SSL-based protocols such as FTPS, IMAPS, POPS and others. + in :rfc:`2818`, :rfc:`5280` and :rfc:`6125`. In addition to HTTPS, this + function should be suitable for checking the identity of servers in + various SSL-based protocols such as FTPS, IMAPS, POPS and others. :exc:`CertificateError` is raised on failure. On success, the function returns nothing:: @@ -820,7 +820,7 @@ Constants .. data:: HAS_SNI Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`4366`). + Indication* extension (as defined in :rfc:`6066`). .. versionadded:: 3.2 @@ -2306,14 +2306,11 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent - `RFC 1750: Randomness Recommendations for Security `_ - D. Eastlake et. al. + `RFC 4086: Randomness Requirements for Security `_ + Donald E., Jeffrey I. Schiller - `RFC 3280: Internet X.509 Public Key Infrastructure Certificate and CRL Profile `_ - Housley et. al. - - `RFC 4366: Transport Layer Security (TLS) Extensions `_ - Blake-Wilson et. al. + `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_ + D. Cooper `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_ T. Dierks et. al. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index dbfbd44399299b..a421fc0d2d6ff2 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4475,7 +4475,7 @@ _ssl.RAND_add Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in -string. See RFC 1750. +string. See RFC 4086. [clinic start generated code]*/ static PyObject * diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 29f58384394c98..333aad76b404fc 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -839,7 +839,7 @@ PyDoc_STRVAR(_ssl_RAND_add__doc__, "Mix string into the OpenSSL PRNG state.\n" "\n" "entropy (a float) is a lower bound on the entropy contained in\n" -"string. See RFC 1750."); +"string. See RFC 4086."); #define _SSL_RAND_ADD_METHODDEF \ {"RAND_add", (PyCFunction)_ssl_RAND_add, METH_VARARGS, _ssl_RAND_add__doc__}, From af609a00a64fd58bd67c102f208ed9fe15c2748d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 9 Jun 2017 07:32:46 -0700 Subject: [PATCH 0379/1345] bpo-24755: Document asyncio.wrap_future (GH-603) (GH-2019) (cherry picked from commit 824f6879121413e09439fffef54580413e44bf46) --- Doc/library/asyncio-task.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 558d17c0969793..804f1925b40929 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -540,6 +540,11 @@ Task functions .. deprecated:: 3.4.4 +.. function:: wrap_future(future, \*, loop=None) + + Wrap a :class:`concurrent.futures.Future` object in a :class:`Future` + object. + .. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) Return a future aggregating results from the given coroutine objects or From bbeaccc76b2a4a7e2601579446c84cb12fea5f05 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 9 Jun 2017 15:59:31 -0400 Subject: [PATCH 0380/1345] [3.6] IDLE test_textview: add comments and test, increase coverage to 100% (GH-1641) (#2018) (cherry picked from commit 295304d412700cc6621bb592109fa42249a9dcdb) --- Lib/idlelib/idle_test/test_textview.py | 33 +++++++++++++++++++------- Lib/idlelib/textview.py | 1 + 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index 67fc0b7c4f1c6d..57b886aba11f31 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -5,7 +5,7 @@ Using mock Text would not change this. Other mocks are used to retrieve information about calls. -Coverage: 94%. +Coverage: 100%. ''' from idlelib import textview as tv from test.support import requires @@ -28,14 +28,20 @@ def tearDownModule(): root.destroy() # Pyflakes falsely sees root as undefined. del root +# If we call TextViewer or wrapper functions with defaults +# modal=True, _utest=False, test hangs on call to wait_window. +# Have also gotten tk error 'can't invoke "event" command'. + class TV(tv.TextViewer): # Used in TextViewTest. transient = Func() grab_set = Func() wait_window = Func() -class TextViewTest(unittest.TestCase): +# Call wrapper class with mock wait_window. +class TextViewTest(unittest.TestCase): + def setUp(self): TV.transient.__init__() TV.grab_set.__init__() @@ -64,6 +70,7 @@ def test_ok(self): view.destroy() +# Call TextViewer with modal=False. class ViewFunctionTest(unittest.TestCase): @classmethod @@ -77,26 +84,34 @@ def tearDownClass(cls): del cls.orig_error def test_view_text(self): - # If modal True, get tk error 'can't invoke "event" command'. view = tv.view_text(root, 'Title', 'test text', modal=False) self.assertIsInstance(view, tv.TextViewer) view.Ok() def test_view_file(self): - test_dir = os.path.dirname(__file__) - testfile = os.path.join(test_dir, 'test_textview.py') - view = tv.view_file(root, 'Title', testfile, modal=False) + view = tv.view_file(root, 'Title', __file__, modal=False) self.assertIsInstance(view, tv.TextViewer) self.assertIn('Test', view.textView.get('1.0', '1.end')) view.Ok() + def test_bad_file(self): # Mock showerror will be used; view_file will return None. - testfile = os.path.join(test_dir, '../notthere.py') - view = tv.view_file(root, 'Title', testfile, modal=False) + view = tv.view_file(root, 'Title', 'abc.xyz', modal=False) self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'File Load Error') + + def test_bad_encoding(self): + p = os.path + fn = p.abspath(p.join(p.dirname(__file__), '..', 'CREDITS.txt')) + tv.showerror.title = None + view = tv.view_file(root, 'Title', fn, 'ascii', modal=False) + self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'Unicode Decode Error') + -class ButtonClickTextViewTest(unittest.TestCase): +# Call TextViewer with _utest=True. +class ButtonClickTest(unittest.TestCase): def setUp(self): self.view = None diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index f33ac32a33a3bb..dc1a7bfc9eff38 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -87,6 +87,7 @@ def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): parent=parent) else: return view_text(parent, title, contents, modal, _utest=_utest) + return None if __name__ == '__main__': From b7577456c430283f8b7ec4e914b701cb943cc69b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Jun 2017 22:28:32 +0200 Subject: [PATCH 0381/1345] bpo-30524: Write unit tests for FASTCALL (#2022) (#2030) Test C functions: * _PyObject_FastCall() * _PyObject_FastCallDict() * _PyObject_FastCallKeywords() (cherry picked from commit 3b5cf85edc188345668f987c824a2acb338a7816) --- Lib/test/test_call.py | 176 ++++++++++++++++++++++++++++++++++++++ Modules/_testcapimodule.c | 101 ++++++++++++++++++++++ 2 files changed, 277 insertions(+) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index e2b8e0fd1233d9..2e8819be5dac68 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,4 +1,10 @@ +import datetime import unittest +from test.support import cpython_only +try: + import _testcapi +except ImportError: + _testcapi = None # The test cases here cover several paths through the function calling # code. They depend on the METH_XXX flag that is used to define a C @@ -122,5 +128,175 @@ def test_oldargs1_2_kw(self): self.assertRaises(TypeError, [].count, x=2, y=2) +def pyfunc(arg1, arg2): + return [arg1, arg2] + + +def pyfunc_noarg(): + return "noarg" + + +class PythonClass: + def method(self, arg1, arg2): + return [arg1, arg2] + + def method_noarg(self): + return "noarg" + + @classmethod + def class_method(cls): + return "classmethod" + + @staticmethod + def static_method(): + return "staticmethod" + + +PYTHON_INSTANCE = PythonClass() + + +IGNORE_RESULT = object() + + +@cpython_only +class FastCallTests(unittest.TestCase): + # Test calls with positional arguments + CALLS_POSARGS = ( + # (func, args: tuple, result) + + # Python function with 2 arguments + (pyfunc, (1, 2), [1, 2]), + + # Python function without argument + (pyfunc_noarg, (), "noarg"), + + # Python class methods + (PythonClass.class_method, (), "classmethod"), + (PythonClass.static_method, (), "staticmethod"), + + # Python instance methods + (PYTHON_INSTANCE.method, (1, 2), [1, 2]), + (PYTHON_INSTANCE.method_noarg, (), "noarg"), + (PYTHON_INSTANCE.class_method, (), "classmethod"), + (PYTHON_INSTANCE.static_method, (), "staticmethod"), + + # C function: METH_NOARGS + (globals, (), IGNORE_RESULT), + + # C function: METH_O + (id, ("hello",), IGNORE_RESULT), + + # C function: METH_VARARGS + (dir, (1,), IGNORE_RESULT), + + # C function: METH_VARARGS | METH_KEYWORDS + (min, (5, 9), 5), + + # C function: METH_FASTCALL + (divmod, (1000, 33), (30, 10)), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00', 'little'), 1), + + # bpo-30524: Test that calling a C type static method with no argument + # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS + (datetime.datetime.now, (), IGNORE_RESULT), + ) + + # Test calls with positional and keyword arguments + CALLS_KWARGS = ( + # (func, args: tuple, kwargs: dict, result) + + # Python function with 2 arguments + (pyfunc, (1,), {'arg2': 2}, [1, 2]), + (pyfunc, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # Python instance methods + (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), + (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # C function: METH_VARARGS | METH_KEYWORDS + (max, ([],), {'default': 9}, 9), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), + (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), + ) + + def check_result(self, result, expected): + if expected is IGNORE_RESULT: + return + self.assertEqual(result, expected) + + def test_fastcall(self): + # Test _PyObject_FastCall() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + result = _testcapi.pyobject_fastcall(func, args) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0 + result = _testcapi.pyobject_fastcall(func, None) + self.check_result(result, expected) + + def test_fastcall_dict(self): + # Test _PyObject_FastCallDict() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, args, None) + self.check_result(result, expected) + + # kwargs={} + result = _testcapi.pyobject_fastcalldict(func, args, {}) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0, kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, None, None) + self.check_result(result, expected) + + # args=NULL, nargs=0, kwargs={} + result = _testcapi.pyobject_fastcalldict(func, None, {}) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + result = _testcapi.pyobject_fastcalldict(func, args, kwargs) + self.check_result(result, expected) + + def test_fastcall_keywords(self): + # Test _PyObject_FastCallKeywords() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, args, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, args, ()) + self.check_result(result, expected) + + if not args: + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, None, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, None, ()) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + kwnames = tuple(kwargs.keys()) + args = args + tuple(kwargs.values()) + result = _testcapi.pyobject_fastcallkeywords(func, args, kwnames) + self.check_result(result, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c76eefab4e80cf..5e73293e0e0eaf 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4027,6 +4027,104 @@ dict_get_version(PyObject *self, PyObject *args) } +static int +fastcall_args(PyObject *args, PyObject ***stack, Py_ssize_t *nargs) +{ + if (args == Py_None) { + *stack = NULL; + *nargs = 0; + } + else if (PyTuple_Check(args)) { + *stack = &PyTuple_GET_ITEM(args, 0); + *nargs = PyTuple_GET_SIZE(args); + } + else { + PyErr_SetString(PyExc_TypeError, "args must be None or a tuple"); + return -1; + } + return 0; +} + + +static PyObject * +test_pyobject_fastcall(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OO", &func, &func_args)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + return _PyObject_FastCall(func, stack, nargs); +} + + +static PyObject * +test_pyobject_fastcalldict(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwargs; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwargs)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwargs == Py_None) { + kwargs = NULL; + } + else if (!PyDict_Check(kwargs)) { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a dict"); + return NULL; + } + + return _PyObject_FastCallDict(func, stack, nargs, kwargs); +} + + +static PyObject * +test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwnames = NULL; + PyObject **stack; + Py_ssize_t nargs, nkw; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwnames)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwnames == Py_None) { + kwnames = NULL; + } + else if (PyTuple_Check(kwnames)) { + nkw = PyTuple_GET_SIZE(kwnames); + if (nargs < nkw) { + PyErr_SetString(PyExc_ValueError, "kwnames longer than args"); + return NULL; + } + nargs -= nkw; + } + else { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a tuple"); + return NULL; + } + return _PyObject_FastCallKeywords(func, stack, nargs, kwnames); +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4230,6 +4328,9 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, + {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, + {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, + {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From e89f95bfd0881a9b80c3b1430d154a77bdf5a824 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 9 Jun 2017 17:06:39 -0400 Subject: [PATCH 0382/1345] [3.6] bpo-30039: Don't run signal handlers while resuming a yield from stack (GH-1081) (#1640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we have a chain of generators/coroutines that are 'yield from'ing each other, then resuming the stack works like: - call send() on the outermost generator - this enters _PyEval_EvalFrameDefault, which re-executes the YIELD_FROM opcode - which calls send() on the next generator - which enters _PyEval_EvalFrameDefault, which re-executes the YIELD_FROM opcode - ...etc. However, every time we enter _PyEval_EvalFrameDefault, the first thing we do is to check for pending signals, and if there are any then we run the signal handler. And if it raises an exception, then we immediately propagate that exception *instead* of starting to execute bytecode. This means that e.g. a SIGINT at the wrong moment can "break the chain" – it can be raised in the middle of our yield from chain, with the bottom part of the stack abandoned for the garbage collector. The fix is pretty simple: there's already a special case in _PyEval_EvalFrameEx where it skips running signal handlers if the next opcode is SETUP_FINALLY. (I don't see how this accomplishes anything useful, but that's another story.) If we extend this check to also skip running signal handlers when the next opcode is YIELD_FROM, then that closes the hole – now the exception can only be raised at the innermost stack frame. This shouldn't have any performance implications, because the opcode check happens inside the "slow path" after we've already determined that there's a pending signal or something similar for us to process; the vast majority of the time this isn't true and the new check doesn't run at all.. (cherry picked from commit ab4413a7e9bda95b6fcd517073e2a51dafaa1624) --- Lib/test/test_generators.py | 29 +++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ Modules/_testcapimodule.c | 24 ++++++++++++++++++++++++ Python/ceval.c | 17 ++++++++++++++--- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index f81c82f705e43c..7107bd9993258f 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -10,6 +10,35 @@ from test import support +_testcapi = support.import_module('_testcapi') + + +# This tests to make sure that if a SIGINT arrives just before we send into a +# yield from chain, the KeyboardInterrupt is raised in the innermost +# generator (see bpo-30039). +class SignalAndYieldFromTest(unittest.TestCase): + + def generator1(self): + return (yield from self.generator2()) + + def generator2(self): + try: + yield + except KeyboardInterrupt: + return "PASSED" + else: + return "FAILED" + + def test_raise_and_yield_from(self): + gen = self.generator1() + gen.send(None) + try: + _testcapi.raise_SIGINT_then_send_None(gen) + except BaseException as _exc: + exc = _exc + self.assertIs(type(exc), StopIteration) + self.assertEqual(exc.value, "PASSED") + class FinalizationTest(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index d10e7a371f489d..580f5793a1e94c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,10 @@ Core and Builtins - bpo-25794: Fixed type.__setattr__() and type.__delattr__() for non-interned attribute names. Based on patch by Eryk Sun. +- bpo-30039: If a KeyboardInterrupt happens when the interpreter is in + the middle of resuming a chain of nested 'yield from' or 'await' + calls, it's now correctly delivered to the innermost frame. + - bpo-12414: sys.getsizeof() on a code object now returns the sizes which includes the code struct and sizes of objects which it references. Patch by Dong-hee Na. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5e73293e0e0eaf..c3d829c7139cad 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4125,6 +4125,29 @@ test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) } +static PyObject * +raise_SIGINT_then_send_None(PyObject *self, PyObject *args) +{ + PyGenObject *gen; + + if (!PyArg_ParseTuple(args, "O!", &PyGen_Type, &gen)) + return NULL; + + /* This is used in a test to check what happens if a signal arrives just + as we're in the process of entering a yield from chain (see + bpo-30039). + + Needs to be done in C, because: + - we don't have a Python wrapper for raise() + - we need to make sure that the Python-level signal handler doesn't run + *before* we enter the generator frame, which is impossible in Python + because we check for signals before every bytecode operation. + */ + raise(SIGINT); + return _PyGen_Send(gen, Py_None); +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4331,6 +4354,7 @@ static PyMethodDef TestMethods[] = { {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, + {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval.c b/Python/ceval.c index 5dc0444a1acf5c..eba892c1ce3b3c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1119,9 +1119,20 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) Py_MakePendingCalls() above. */ if (_Py_atomic_load_relaxed(&eval_breaker)) { - if (_Py_OPCODE(*next_instr) == SETUP_FINALLY) { - /* Make the last opcode before - a try: finally: block uninterruptible. */ + if (_Py_OPCODE(*next_instr) == SETUP_FINALLY || + _Py_OPCODE(*next_instr) == YIELD_FROM) { + /* Two cases where we skip running signal handlers and other + pending calls: + - If we're about to enter the try: of a try/finally (not + *very* useful, but might help in some cases and it's + traditional) + - If we're resuming a chain of nested 'yield from' or + 'await' calls, then each frame is parked with YIELD_FROM + as its next opcode. If the user hit control-C we want to + wait until we've reached the innermost frame before + running the signal handler and raising KeyboardInterrupt + (see bpo-30039). + */ goto fast_next_opcode; } if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) { From fa7f519113fd32f432f9f52a956bbcdc542dce93 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 9 Jun 2017 17:07:48 -0400 Subject: [PATCH 0383/1345] Fix waiter cancellation in asyncio.Lock (#1031) (#2037) Avoid a deadlock when the waiter who is about to take the lock is cancelled Issue #27585 --- Lib/asyncio/locks.py | 17 ++++++++++++----- Lib/test/test_asyncio/test_locks.py | 22 ++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index deefc938ecfb01..92661830a06228 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -176,6 +176,10 @@ def acquire(self): yield from fut self._locked = True return True + except futures.CancelledError: + if not self._locked: + self._wake_up_first() + raise finally: self._waiters.remove(fut) @@ -192,14 +196,17 @@ def release(self): """ if self._locked: self._locked = False - # Wake up the first waiter who isn't cancelled. - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - break + self._wake_up_first() else: raise RuntimeError('Lock is not acquired.') + def _wake_up_first(self): + """Wake up the first waiter who isn't cancelled.""" + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + break + class Event: """Asynchronous equivalent to threading.Event. diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 152948c8138975..c85e8b1a32f73a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -176,6 +176,28 @@ def lockit(name, blocker): self.assertTrue(tb.cancelled()) self.assertTrue(tc.done()) + def test_finished_waiter_cancelled(self): + lock = asyncio.Lock(loop=self.loop) + + ta = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertTrue(lock.locked()) + + tb = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual(len(lock._waiters), 1) + + # Create a second waiter, wake up the first, and cancel it. + # Without the fix, the second was not woken up. + tc = asyncio.Task(lock.acquire(), loop=self.loop) + lock.release() + tb.cancel() + test_utils.run_briefly(self.loop) + + self.assertTrue(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + def test_release_not_acquired(self): lock = asyncio.Lock(loop=self.loop) diff --git a/Misc/NEWS b/Misc/NEWS index 580f5793a1e94c..0342c71cd65a13 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Core and Builtins Library ------- +- bpo-27585: Fix waiter cancellation in asyncio.Lock. + Patch by Mathieu Sornay. + - bpo-30418: On Windows, subprocess.Popen.communicate() now also ignore EINVAL on stdin.write() if the child process is still running but closed the pipe. From d24429a20def5e77ee415fdd8f37faf020a96bff Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 9 Jun 2017 14:34:32 -0700 Subject: [PATCH 0384/1345] [3.6] Clarify what --enable-optimizations does (GH-1847) (GH-2039) (cherry picked from commit b4e5fee6f5bcc50500ea6261a22021db58955b55) --- configure | 20 ++++---------------- configure.ac | 4 ++-- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/configure b/configure index 09a94624c1b2b4..10ac675ea8624f 100755 --- a/configure +++ b/configure @@ -784,7 +784,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -896,7 +895,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1149,15 +1147,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1295,7 +1284,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1448,7 +1437,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1489,8 +1477,8 @@ Optional Features: Build (MacOSX|Darwin) framework --enable-shared disable/enable building shared python library --enable-profiling enable C-level code profiling - --enable-optimizations Enable expensive optimizations (PGO, etc). Disabled - by default. + --enable-optimizations Enable expensive, stable optimizations (PGO, etc). + Disabled by default. --enable-loadable-sqlite-extensions support loadable extensions in _sqlite module --enable-ipv6 Enable ipv6 (with ipv4) support @@ -17983,7 +17971,7 @@ mv config.c Modules if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then echo "" >&6 echo "" >&6 - echo "If you want a release build with all optimizations active (LTO, PGO, etc)," >&6 + echo "If you want a release build with all stable optimizations active (PGO, etc)," >&6 echo "please run ./configure --enable-optimizations" >&6 echo "" >&6 echo "" >&6 diff --git a/configure.ac b/configure.ac index 146ae22a632a59..962006704f9cbc 100644 --- a/configure.ac +++ b/configure.ac @@ -1284,7 +1284,7 @@ AC_SUBST(DEF_MAKE_ALL_RULE) AC_SUBST(DEF_MAKE_RULE) Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) -AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), +AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive, stable optimizations (PGO, etc). Disabled by default.]), [ if test "$enableval" != no then @@ -5452,7 +5452,7 @@ mv config.c Modules if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then echo "" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD - echo "If you want a release build with all optimizations active (LTO, PGO, etc)," >&AS_MESSAGE_FD + echo "If you want a release build with all stable optimizations active (PGO, etc)," >&AS_MESSAGE_FD echo "please run ./configure --enable-optimizations" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD From 7a16a4535d691eff79fcc65ae57e69741e6c83df Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 9 Jun 2017 18:33:31 -0400 Subject: [PATCH 0385/1345] Closing transport during handshake process leaks socket (#480) (#2044) --- Lib/asyncio/sslproto.py | 7 +++++-- Lib/test/test_asyncio/test_sslproto.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index ab7ff0bf93d076..6e9ce2968a0826 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -551,8 +551,11 @@ def _get_extra_info(self, name, default=None): def _start_shutdown(self): if self._in_shutdown: return - self._in_shutdown = True - self._write_appdata(b'') + if self._in_handshake: + self._abort() + else: + self._in_shutdown = True + self._write_appdata(b'') def _write_appdata(self, data): self._write_backlog.append((data, 0)) diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f1771c5561afea..bcd236ea2632ed 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -42,6 +42,7 @@ def mock_handshake(callback): sslpipe.do_handshake.side_effect = mock_handshake with mock.patch('asyncio.sslproto._SSLPipe', return_value=sslpipe): ssl_proto.connection_made(transport) + return transport def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an @@ -95,6 +96,20 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_close_during_handshake(self): + # bpo-29743 Closing transport during handshake process leaks socket + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + + def do_handshake(callback): + return [] + + transport = self.connection_made(ssl_proto) + test_utils.run_briefly(self.loop) + + ssl_proto._app_transport.close() + self.assertTrue(transport.abort.called) + def test_get_extra_info_on_closed_connection(self): waiter = asyncio.Future(loop=self.loop) ssl_proto = self.ssl_protocol(waiter) diff --git a/Misc/NEWS b/Misc/NEWS index 0342c71cd65a13..7bd46d5578750e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Core and Builtins Library ------- +- bpo-29743: Closing transport during handshake process leaks open socket. + Patch by Nikolay Kim + - bpo-27585: Fix waiter cancellation in asyncio.Lock. Patch by Mathieu Sornay. From fe9c7a0fd3412efb7598b395e70a5a85219e2e6b Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 9 Jun 2017 19:14:35 -0400 Subject: [PATCH 0386/1345] Break circular references when closing SSLTransport objects (#981) (#2049) --- Lib/asyncio/sslproto.py | 10 ++++++---- Misc/NEWS | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 6e9ce2968a0826..7948c4c3b4efe6 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -686,12 +686,14 @@ def _fatal_error(self, exc, message='Fatal error on transport'): self._transport._force_close(exc) def _finalize(self): + self._sslpipe = None + if self._transport is not None: self._transport.close() def _abort(self): - if self._transport is not None: - try: + try: + if self._transport is not None: self._transport.abort() - finally: - self._finalize() + finally: + self._finalize() diff --git a/Misc/NEWS b/Misc/NEWS index 7bd46d5578750e..213a6e9c2f6442 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Core and Builtins Library ------- +- bpo-29870: Fix ssl sockets leaks when connection is aborted in asyncio/ssl + implementation. Patch by Michaël Sghaïer. + - bpo-29743: Closing transport during handshake process leaks open socket. Patch by Nikolay Kim From fe796efa66ff6d45bb9f3e94790b72b4486d803e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 9 Jun 2017 17:02:04 -0700 Subject: [PATCH 0387/1345] [3.6] Make codecov config on master the only config used (GH-2041) (GH-2052) This will allow for centralized management of the Codecov config to prevent skew as well as easier management going forward. Closes python/core-workflowGH-81. (cherry picked from commit 11ffb4543bc000dea527bcc0417e2f8bda13790f) --- .github/codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/codecov.yml b/.github/codecov.yml index fcf9df6a7a698e..dc21321d0baaf0 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,4 +1,5 @@ codecov: + strict_yaml_branch: master notify: require_ci_to_pass: true comment: off From 10c9a09ef4810f0235f5f585dcce70eca049604a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 9 Jun 2017 18:33:31 -0700 Subject: [PATCH 0388/1345] [3.6] bpo-30335: Add deprecation alias entry for assertNotRegexpMatches (GH-1536) (GH-2055) Document that assertNotRegexpMatches is a deprecated alias for assertNotRegex. (cherry picked from commit 74921ed8941fce14c2a53dc7280f43eb01fe4ed8) --- Doc/library/unittest.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 92e567d12824fa..2099bd1e2e979c 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1170,6 +1170,9 @@ Test cases :meth:`.assertRegex`. .. versionadded:: 3.2 :meth:`.assertNotRegex`. + .. versionadded:: 3.5 + The name ``assertNotRegexpMatches`` is a deprecated alias + for :meth:`.assertNotRegex`. .. method:: assertCountEqual(first, second, msg=None) @@ -1435,9 +1438,9 @@ For historical reasons, some of the :class:`TestCase` methods had one or more aliases that are now deprecated. The following table lists the correct names along with their deprecated aliases: - ============================== ====================== ====================== + ============================== ====================== ======================= Method Name Deprecated alias Deprecated alias - ============================== ====================== ====================== + ============================== ====================== ======================= :meth:`.assertEqual` failUnlessEqual assertEquals :meth:`.assertNotEqual` failIfEqual assertNotEquals :meth:`.assertTrue` failUnless assert\_ @@ -1446,8 +1449,9 @@ along with their deprecated aliases: :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals :meth:`.assertRegex` assertRegexpMatches + :meth:`.assertNotRegex` assertNotRegexpMatches :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ====================== + ============================== ====================== ======================= .. deprecated:: 3.1 the fail* aliases listed in the second column. @@ -1455,8 +1459,9 @@ along with their deprecated aliases: the assert* aliases listed in the third column. .. deprecated:: 3.2 ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex` - + :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. + .. deprecated:: 3.5 + the ``assertNotRegexpMatches`` name in favor of :meth:`.assertNotRegex`. .. _testsuite-objects: From e380c19b7e88d97d3497016f1065d6922415db1c Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 9 Jun 2017 22:26:31 -0500 Subject: [PATCH 0389/1345] [3.6] bpo-30417: Disable `cpu` resource on AppVeyor (GH-1951) (cherry picked from commit 42e3acda86829def9adc354fbee77597b849bf9e) --- .github/appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/appveyor.yml b/.github/appveyor.yml index d8bfb9adf9378d..04566eafe58bfd 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -8,7 +8,7 @@ branches: build_script: - cmd: PCbuild\build.bat -e test_script: -- cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 +- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 # Only trigger AppVeyor if actual code or its configuration changes only_commits: From 753422f6e32e13d96319b090788f0474f1e21fc4 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 9 Jun 2017 20:36:28 -0700 Subject: [PATCH 0390/1345] bpo-30266: support "= None" pattern in AbstractContextManager (GH-1448) (GH-2054) contextlib.AbstractContextManager now supports anti-registration by setting __enter__ = None or __exit__ = None, following the pattern introduced in bpo-25958.. (cherry picked from commit 57161aac5eb9bcb0b43e551a1937ff0a84c1ec52) --- Lib/contextlib.py | 5 ++--- Lib/test/test_contextlib.py | 10 ++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5e47054954ba5a..6fcba9c7d28083 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,6 +1,7 @@ """Utilities for with-statement contexts. See PEP 343.""" import abc import sys +import _collections_abc from collections import deque from functools import wraps @@ -25,9 +26,7 @@ def __exit__(self, exc_type, exc_value, traceback): @classmethod def __subclasshook__(cls, C): if cls is AbstractContextManager: - if (any("__enter__" in B.__dict__ for B in C.__mro__) and - any("__exit__" in B.__dict__ for B in C.__mro__)): - return True + return _collections_abc._check_methods(C, "__enter__", "__exit__") return NotImplemented diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index b1a467d952da0f..2301f759d80858 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -44,6 +44,16 @@ def __exit__(self, *args): self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + class NoEnter(ManagerFromScratch): + __enter__ = None + + self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + + class NoExit(ManagerFromScratch): + __exit__ = None + + self.assertFalse(issubclass(NoExit, AbstractContextManager)) + class ContextManagerTestCase(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index 213a6e9c2f6442..e6517695a62c20 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -96,6 +96,10 @@ Library - bpo-30048: Fixed ``Task.cancel()`` can be ignored when the task is running coroutine and the coroutine returned without any more ``await``. +- bpo-30266: contextlib.AbstractContextManager now supports anti-registration + by setting __enter__ = None or __exit__ = None, following the pattern + introduced in bpo-25958. Patch by Jelle Zijlstra. + - bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. Now allowed several subsequential inline modifiers at the start of the pattern (e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments From 361362f3a0d6ebb458a8d826a698463085c5a8f7 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 10 Jun 2017 00:15:28 -0400 Subject: [PATCH 0391/1345] [3.6] Fix TypeError is asyncio/proactor_events (GH-993) (#2061) (cherry picked from commit 34792d25ab7329241ea24595286d65d54c266274) --- Lib/asyncio/proactor_events.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index ff12877fae2f23..a81645d9c5d859 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -232,8 +232,9 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) + msg = ("data argument must be a bytes-like object, not '%s'" % + type(data).__name__) + raise TypeError(msg) if self._eof_written: raise RuntimeError('write_eof() already called') From 12cbd87ac0bb826d653040044c6b526dcdb6f6d1 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sat, 10 Jun 2017 02:53:19 -0400 Subject: [PATCH 0392/1345] [3.6] bpo-30290: IDLE - pep8 names and tests for help-about (#2070) (cherry picked from commit 054e09147aaa6f61aca6cd40c7bf7ce6dc54a04b) * bpo-30290: IDLE: Refactor help_about to PEP8 names (#1714) Patch by Cheryl Sabella. (cherry picked from commit 5a346d5dbc1f0f70eca706a8ba19f7645bf17837) * IDLE test_help_about: edit and add test. (#1838) Coverage is now 100% (cherry picked from commit eca7da0f13c78013b924fe7306f3e2e59c0af40b) --- Lib/idlelib/help_about.py | 210 +++++++++++++---------- Lib/idlelib/idle_test/test_help_about.py | 111 ++++++++++-- 2 files changed, 222 insertions(+), 99 deletions(-) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 071bd3ec0f219c..f0a40e927c3634 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -4,7 +4,8 @@ import os from sys import version -from tkinter import * +from tkinter import Toplevel, Frame, Label, Button +from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW from idlelib import textview @@ -13,9 +14,13 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title, _htest=False): - """ + def __init__(self, parent, title, _htest=False, _utest=False): + """Create popup, do not return until tk widget destroyed. + + parent - parent of this dialog + title - string which is title of popup dialog _htest - bool, change box location when running htest + _utest - bool, don't wait_window when running unittest """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -25,125 +30,152 @@ def __init__(self, parent, title, _htest=False): parent.winfo_rooty()+(30 if not _htest else 100))) self.bg = "#707070" self.fg = "#ffffff" - self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) + self.create_widgets() + self.resizable(height=False, width=False) self.title(title) self.transient(parent) self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Ok) + self.protocol("WM_DELETE_WINDOW", self.ok) self.parent = parent - self.buttonOk.focus_set() - self.bind('',self.Ok) #dismiss dialog - self.bind('',self.Ok) #dismiss dialog - self.wait_window() + self.button_ok.focus_set() + self.bind('', self.ok) # dismiss dialog + self.bind('', self.ok) # dismiss dialog + self._current_textview = None + self._utest = _utest - def CreateWidgets(self): + if not _utest: + self.deiconify() + self.wait_window() + + def create_widgets(self): release = version[:version.index(' ')] - frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - frameButtons = Frame(self) - frameButtons.pack(side=BOTTOM, fill=X) - frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok) - self.buttonOk.pack(padx=5, pady=5) - #self.picture = Image('photo', data=self.pictureData) - frameBg = Frame(frameMain, bg=self.bg) - frameBg.pack(expand=TRUE, fill=BOTH) - labelTitle = Label(frameBg, text='IDLE', fg=self.fg, bg=self.bg, - font=('courier', 24, 'bold')) - labelTitle.grid(row=0, column=0, sticky=W, padx=10, pady=10) - #labelPicture = Label(frameBg, text='[picture]') - #image=self.picture, bg=self.bg) - #labelPicture.grid(row=1, column=1, sticky=W, rowspan=2, - # padx=0, pady=3) - byline = "Python's Integrated DeveLopment Environment" + 5*'\n' - labelDesc = Label(frameBg, text=byline, justify=LEFT, - fg=self.fg, bg=self.bg) - labelDesc.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - labelEmail = Label(frameBg, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - labelEmail.grid(row=6, column=0, columnspan=2, - sticky=W, padx=10, pady=0) - labelWWW = Label(frameBg, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', - justify=LEFT, fg=self.fg, bg=self.bg) - labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + frame = Frame(self, borderwidth=2, relief=SUNKEN) + frame_buttons = Frame(self) + frame_buttons.pack(side=BOTTOM, fill=X) + frame.pack(side=TOP, expand=True, fill=BOTH) + self.button_ok = Button(frame_buttons, text='Close', + command=self.ok) + self.button_ok.pack(padx=5, pady=5) + + frame_background = Frame(frame, bg=self.bg) + frame_background.pack(expand=True, fill=BOTH) + + header = Label(frame_background, text='IDLE', fg=self.fg, + bg=self.bg, font=('courier', 24, 'bold')) + header.grid(row=0, column=0, sticky=W, padx=10, pady=10) + byline_text = "Python's Integrated DeveLopment Environment" + 5*'\n' + byline = Label(frame_background, text=byline_text, justify=LEFT, + fg=self.fg, bg=self.bg) + byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) + email = Label(frame_background, text='email: idle-dev@python.org', + justify=LEFT, fg=self.fg, bg=self.bg) + email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) + docs = Label(frame_background, text='https://docs.python.org/' + + version[:3] + '/library/idle.html', + justify=LEFT, fg=self.fg, bg=self.bg) + docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - labelPythonVer = Label(frameBg, text='Python version: ' + - release, fg=self.fg, bg=self.bg) - labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkVer = self.tk.call('info', 'patchlevel') - labelTkVer = Label(frameBg, text='Tk version: '+ - tkVer, fg=self.fg, bg=self.bg) - labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) - py_button_f = Frame(frameBg, bg=self.bg) - py_button_f.grid(row=10, column=0, columnspan=2, sticky=NSEW) - buttonLicense = Button(py_button_f, text='License', width=8, - highlightbackground=self.bg, - command=self.ShowLicense) - buttonLicense.pack(side=LEFT, padx=10, pady=10) - buttonCopyright = Button(py_button_f, text='Copyright', width=8, + + pyver = Label(frame_background, text='Python version: ' + release, + fg=self.fg, bg=self.bg) + pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) + tk_patchlevel = self.tk.call('info', 'patchlevel') + tkver = Label(frame_background, text='Tk version: ' + tk_patchlevel, + fg=self.fg, bg=self.bg) + tkver.grid(row=9, column=1, sticky=W, padx=2, pady=0) + py_buttons = Frame(frame_background, bg=self.bg) + py_buttons.grid(row=10, column=0, columnspan=2, sticky=NSEW) + self.py_license = Button(py_buttons, text='License', width=8, + highlightbackground=self.bg, + command=self.show_py_license) + self.py_license.pack(side=LEFT, padx=10, pady=10) + self.py_copyright = Button(py_buttons, text='Copyright', width=8, + highlightbackground=self.bg, + command=self.show_py_copyright) + self.py_copyright.pack(side=LEFT, padx=10, pady=10) + self.py_credits = Button(py_buttons, text='Credits', width=8, highlightbackground=self.bg, - command=self.ShowCopyright) - buttonCopyright.pack(side=LEFT, padx=10, pady=10) - buttonCredits = Button(py_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowPythonCredits) - buttonCredits.pack(side=LEFT, padx=10, pady=10) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + command=self.show_py_credits) + self.py_credits.pack(side=LEFT, padx=10, pady=10) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle_v = Label(frameBg, text='IDLE version: ' + release, - fg=self.fg, bg=self.bg) - idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) - idle_button_f = Frame(frameBg, bg=self.bg) - idle_button_f.grid(row=13, column=0, columnspan=3, sticky=NSEW) - idle_about_b = Button(idle_button_f, text='README', width=8, - highlightbackground=self.bg, - command=self.ShowIDLEAbout) - idle_about_b.pack(side=LEFT, padx=10, pady=10) - idle_news_b = Button(idle_button_f, text='NEWS', width=8, - highlightbackground=self.bg, - command=self.ShowIDLENEWS) - idle_news_b.pack(side=LEFT, padx=10, pady=10) - idle_credits_b = Button(idle_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowIDLECredits) - idle_credits_b.pack(side=LEFT, padx=10, pady=10) - # License, et all, are of type _sitebuiltins._Printer - def ShowLicense(self): + idlever = Label(frame_background, text='IDLE version: ' + release, + fg=self.fg, bg=self.bg) + idlever.grid(row=12, column=0, sticky=W, padx=10, pady=0) + idle_buttons = Frame(frame_background, bg=self.bg) + idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) + self.readme = Button(idle_buttons, text='README', width=8, + highlightbackground=self.bg, + command=self.show_readme) + self.readme.pack(side=LEFT, padx=10, pady=10) + self.idle_news = Button(idle_buttons, text='NEWS', width=8, + highlightbackground=self.bg, + command=self.show_idle_news) + self.idle_news.pack(side=LEFT, padx=10, pady=10) + self.idle_credits = Button(idle_buttons, text='Credits', width=8, + highlightbackground=self.bg, + command=self.show_idle_credits) + self.idle_credits.pack(side=LEFT, padx=10, pady=10) + + # License, copyright, and credits are of type _sitebuiltins._Printer + def show_py_license(self): + "Handle License button event." self.display_printer_text('About - License', license) - def ShowCopyright(self): + def show_py_copyright(self): + "Handle Copyright button event." self.display_printer_text('About - Copyright', copyright) - def ShowPythonCredits(self): + def show_py_credits(self): + "Handle Python Credits button event." self.display_printer_text('About - Python Credits', credits) # Encode CREDITS.txt to utf-8 for proper version of Loewis. # Specify others as ascii until need utf-8, so catch errors. - def ShowIDLECredits(self): + def show_idle_credits(self): + "Handle Idle Credits button event." self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') - def ShowIDLEAbout(self): + def show_readme(self): + "Handle Readme button event." self.display_file_text('About - Readme', 'README.txt', 'ascii') - def ShowIDLENEWS(self): + def show_idle_news(self): + "Handle News button event." self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') def display_printer_text(self, title, printer): + """Create textview for built-in constants. + + Built-in constants have type _sitebuiltins._Printer. The + text is extracted from the built-in and then sent to a text + viewer with self as the parent and title as the title of + the popup. + """ printer._Printer__setup() text = '\n'.join(printer._Printer__lines) - textview.view_text(self, title, text) + self._current_textview = textview.view_text( + self, title, text, _utest=self._utest) def display_file_text(self, title, filename, encoding=None): + """Create textview for filename. + + The filename needs to be in the current directory. The path + is sent to a text viewer with self as the parent, title as + the title of the popup, and the file encoding. + """ fn = os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) - textview.view_file(self, title, fn, encoding) + self._current_textview = textview.view_file( + self, title, fn, encoding, _utest=self._utest) - def Ok(self, event=None): + def ok(self, event=None): + "Dismiss help_about dialog." self.destroy() diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 843efb9ad2458c..b98405df52a0c7 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -1,24 +1,116 @@ '''Test idlelib.help_about. -Coverage: +Coverage: 100% ''' -from idlelib import help_about -from idlelib import textview +from test.support import requires, findfile +from tkinter import Tk, TclError +import unittest from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func -import unittest +from idlelib.help_about import AboutDialog as About +from idlelib import textview + +class LiveDialogTest(unittest.TestCase): + """Simulate user clicking buttons other than [Close]. + + Test that invoked textview has text from source. + """ + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_dialog_title(self): + """Test about dialog title""" + self.assertEqual(self.dialog.title(), 'About IDLE') + + def test_printer_buttons(self): + """Test buttons whose commands use printer function.""" + dialog = self.dialog + button_sources = [(self.dialog.py_license, license), + (self.dialog.py_copyright, copyright), + (self.dialog.py_credits, credits)] + + for button, printer in button_sources: + printer._Printer__setup() + button.invoke() + self.assertEqual( + printer._Printer__lines[0], + dialog._current_textview.textView.get('1.0', '1.end')) + self.assertEqual( + printer._Printer__lines[1], + dialog._current_textview.textView.get('2.0', '2.end')) + dialog._current_textview.destroy() + + def test_file_buttons(self): + """Test buttons that display files.""" + dialog = self.dialog + button_sources = [(self.dialog.readme, 'README.txt'), + (self.dialog.idle_news, 'NEWS.txt'), + (self.dialog.idle_credits, 'CREDITS.txt')] + + for button, filename in button_sources: + button.invoke() + fn = findfile(filename, subdir='idlelib') + with open(fn) as f: + self.assertEqual( + f.readline().strip(), + dialog._current_textview.textView.get('1.0', '1.end')) + f.readline() + self.assertEqual(f.readline().strip(), + dialog._current_textview.textView.get('3.0', '3.end')) + dialog._current_textview.destroy() + + +class CloseTest(unittest.TestCase): + """Simulate user clicking [Close] button""" + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_close(self): + self.assertEqual(self.dialog.winfo_class(), 'Toplevel') + self.dialog.button_ok.invoke() + with self.assertRaises(TclError): + self.dialog.winfo_class() + -About = help_about.AboutDialog class Dummy_about_dialog(): # Dummy class for testing file display functions. - idle_credits = About.ShowIDLECredits - idle_readme = About.ShowIDLEAbout - idle_news = About.ShowIDLENEWS + idle_credits = About.show_idle_credits + idle_readme = About.show_readme + idle_news = About.show_idle_news # Called by the above display_file_text = About.display_file_text + _utest = True class DisplayFileTest(unittest.TestCase): + """Test functions that display files. + + While somewhat redundant with gui-based test_file_dialog, + these unit tests run on all buildbots, not just a few. + """ dialog = Dummy_about_dialog() @classmethod @@ -29,14 +121,13 @@ def setUpClass(cls): cls.view = Func() textview.showerror = cls.error textview.view_text = cls.view - cls.About = Dummy_about_dialog() @classmethod def tearDownClass(cls): textview.showerror = cls.orig_error textview.view_text = cls.orig_view - def test_file_isplay(self): + def test_file_display(self): for handler in (self.dialog.idle_credits, self.dialog.idle_readme, self.dialog.idle_news): From 0b7629cd846e3e160ac07cafc4e97644024f11b8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 10 Jun 2017 11:20:03 +0200 Subject: [PATCH 0393/1345] bpo-30038: fix race condition in signal delivery + wakeup fd (#1082) (#2075) Before, it was possible to get the following sequence of events (especially on Windows, where the C-level signal handler for SIGINT is run in a separate thread): - SIGINT arrives - trip_signal is called - trip_signal writes to the wakeup fd - the main thread wakes up from select()-or-equivalent - the main thread checks for pending signals, but doesn't see any - the main thread drains the wakeup fd - the main thread goes back to sleep - trip_signal sets is_tripped=1 and calls Py_AddPendingCall to notify the main thread the it should run the Python-level signal handler - the main thread doesn't notice because it's asleep This has been causing repeated failures in the Trio test suite: https://github.com/python-trio/trio/issues/119 (cherry picked from commit 4ae01496971624c75080431806ed1c08e00f22c7) --- Modules/signalmodule.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index e27075b2001946..ed2f4e8cf0a2dc 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -244,6 +244,32 @@ trip_signal(int sig_num) Handlers[sig_num].tripped = 1; + if (!is_tripped) { + /* Set is_tripped after setting .tripped, as it gets + cleared in PyErr_CheckSignals() before .tripped. */ + is_tripped = 1; + Py_AddPendingCall(checksignals_witharg, NULL); + } + + /* And then write to the wakeup fd *after* setting all the globals and + doing the Py_AddPendingCall. We used to write to the wakeup fd and then + set the flag, but this allowed the following sequence of events + (especially on windows, where trip_signal runs in a new thread): + + - main thread blocks on select([wakeup_fd], ...) + - signal arrives + - trip_signal writes to the wakeup fd + - the main thread wakes up + - the main thread checks the signal flags, sees that they're unset + - the main thread empties the wakeup fd + - the main thread goes back to sleep + - trip_signal sets the flags to request the Python-level signal handler + be run + - the main thread doesn't notice, because it's asleep + + See bpo-30038 for more details. + */ + #ifdef MS_WINDOWS fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); #else @@ -281,13 +307,6 @@ trip_signal(int sig_num) } } } - - if (!is_tripped) { - /* Set is_tripped after setting .tripped, as it gets - cleared in PyErr_CheckSignals() before .tripped. */ - is_tripped = 1; - Py_AddPendingCall(checksignals_witharg, NULL); - } } static void From 911068e250ee2b0847ca78d9ff527e0d6bf94551 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 10 Jun 2017 13:48:53 +0300 Subject: [PATCH 0394/1345] [3.6] Regenerate Argument Clinic code for bpo-19180. (GH-2073). (#2077) (cherry picked from commit 5f31d5cf6efa8c304d352e34f9f2a1ed0074298e) --- Modules/_ssl.c | 2 +- Modules/clinic/_ssl.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index a421fc0d2d6ff2..0fffaaceb26419 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4480,7 +4480,7 @@ string. See RFC 4086. static PyObject * _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) -/*[clinic end generated code: output=e6dd48df9c9024e9 input=580c85e6a3a4fe29]*/ +/*[clinic end generated code: output=e6dd48df9c9024e9 input=5c33017422828f5c]*/ { const char *buf; Py_ssize_t len, written; diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 333aad76b404fc..75f8f5a60bb088 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -1168,4 +1168,4 @@ _ssl_enum_crls(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kw #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=a859b21fe68a6115 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=56cead8610faa505 input=a9049054013a1b77]*/ From 6e14fd2a14cef6ea0709ad234ab41198c2195591 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 10 Jun 2017 10:00:45 -0400 Subject: [PATCH 0395/1345] [3.6] bpo-29406: asyncio SSL contexts leak sockets after calling close with certain servers (GH-409) (#2062) * bpo-29406: asyncio SSL contexts leak sockets after calling close with certain servers (#409) (cherry picked from commit a608d2d5a7f1aabe9bcbfc220135c5e126189390) * [3.6] bpo-29406: asyncio SSL contexts leak sockets after calling close with certain servers (GH-409) * asyncio SSL contexts leak sockets after calling close with certain servers * cleanup _shutdown_timeout_handle on _fatal_error. (cherry picked from commit a608d2d5a7f1aabe9bcbfc220135c5e126189390) --- Lib/asyncio/sslproto.py | 24 +++++++++++++++++- Lib/test/test_asyncio/test_sslproto.py | 34 ++++++++++++++++++++++++++ Misc/NEWS | 4 +++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7948c4c3b4efe6..3b1eb993dfc7a8 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -7,6 +7,7 @@ from . import base_events from . import compat +from . import futures from . import protocols from . import transports from .log import logger @@ -412,7 +413,7 @@ class SSLProtocol(protocols.Protocol): def __init__(self, loop, app_protocol, sslcontext, waiter, server_side=False, server_hostname=None, - call_connection_made=True): + call_connection_made=True, shutdown_timeout=5.0): if ssl is None: raise RuntimeError('stdlib ssl module not available') @@ -443,6 +444,8 @@ def __init__(self, loop, app_protocol, sslcontext, waiter, self._session_established = False self._in_handshake = False self._in_shutdown = False + self._shutdown_timeout = shutdown_timeout + self._shutdown_timeout_handle = None # transport, ex: SelectorSocketTransport self._transport = None self._call_connection_made = call_connection_made @@ -557,6 +560,15 @@ def _start_shutdown(self): self._in_shutdown = True self._write_appdata(b'') + if self._shutdown_timeout is not None: + self._shutdown_timeout_handle = self._loop.call_later( + self._shutdown_timeout, self._on_shutdown_timeout) + + def _on_shutdown_timeout(self): + if self._transport is not None: + self._fatal_error( + futures.TimeoutError(), 'Can not complete shitdown operation') + def _write_appdata(self, data): self._write_backlog.append((data, 0)) self._write_buffer_size += len(data) @@ -684,12 +696,22 @@ def _fatal_error(self, exc, message='Fatal error on transport'): }) if self._transport: self._transport._force_close(exc) + self._transport = None + + if self._shutdown_timeout_handle is not None: + self._shutdown_timeout_handle.cancel() + self._shutdown_timeout_handle = None def _finalize(self): self._sslpipe = None if self._transport is not None: self._transport.close() + self._transport = None + + if self._shutdown_timeout_handle is not None: + self._shutdown_timeout_handle.cancel() + self._shutdown_timeout_handle = None def _abort(self): try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index bcd236ea2632ed..6bcaa9edb4306d 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -96,6 +96,40 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_close_abort(self): + # From issue #bpo-29406 + # abort connection if server does not complete shutdown procedure + ssl_proto = self.ssl_protocol() + transport = self.connection_made(ssl_proto) + ssl_proto._on_handshake_complete(None) + ssl_proto._start_shutdown() + self.assertIsNotNone(ssl_proto._shutdown_timeout_handle) + + exc_handler = mock.Mock() + self.loop.set_exception_handler(exc_handler) + ssl_proto._shutdown_timeout_handle._run() + + exc_handler.assert_called_with( + self.loop, {'message': 'Can not complete shitdown operation', + 'exception': mock.ANY, + 'transport': transport, + 'protocol': ssl_proto} + ) + self.assertIsNone(ssl_proto._shutdown_timeout_handle) + + def test_close(self): + # From issue #bpo-29406 + # abort connection if server does not complete shutdown procedure + ssl_proto = self.ssl_protocol() + transport = self.connection_made(ssl_proto) + ssl_proto._on_handshake_complete(None) + ssl_proto._start_shutdown() + self.assertIsNotNone(ssl_proto._shutdown_timeout_handle) + + ssl_proto._finalize() + self.assertIsNone(ssl_proto._transport) + self.assertIsNone(ssl_proto._shutdown_timeout_handle) + def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) diff --git a/Misc/NEWS b/Misc/NEWS index e6517695a62c20..9d4f5d432a1ec4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,10 @@ Core and Builtins Library ------- +- bpo-29406: asyncio SSL contexts leak sockets after calling close with + certain servers. + Patch by Nikolay Kim + - bpo-29870: Fix ssl sockets leaks when connection is aborted in asyncio/ssl implementation. Patch by Michaël Sghaïer. From 76eabd3a21dc578de6093d45dd8d69a5ec4b9afe Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 10 Jun 2017 07:25:33 -0700 Subject: [PATCH 0396/1345] bpo-25409: Clarify fnmatch and fnmatchcase documentation (GH-1535) (GH-2065) Mention that fnmatchcase does not call normcase, and fnmatch does. (cherry picked from commit e5f6e86c48c7b2eb9e1d6a0e72867b4d8b4720f3) --- Doc/library/fnmatch.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index c03a9d31123739..634c26e95702e9 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -43,9 +43,8 @@ patterns. .. function:: fnmatch(filename, pattern) Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. If the operating system is case-insensitive, - then both parameters will be normalized to all lower- or upper-case before - the comparison is performed. :func:`fnmatchcase` can be used to perform a + :const:`True` or :const:`False`. Both parameters are case-normalized + using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a case-sensitive comparison, regardless of whether that's standard for the operating system. @@ -63,7 +62,8 @@ patterns. .. function:: fnmatchcase(filename, pattern) Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive. + :const:`False`; the comparison is case-sensitive and does not apply + :func:`os.path.normcase`. .. function:: filter(names, pattern) From 865ed9ea67cf0d8a8dead91f3eac527553d92284 Mon Sep 17 00:00:00 2001 From: KatherineMichel Date: Sat, 10 Jun 2017 15:33:57 -0500 Subject: [PATCH 0397/1345] bpo-30312: Small correction in datastructures set code sample (GH-2081) (GH-2085) (cherry picked from commit ca816153445cba3baec15f7e890c71abfe495340) --- Doc/tutorial/datastructures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 1a73ac9d05936a..f9ddf06c5e0e9e 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -466,7 +466,7 @@ Here is a brief demonstration:: {'a', 'r', 'b', 'c', 'd'} >>> a - b # letters in a but not in b {'r', 'd', 'b'} - >>> a | b # letters in either a or b + >>> a | b # letters in a or b or both {'a', 'c', 'r', 'd', 'b', 'm', 'z', 'l'} >>> a & b # letters in both a and b {'a', 'c'} From 964c261dc9a6a901f50d5596d88248bfc4251a55 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 10 Jun 2017 15:39:29 -0500 Subject: [PATCH 0398/1345] [3.6] bpo-27425: Be more explicit in .gitattributes (GH-840) (GH-2083) Also updates checked-in line endings on some files --- .gitattributes | 45 +- Lib/venv/scripts/nt/Activate.ps1 | 102 +- Lib/venv/scripts/nt/activate.bat | 64 +- Lib/venv/scripts/nt/deactivate.bat | 42 +- Modules/_decimal/tests/runall.bat | 222 ++-- PC/bdist_wininst/build.bat | 44 +- PCbuild/build.bat | 312 +++--- PCbuild/build_env.bat | 2 +- PCbuild/clean.bat | 10 +- PCbuild/env.bat | 32 +- PCbuild/get_externals.bat | 208 ++-- PCbuild/idle.bat | 30 +- PCbuild/pcbuild.sln | 1538 ++++++++++++++-------------- PCbuild/prepare_ssl.bat | 24 +- PCbuild/readme.txt | 600 +++++------ PCbuild/rt.bat | 126 +-- Tools/buildbot/build.bat | 34 +- Tools/buildbot/buildmsi.bat | 16 +- Tools/buildbot/clean.bat | 34 +- Tools/buildbot/test.bat | 38 +- Tools/msi/build.bat | 158 +-- Tools/msi/buildrelease.bat | 470 ++++----- Tools/msi/get_externals.bat | 54 +- Tools/msi/testrelease.bat | 234 ++--- Tools/msi/uploadrelease.bat | 152 +-- Tools/nuget/build.bat | 110 +- Tools/unicode/genwincodecs.bat | 14 +- 27 files changed, 2374 insertions(+), 2341 deletions(-) diff --git a/.gitattributes b/.gitattributes index 82694d81f276b2..5eead66489830d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,8 +1,41 @@ +# Binary data types +*.aif binary +*.aifc binary +*.aiff binary +*.au binary +*.bmp binary +*.exe binary +*.icns binary +*.gif binary +*.ico binary +*.jpg binary *.pck binary -Lib/test/cjkencodings/* binary -Lib/test/decimaltestdata/*.decTest binary +*.png binary +*.psd binary +*.tar binary +*.wav binary +*.whl binary +*.zip binary + +# Specific binary files Lib/test/sndhdrdata/sndhdr.* binary -Lib/test/test_email/data/msg_26.txt binary -Lib/test/xmltestdata/* binary -Lib/venv/scripts/nt/* binary -Lib/test/coding20731.py binary + +# Text files that should not be subject to eol conversion +Lib/test/cjkencodings/* -text +Lib/test/decimaltestdata/*.decTest -text +Lib/test/test_email/data/*.txt -text +Lib/test/xmltestdata/* -text +Lib/test/coding20731.py -text + +# Special files in third party code +Modules/zlib/zlib.map -text + +# CRLF files +*.bat text eol=crlf +*.ps1 text eol=crlf +*.sln text eol=crlf +*.vcxproj* text eol=crlf +*.props text eol=crlf +*.proj text eol=crlf +PCbuild/readme.txt text eol=crlf +PC/readme.txt text eol=crlf diff --git a/Lib/venv/scripts/nt/Activate.ps1 b/Lib/venv/scripts/nt/Activate.ps1 index 85646c89a81cb3..bf60869e552e97 100644 --- a/Lib/venv/scripts/nt/Activate.ps1 +++ b/Lib/venv/scripts/nt/Activate.ps1 @@ -1,51 +1,51 @@ -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - if (Test-Path function:_OLD_VIRTUAL_PROMPT) { - copy-item function:_OLD_VIRTUAL_PROMPT function:prompt - remove-item function:_OLD_VIRTUAL_PROMPT - } - - if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) { - copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME - remove-item env:_OLD_VIRTUAL_PYTHONHOME - } - - if (Test-Path env:_OLD_VIRTUAL_PATH) { - copy-item env:_OLD_VIRTUAL_PATH env:PATH - remove-item env:_OLD_VIRTUAL_PATH - } - - if (Test-Path env:VIRTUAL_ENV) { - remove-item env:VIRTUAL_ENV - } - - if (!$NonDestructive) { - # Self destruct! - remove-item function:deactivate - } -} - -deactivate -nondestructive - -$env:VIRTUAL_ENV="__VENV_DIR__" - -if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) { - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT {""} - copy-item function:prompt function:_OLD_VIRTUAL_PROMPT - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green '__VENV_PROMPT__' - _OLD_VIRTUAL_PROMPT - } -} - -# Clear PYTHONHOME -if (Test-Path env:PYTHONHOME) { - copy-item env:PYTHONHOME env:_OLD_VIRTUAL_PYTHONHOME - remove-item env:PYTHONHOME -} - -# Add the venv to the PATH -copy-item env:PATH env:_OLD_VIRTUAL_PATH -$env:PATH = "$env:VIRTUAL_ENV\__VENV_BIN_NAME__;$env:PATH" +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + if (Test-Path function:_OLD_VIRTUAL_PROMPT) { + copy-item function:_OLD_VIRTUAL_PROMPT function:prompt + remove-item function:_OLD_VIRTUAL_PROMPT + } + + if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) { + copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME + remove-item env:_OLD_VIRTUAL_PYTHONHOME + } + + if (Test-Path env:_OLD_VIRTUAL_PATH) { + copy-item env:_OLD_VIRTUAL_PATH env:PATH + remove-item env:_OLD_VIRTUAL_PATH + } + + if (Test-Path env:VIRTUAL_ENV) { + remove-item env:VIRTUAL_ENV + } + + if (!$NonDestructive) { + # Self destruct! + remove-item function:deactivate + } +} + +deactivate -nondestructive + +$env:VIRTUAL_ENV="__VENV_DIR__" + +if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) { + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT {""} + copy-item function:prompt function:_OLD_VIRTUAL_PROMPT + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green '__VENV_PROMPT__' + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path env:PYTHONHOME) { + copy-item env:PYTHONHOME env:_OLD_VIRTUAL_PYTHONHOME + remove-item env:PYTHONHOME +} + +# Add the venv to the PATH +copy-item env:PATH env:_OLD_VIRTUAL_PATH +$env:PATH = "$env:VIRTUAL_ENV\__VENV_BIN_NAME__;$env:PATH" diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 9eab147f314d3b..d76ca1359627ee 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -1,32 +1,32 @@ -@echo off -set "VIRTUAL_ENV=__VENV_DIR__" - -if not defined PROMPT ( - set "PROMPT=$P$G" -) - -if defined _OLD_VIRTUAL_PROMPT ( - set "PROMPT=%_OLD_VIRTUAL_PROMPT%" -) - -if defined _OLD_VIRTUAL_PYTHONHOME ( - set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" -) - -set "_OLD_VIRTUAL_PROMPT=%PROMPT%" -set "PROMPT=__VENV_PROMPT__%PROMPT%" - -if defined PYTHONHOME ( - set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%" - set PYTHONHOME= -) - -if defined _OLD_VIRTUAL_PATH ( - set "PATH=%_OLD_VIRTUAL_PATH%" -) else ( - set "_OLD_VIRTUAL_PATH=%PATH%" -) - -set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" - -:END +@echo off +set "VIRTUAL_ENV=__VENV_DIR__" + +if not defined PROMPT ( + set "PROMPT=$P$G" +) + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" +) + +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=__VENV_PROMPT__%PROMPT%" + +if defined PYTHONHOME ( + set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%" + set PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) else ( + set "_OLD_VIRTUAL_PATH=%PATH%" +) + +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" + +:END diff --git a/Lib/venv/scripts/nt/deactivate.bat b/Lib/venv/scripts/nt/deactivate.bat index 313c0791173682..1205c618686fbb 100644 --- a/Lib/venv/scripts/nt/deactivate.bat +++ b/Lib/venv/scripts/nt/deactivate.bat @@ -1,21 +1,21 @@ -@echo off - -if defined _OLD_VIRTUAL_PROMPT ( - set "PROMPT=%_OLD_VIRTUAL_PROMPT%" -) -set _OLD_VIRTUAL_PROMPT= - -if defined _OLD_VIRTUAL_PYTHONHOME ( - set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" - set _OLD_VIRTUAL_PYTHONHOME= -) - -if defined _OLD_VIRTUAL_PATH ( - set "PATH=%_OLD_VIRTUAL_PATH%" -) - -set _OLD_VIRTUAL_PATH= - -set VIRTUAL_ENV= - -:END +@echo off + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) +set _OLD_VIRTUAL_PROMPT= + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" + set _OLD_VIRTUAL_PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) + +set _OLD_VIRTUAL_PATH= + +set VIRTUAL_ENV= + +:END diff --git a/Modules/_decimal/tests/runall.bat b/Modules/_decimal/tests/runall.bat index 568f92f6ddf392..5bc872a63f83aa 100755 --- a/Modules/_decimal/tests/runall.bat +++ b/Modules/_decimal/tests/runall.bat @@ -1,111 +1,111 @@ -@ECHO OFF - -rem Test all machine configurations, pydebug, refleaks, release build. - -cd ..\..\..\ - - -echo. -echo # ====================================================================== -echo # Building Python -echo # ====================================================================== -echo. - -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x64 -msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 -msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=x64 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 - -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 -echo. -echo. - -echo. -echo # ====================================================================== -echo # test_decimal: platform=x64 -echo # ====================================================================== -echo. - -cd PCbuild\amd64 - -echo # ==================== refleak tests ======================= -echo. -python_d.exe -m test -uall -R 2:2 test_decimal -echo. -echo. - -echo # ==================== regular tests ======================= -echo. -python.exe -m test -uall test_decimal -echo. -echo. - -cd .. - -echo. -echo # ====================================================================== -echo # test_decimal: platform=x86 -echo # ====================================================================== -echo. - -echo # ==================== refleak tests ======================= -echo. -python_d.exe -m test -uall -R 2:2 test_decimal -echo. -echo. - -echo # ==================== regular tests ======================= -echo. -python.exe -m test -uall test_decimal -echo. -echo. - -cd amd64 - -echo. -echo # ====================================================================== -echo # deccheck: platform=x64 -echo # ====================================================================== -echo. - -echo # ==================== debug build ======================= -echo. -python_d.exe ..\..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -echo # =================== release build ====================== -echo. -python.exe ..\..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -cd .. - -echo. -echo # ====================================================================== -echo # deccheck: platform=x86 -echo # ====================================================================== -echo. -echo. - -echo # ==================== debug build ======================= -echo. -python_d.exe ..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -echo # =================== release build ====================== -echo. -python.exe ..\Modules\_decimal\tests\deccheck.py -echo. -echo. - - -cd ..\Modules\_decimal\tests - - - +@ECHO OFF + +rem Test all machine configurations, pydebug, refleaks, release build. + +cd ..\..\..\ + + +echo. +echo # ====================================================================== +echo # Building Python +echo # ====================================================================== +echo. + +call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x64 +msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 +msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=x64 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 + +call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 +echo. +echo. + +echo. +echo # ====================================================================== +echo # test_decimal: platform=x64 +echo # ====================================================================== +echo. + +cd PCbuild\amd64 + +echo # ==================== refleak tests ======================= +echo. +python_d.exe -m test -uall -R 2:2 test_decimal +echo. +echo. + +echo # ==================== regular tests ======================= +echo. +python.exe -m test -uall test_decimal +echo. +echo. + +cd .. + +echo. +echo # ====================================================================== +echo # test_decimal: platform=x86 +echo # ====================================================================== +echo. + +echo # ==================== refleak tests ======================= +echo. +python_d.exe -m test -uall -R 2:2 test_decimal +echo. +echo. + +echo # ==================== regular tests ======================= +echo. +python.exe -m test -uall test_decimal +echo. +echo. + +cd amd64 + +echo. +echo # ====================================================================== +echo # deccheck: platform=x64 +echo # ====================================================================== +echo. + +echo # ==================== debug build ======================= +echo. +python_d.exe ..\..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +echo # =================== release build ====================== +echo. +python.exe ..\..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +cd .. + +echo. +echo # ====================================================================== +echo # deccheck: platform=x86 +echo # ====================================================================== +echo. +echo. + +echo # ==================== debug build ======================= +echo. +python_d.exe ..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +echo # =================== release build ====================== +echo. +python.exe ..\Modules\_decimal\tests\deccheck.py +echo. +echo. + + +cd ..\Modules\_decimal\tests + + + diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index 25f565ce0a16d8..ee6856754faa8c 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -1,22 +1,22 @@ -@echo off -setlocal - -set D=%~dp0 -set PCBUILD=%~dp0..\..\PCBuild\ - - -echo Building Lib\distutils\command\wininst-xx.0.exe - -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 -if errorlevel 1 goto :eof - - -echo Building Lib\distutils\command\wininst-xx.0-amd64.exe - -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +@echo off +setlocal + +set D=%~dp0 +set PCBUILD=%~dp0..\..\PCBuild\ + + +echo Building Lib\distutils\command\wininst-xx.0.exe + +call "%PCBUILD%env.bat" x86 +if errorlevel 1 goto :eof + +msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +if errorlevel 1 goto :eof + + +echo Building Lib\distutils\command\wininst-xx.0-amd64.exe + +call "%PCBUILD%env.bat" x86_amd64 +if errorlevel 1 goto :eof + +msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PCbuild/build.bat b/PCbuild/build.bat index f7f2858d7d9d99..8e1c168a8591f8 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -1,156 +1,156 @@ -@echo off -goto Run -:Usage -echo.%~nx0 [flags and arguments] [quoted MSBuild options] -echo. -echo.Build CPython from the command line. Requires the appropriate -echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). -echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is -echo.given. -echo. -echo.After the flags recognized by this script, up to 9 arguments to be passed -echo.directly to MSBuild may be passed. If the argument contains an '=', the -echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`) -echo. -echo.Available flags: -echo. -h Display this help message -echo. -V Display version information for the current build -echo. -r Target Rebuild instead of Build -echo. -d Set the configuration to Debug -echo. -e Build external libraries fetched by get_externals.bat -echo. Extension modules that depend on external libraries will not attempt -echo. to build if this flag is not present -echo. -m Enable parallel build (enabled by default) -echo. -M Disable parallel build -echo. -v Increased output messages -echo. -k Attempt to kill any running Pythons before building (usually done -echo. automatically by the pythoncore project) -echo. --pgo Build with Profile-Guided Optimization. This flag -echo. overrides -c and -d -echo. --test-marker Enable the test marker within the build. -echo. -echo.Available flags to avoid building certain modules. -echo.These flags have no effect if '-e' is not given: -echo. --no-ssl Do not attempt to build _ssl -echo. --no-tkinter Do not attempt to build Tkinter -echo. -echo.Available arguments: -echo. -c Release ^| Debug ^| PGInstrument ^| PGUpdate -echo. Set the configuration (default: Release) -echo. -p x64 ^| Win32 -echo. Set the platform (default: Win32) -echo. -t Build ^| Rebuild ^| Clean ^| CleanAll -echo. Set the target manually -echo. --pgo-job The job to use for PGO training; implies --pgo -echo. (default: "-m test --pgo") -exit /b 127 - -:Run -setlocal -set platf=Win32 -set vs_platf=x86 -set conf=Release -set target=Build -set dir=%~dp0 -set parallel=/m -set verbose=/nologo /v:m -set kill= -set do_pgo= -set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) - -:CheckOpts -if "%~1"=="-h" goto Usage -if "%~1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts -if "%~1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts -if "%~1"=="-r" (set target=Rebuild) & shift & goto CheckOpts -if "%~1"=="-t" (set target=%2) & shift & shift & goto CheckOpts -if "%~1"=="-d" (set conf=Debug) & shift & goto CheckOpts -if "%~1"=="-m" (set parallel=/m) & shift & goto CheckOpts -if "%~1"=="-M" (set parallel=) & shift & goto CheckOpts -if "%~1"=="-v" (set verbose=/v:n) & shift & goto CheckOpts -if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts -if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts -if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts -if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts -if "%~1"=="-V" shift & goto Version -rem These use the actual property names used by MSBuild. We could just let -rem them in through the environment, but we specify them on the command line -rem anyway for visibility so set defaults after this -if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts -if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts -if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts - -if "%IncludeExternals%"=="" set IncludeExternals=false -if "%IncludeSSL%"=="" set IncludeSSL=true -if "%IncludeTkinter%"=="" set IncludeTkinter=true - -if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" - -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 - ) -) - -if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" -if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" -if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= - -rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul - -if "%kill%"=="true" call :Kill - -if "%do_pgo%"=="true" ( - set conf=PGInstrument - call :Build - del /s "%dir%\*.pgc" - del /s "%dir%\..\Lib\*.pyc" - echo on - call "%dir%\..\python.bat" %pgo_job% - @echo off - call :Kill - set conf=PGUpdate - set target=Build -) -goto Build -:Kill -echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ - /p:Configuration=%conf% /p:Platform=%platf%^ - /p:KillPython=true - -@echo off -goto :eof - -:Build -rem Call on MSBuild to do the work, echo the command. -rem Passing %1-9 is not the preferred option, but argument parsing in -rem batch is, shall we say, "lackluster" -echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ - /p:Configuration=%conf% /p:Platform=%platf%^ - /p:IncludeExternals=%IncludeExternals%^ - /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %GITProperty%^ - %1 %2 %3 %4 %5 %6 %7 %8 %9 - -@echo off -goto :eof - -:Version -rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +@echo off +goto Run +:Usage +echo.%~nx0 [flags and arguments] [quoted MSBuild options] +echo. +echo.Build CPython from the command line. Requires the appropriate +echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). +echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is +echo.given. +echo. +echo.After the flags recognized by this script, up to 9 arguments to be passed +echo.directly to MSBuild may be passed. If the argument contains an '=', the +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`) +echo. +echo.Available flags: +echo. -h Display this help message +echo. -V Display version information for the current build +echo. -r Target Rebuild instead of Build +echo. -d Set the configuration to Debug +echo. -e Build external libraries fetched by get_externals.bat +echo. Extension modules that depend on external libraries will not attempt +echo. to build if this flag is not present +echo. -m Enable parallel build (enabled by default) +echo. -M Disable parallel build +echo. -v Increased output messages +echo. -k Attempt to kill any running Pythons before building (usually done +echo. automatically by the pythoncore project) +echo. --pgo Build with Profile-Guided Optimization. This flag +echo. overrides -c and -d +echo. --test-marker Enable the test marker within the build. +echo. +echo.Available flags to avoid building certain modules. +echo.These flags have no effect if '-e' is not given: +echo. --no-ssl Do not attempt to build _ssl +echo. --no-tkinter Do not attempt to build Tkinter +echo. +echo.Available arguments: +echo. -c Release ^| Debug ^| PGInstrument ^| PGUpdate +echo. Set the configuration (default: Release) +echo. -p x64 ^| Win32 +echo. Set the platform (default: Win32) +echo. -t Build ^| Rebuild ^| Clean ^| CleanAll +echo. Set the target manually +echo. --pgo-job The job to use for PGO training; implies --pgo +echo. (default: "-m test --pgo") +exit /b 127 + +:Run +setlocal +set platf=Win32 +set vs_platf=x86 +set conf=Release +set target=Build +set dir=%~dp0 +set parallel=/m +set verbose=/nologo /v:m +set kill= +set do_pgo= +set pgo_job=-m test --pgo +set on_64_bit=true + +rem This may not be 100% accurate, but close enough. +if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) + +:CheckOpts +if "%~1"=="-h" goto Usage +if "%~1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-r" (set target=Rebuild) & shift & goto CheckOpts +if "%~1"=="-t" (set target=%2) & shift & shift & goto CheckOpts +if "%~1"=="-d" (set conf=Debug) & shift & goto CheckOpts +if "%~1"=="-m" (set parallel=/m) & shift & goto CheckOpts +if "%~1"=="-M" (set parallel=) & shift & goto CheckOpts +if "%~1"=="-v" (set verbose=/v:n) & shift & goto CheckOpts +if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts +if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts +if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts +if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts +if "%~1"=="-V" shift & goto Version +rem These use the actual property names used by MSBuild. We could just let +rem them in through the environment, but we specify them on the command line +rem anyway for visibility so set defaults after this +if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts +if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts + +if "%IncludeExternals%"=="" set IncludeExternals=false +if "%IncludeSSL%"=="" set IncludeSSL=true +if "%IncludeTkinter%"=="" set IncludeTkinter=true + +if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" + +if "%platf%"=="x64" ( + if "%on_64_bit%"=="true" ( + rem This ought to always be correct these days... + set vs_platf=amd64 + ) else ( + if "%do_pgo%"=="true" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected, if this is incorrect, + echo. make sure the ProgramFiles(x86^) environment variable is set + exit /b 1 + ) + set vs_platf=x86_amd64 + ) +) + +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= + +rem Setup the environment +call "%dir%env.bat" %vs_platf% >nul + +if "%kill%"=="true" call :Kill + +if "%do_pgo%"=="true" ( + set conf=PGInstrument + call :Build + del /s "%dir%\*.pgc" + del /s "%dir%\..\Lib\*.pyc" + echo on + call "%dir%\..\python.bat" %pgo_job% + @echo off + call :Kill + set conf=PGUpdate + set target=Build +) +goto Build +:Kill +echo on +msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:KillPython=true + +@echo off +goto :eof + +:Build +rem Call on MSBuild to do the work, echo the command. +rem Passing %1-9 is not the preferred option, but argument parsing in +rem batch is, shall we say, "lackluster" +echo on +msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:IncludeExternals=%IncludeExternals%^ + /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + %1 %2 %3 %4 %5 %6 %7 %8 %9 + +@echo off +goto :eof + +:Version +rem Display the current build version information +msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/build_env.bat b/PCbuild/build_env.bat index 01024cff215d39..4c67ae3ac253c7 100644 --- a/PCbuild/build_env.bat +++ b/PCbuild/build_env.bat @@ -1 +1 @@ -@%comspec% /k env.bat %* +@%comspec% /k env.bat %* diff --git a/PCbuild/clean.bat b/PCbuild/clean.bat index 78517d225f9f44..6144c715f64e43 100644 --- a/PCbuild/clean.bat +++ b/PCbuild/clean.bat @@ -1,5 +1,5 @@ -@echo off -rem A batch program to clean a particular configuration, -rem just for convenience. - -call %~dp0build.bat -t Clean %* +@echo off +rem A batch program to clean a particular configuration, +rem just for convenience. + +call %~dp0build.bat -t Clean %* diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 2b2c0051ce912f..9d4c9d1c32f7a4 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -1,16 +1,16 @@ -@echo off -rem This script adds the latest available tools to the path for the current -rem command window. However, most builds of Python will ignore the version -rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of -rem tools should be the same version to avoid potential conflicts. -rem -rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or -rem 'v110', 'v120' or 'v140') to the build script. - -echo Build environments: x86, amd64, x86_amd64 -echo. -set VSTOOLS=%VS140COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS120COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS110COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS100COMNTOOLS% -call "%VSTOOLS%..\..\VC\vcvarsall.bat" %* +@echo off +rem This script adds the latest available tools to the path for the current +rem command window. However, most builds of Python will ignore the version +rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of +rem tools should be the same version to avoid potential conflicts. +rem +rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or +rem 'v110', 'v120' or 'v140') to the build script. + +echo Build environments: x86, amd64, x86_amd64 +echo. +set VSTOOLS=%VS140COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS120COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS110COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS100COMNTOOLS% +call "%VSTOOLS%..\..\VC\vcvarsall.bat" %* diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 3a2656e961ef7f..5767ab2eacf25b 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -1,104 +1,104 @@ -@echo off -setlocal -rem Simple script to fetch source for external libraries - -if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" -pushd "%~dp0..\externals" - -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ - -rem Optionally clean up first. Be warned that this can be very destructive! -if not "%1"=="" ( - for %%c in (-c --clean --clean-only) do ( - if "%1"=="%%c" goto clean - ) - goto usage -) -goto fetch - -:clean -echo.Cleaning up external libraries. -for /D %%d in ( - bzip2-* - db-* - nasm-* - openssl-* - tcl-* - tcltk* - tk-* - tix-* - sqlite-* - xz-* - ) do ( - echo.Removing %%d - rmdir /s /q %%d -) -if "%1"=="--clean-only" ( - goto end -) - -:fetch -rem Fetch current versions - -svn --version > nul 2>&1 -if ERRORLEVEL 9009 ( - echo.svn.exe must be on your PATH. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 -) - -echo.Fetching external libraries... - -set libraries= -set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k -set libraries=%libraries% sqlite-3.14.2.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 -set libraries=%libraries% xz-5.2.2 - -for %%e in (%libraries%) do ( - if exist %%e ( - echo.%%e already exists, skipping. - ) else ( - echo.Fetching %%e... - svn export -q %SVNROOT%%%e - ) -) - -goto end - -:usage -echo.invalid argument: %1 -echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] -echo. -echo.Pull all sources necessary for compiling optional extension modules -echo.that rely on external libraries. Requires svn.exe to be on your PATH -echo.and pulls sources from %SVNROOT%. -echo. -echo.Use the -c or --clean option to clean up all external library sources -echo.before pulling in the current versions. -echo. -echo.Use the --clean-only option to do the same cleaning, without pulling in -echo.anything new. -echo. -echo.Only the first argument is checked, all others are ignored. -echo. -echo.**WARNING**: the cleaning options unconditionally remove any directory -echo.that is a child of -echo. %CD% -echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, -echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential -echo.to be very destructive if you are not aware of what it is doing. Use with -echo.caution! -popd -exit /b -1 - - -:end -echo Finished. -popd +@echo off +setlocal +rem Simple script to fetch source for external libraries + +if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" +pushd "%~dp0..\externals" + +if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ + +rem Optionally clean up first. Be warned that this can be very destructive! +if not "%1"=="" ( + for %%c in (-c --clean --clean-only) do ( + if "%1"=="%%c" goto clean + ) + goto usage +) +goto fetch + +:clean +echo.Cleaning up external libraries. +for /D %%d in ( + bzip2-* + db-* + nasm-* + openssl-* + tcl-* + tcltk* + tk-* + tix-* + sqlite-* + xz-* + ) do ( + echo.Removing %%d + rmdir /s /q %%d +) +if "%1"=="--clean-only" ( + goto end +) + +:fetch +rem Fetch current versions + +svn --version > nul 2>&1 +if ERRORLEVEL 9009 ( + echo.svn.exe must be on your PATH. + echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the + echo.command line tools option. + popd + exit /b 1 +) + +echo.Fetching external libraries... + +set libraries= +set libraries=%libraries% bzip2-1.0.6 +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k +set libraries=%libraries% sqlite-3.14.2.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 +set libraries=%libraries% xz-5.2.2 + +for %%e in (%libraries%) do ( + if exist %%e ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + svn export -q %SVNROOT%%%e + ) +) + +goto end + +:usage +echo.invalid argument: %1 +echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo. +echo.Pull all sources necessary for compiling optional extension modules +echo.that rely on external libraries. Requires svn.exe to be on your PATH +echo.and pulls sources from %SVNROOT%. +echo. +echo.Use the -c or --clean option to clean up all external library sources +echo.before pulling in the current versions. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +echo.Only the first argument is checked, all others are ignored. +echo. +echo.**WARNING**: the cleaning options unconditionally remove any directory +echo.that is a child of +echo. %CD% +echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, +echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential +echo.to be very destructive if you are not aware of what it is doing. Use with +echo.caution! +popd +exit /b -1 + + +:end +echo Finished. +popd diff --git a/PCbuild/idle.bat b/PCbuild/idle.bat index bacaaa8414a267..1978b99f6ee19d 100644 --- a/PCbuild/idle.bat +++ b/PCbuild/idle.bat @@ -1,15 +1,15 @@ -@echo off -rem start idle -rem Usage: idle [-d] -rem -d Run Debug build (python_d.exe). Else release build. - -setlocal -set exe=win32\python -PATH %PATH%;..\externals\tcltk\bin - -if "%1"=="-d" (set exe=%exe%_d) & shift - -set cmd=%exe% ../Lib/idlelib/idle.py %1 %2 %3 %4 %5 %6 %7 %8 %9 - -echo on -%cmd% +@echo off +rem start idle +rem Usage: idle [-d] +rem -d Run Debug build (python_d.exe). Else release build. + +setlocal +set exe=win32\python +PATH %PATH%;..\externals\tcltk\bin + +if "%1"=="-d" (set exe=%exe%_d) & shift + +set cmd=%exe% ../Lib/idlelib/idle.py %1 %2 %3 %4 %5 %6 %7 %8 %9 + +echo on +%cmd% diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 580879930a83d0..736a5997037208 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,769 +1,769 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25123.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" - ProjectSection(SolutionItems) = preProject - ..\Modules\getbuildinfo.c = ..\Modules\getbuildinfo.c - readme.txt = readme.txt - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonw", "pythonw.vcxproj", "{F4229CC3-873C-49AE-9729-DD308ED4CD4A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winsound", "winsound.vcxproj", "{28B5D777-DDF2-4B6B-B34F-31D938813856}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes_test", "_ctypes_test.vcxproj", "{9EC7190A-249F-4180-A900-548FDCF3055F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_msi", "_msi.vcxproj", "{31FFC478-7B4A-43E8-9954-8D03E2187E9C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_sqlite3", "_sqlite3.vcxproj", "{13CECB97-4119-4316-9D42-8534019A5A44}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testcapi", "_testcapi.vcxproj", "{6901D91C-6E48-4BB7-9FEC-700C8131DF1D}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_tkinter", "_tkinter.vcxproj", "{4946ECAC-2E69-4BF8-A90A-F5136F5094DF}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_bz2", "_bz2.vcxproj", "{73FCD2BD-F133-46B7-8EC1-144CD82A59D5}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_lzma", "_lzma.vcxproj", "{F9D71780-F393-11E0-BE50-0800200C9A66}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bdist_wininst", "..\PC\bdist_wininst\bdist_wininst.vcxproj", "{EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sqlite3", "sqlite3.vcxproj", "{A1A295E5-463C-437F-81CA-1F32367685DA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3dll", "python3dll.vcxproj", "{885D4898-D08D-4091-9C40-C700CFE3FC5A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxproj", "{F749B822-B489-4CA5-A3AD-CE078F5F338A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pylauncher", "pylauncher.vcxproj", "{7B2727B5-5A3F-40EE-A866-43A13CD31446}" - ProjectSection(ProjectDependencies) = postProject - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} = {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.vcxproj", "{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}" - ProjectSection(ProjectDependencies) = postProject - {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testembed", "_testembed.vcxproj", "{6DAC66D9-E703-4624-BE03-49112AB5AA62}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmultiphase.vcxproj", "{16BFE6F0-22EF-40B5-B831-7E937119EF10}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tcl", "tcl.vcxproj", "{B5FD6F1D-129E-4BFF-9340-03606FAC7283}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tix", "tix.vcxproj", "{C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tk", "tk.vcxproj", "{7E85ECCF-A72C-4DA4-9E52-884508E80BA1}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay", "libeay.vcxproj", "{E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay", "ssleay.vcxproj", "{10615B24-73BF-4EFA-93AA-236916321317}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - PGInstrument|Win32 = PGInstrument|Win32 - PGInstrument|x64 = PGInstrument|x64 - PGUpdate|Win32 = PGUpdate|Win32 - PGUpdate|x64 = PGUpdate|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.ActiveCfg = Debug|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.Build.0 = Debug|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.ActiveCfg = Debug|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.Build.0 = Debug|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.ActiveCfg = Release|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.Build.0 = Release|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.ActiveCfg = Release|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.Build.0 = Release|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.ActiveCfg = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.Build.0 = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.ActiveCfg = Release|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.ActiveCfg = Debug|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.Build.0 = Debug|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.ActiveCfg = Debug|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.Build.0 = Debug|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.Build.0 = Release|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.ActiveCfg = Debug|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.Build.0 = Debug|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.ActiveCfg = Debug|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.Build.0 = Debug|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.ActiveCfg = Release|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.Build.0 = Release|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.ActiveCfg = Release|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.ActiveCfg = Debug|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.Build.0 = Debug|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.ActiveCfg = Debug|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.Build.0 = Debug|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.Build.0 = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.ActiveCfg = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.Build.0 = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.ActiveCfg = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.Build.0 = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.ActiveCfg = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.Build.0 = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.ActiveCfg = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.Build.0 = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.ActiveCfg = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.Build.0 = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.ActiveCfg = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.Build.0 = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.ActiveCfg = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.Build.0 = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.ActiveCfg = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.Build.0 = Release|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.ActiveCfg = Debug|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.Build.0 = Debug|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.ActiveCfg = Debug|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.Build.0 = Debug|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.ActiveCfg = Release|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.Build.0 = Release|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.ActiveCfg = Release|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.Build.0 = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.ActiveCfg = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.Build.0 = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.ActiveCfg = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.Build.0 = Release|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.ActiveCfg = Debug|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.Build.0 = Debug|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.ActiveCfg = Debug|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.Build.0 = Debug|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.ActiveCfg = Release|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.Build.0 = Release|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.ActiveCfg = Release|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.ActiveCfg = Debug|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.Build.0 = Debug|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.ActiveCfg = Debug|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.Build.0 = Debug|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.Build.0 = Release|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.ActiveCfg = Debug|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.Build.0 = Debug|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.ActiveCfg = Debug|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.Build.0 = Debug|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.ActiveCfg = Release|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.Build.0 = Release|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.ActiveCfg = Release|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.Build.0 = Release|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.ActiveCfg = Debug|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.Build.0 = Debug|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.ActiveCfg = Debug|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.Build.0 = Debug|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.ActiveCfg = Release|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.Build.0 = Release|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.ActiveCfg = Release|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.Build.0 = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.ActiveCfg = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.Build.0 = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.ActiveCfg = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.Build.0 = Release|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.ActiveCfg = Debug|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.Build.0 = Debug|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.ActiveCfg = Debug|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.Build.0 = Debug|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.ActiveCfg = Release|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.Build.0 = Release|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.ActiveCfg = Release|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.Build.0 = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.ActiveCfg = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.Build.0 = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.ActiveCfg = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.Build.0 = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.ActiveCfg = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.Build.0 = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.ActiveCfg = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.Build.0 = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|Win32.ActiveCfg = Debug|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|x64.ActiveCfg = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.ActiveCfg = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.Build.0 = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.ActiveCfg = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.Build.0 = Release|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.ActiveCfg = Debug|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.Build.0 = Debug|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.ActiveCfg = Debug|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.Build.0 = Debug|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.ActiveCfg = Release|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.Build.0 = Release|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.ActiveCfg = Release|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.Build.0 = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.ActiveCfg = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.Build.0 = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.ActiveCfg = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.Build.0 = Release|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.ActiveCfg = Release|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.Build.0 = Release|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.ActiveCfg = Release|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.Build.0 = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|Win32.ActiveCfg = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.Build.0 = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.ActiveCfg = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.Build.0 = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.ActiveCfg = Debug|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.Build.0 = Debug|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.ActiveCfg = Debug|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.Build.0 = Debug|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.ActiveCfg = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.Build.0 = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.ActiveCfg = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.Build.0 = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.ActiveCfg = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.Build.0 = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.ActiveCfg = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.Build.0 = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.ActiveCfg = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.Build.0 = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.ActiveCfg = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.Build.0 = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.ActiveCfg = Debug|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.Build.0 = Debug|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.ActiveCfg = Debug|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.Build.0 = Debug|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.Build.0 = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.Build.0 = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.Build.0 = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.Build.0 = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.ActiveCfg = Debug|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.Build.0 = Debug|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.ActiveCfg = Debug|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.Build.0 = Debug|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.Build.0 = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.Build.0 = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.Build.0 = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.ActiveCfg = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.Build.0 = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.ActiveCfg = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.Build.0 = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.ActiveCfg = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.Build.0 = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.ActiveCfg = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.Build.0 = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.ActiveCfg = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.Build.0 = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.ActiveCfg = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.Build.0 = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.ActiveCfg = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.Build.0 = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.ActiveCfg = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.Build.0 = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.ActiveCfg = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.Build.0 = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.ActiveCfg = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.Build.0 = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.Build.0 = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.Build.0 = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.ActiveCfg = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.ActiveCfg = Debug|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.Build.0 = Debug|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.ActiveCfg = Debug|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.Build.0 = Debug|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 - {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25123.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" + ProjectSection(SolutionItems) = preProject + ..\Modules\getbuildinfo.c = ..\Modules\getbuildinfo.c + readme.txt = readme.txt + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonw", "pythonw.vcxproj", "{F4229CC3-873C-49AE-9729-DD308ED4CD4A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winsound", "winsound.vcxproj", "{28B5D777-DDF2-4B6B-B34F-31D938813856}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes_test", "_ctypes_test.vcxproj", "{9EC7190A-249F-4180-A900-548FDCF3055F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_msi", "_msi.vcxproj", "{31FFC478-7B4A-43E8-9954-8D03E2187E9C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_sqlite3", "_sqlite3.vcxproj", "{13CECB97-4119-4316-9D42-8534019A5A44}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testcapi", "_testcapi.vcxproj", "{6901D91C-6E48-4BB7-9FEC-700C8131DF1D}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_tkinter", "_tkinter.vcxproj", "{4946ECAC-2E69-4BF8-A90A-F5136F5094DF}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_bz2", "_bz2.vcxproj", "{73FCD2BD-F133-46B7-8EC1-144CD82A59D5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_lzma", "_lzma.vcxproj", "{F9D71780-F393-11E0-BE50-0800200C9A66}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bdist_wininst", "..\PC\bdist_wininst\bdist_wininst.vcxproj", "{EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sqlite3", "sqlite3.vcxproj", "{A1A295E5-463C-437F-81CA-1F32367685DA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3dll", "python3dll.vcxproj", "{885D4898-D08D-4091-9C40-C700CFE3FC5A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxproj", "{F749B822-B489-4CA5-A3AD-CE078F5F338A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pylauncher", "pylauncher.vcxproj", "{7B2727B5-5A3F-40EE-A866-43A13CD31446}" + ProjectSection(ProjectDependencies) = postProject + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} = {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.vcxproj", "{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}" + ProjectSection(ProjectDependencies) = postProject + {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testembed", "_testembed.vcxproj", "{6DAC66D9-E703-4624-BE03-49112AB5AA62}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmultiphase.vcxproj", "{16BFE6F0-22EF-40B5-B831-7E937119EF10}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tcl", "tcl.vcxproj", "{B5FD6F1D-129E-4BFF-9340-03606FAC7283}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tix", "tix.vcxproj", "{C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tk", "tk.vcxproj", "{7E85ECCF-A72C-4DA4-9E52-884508E80BA1}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay", "libeay.vcxproj", "{E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay", "ssleay.vcxproj", "{10615B24-73BF-4EFA-93AA-236916321317}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + PGInstrument|Win32 = PGInstrument|Win32 + PGInstrument|x64 = PGInstrument|x64 + PGUpdate|Win32 = PGUpdate|Win32 + PGUpdate|x64 = PGUpdate|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.ActiveCfg = Debug|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.Build.0 = Debug|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.ActiveCfg = Debug|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.Build.0 = Debug|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.ActiveCfg = Release|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.Build.0 = Release|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.ActiveCfg = Release|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.Build.0 = Release|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.ActiveCfg = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.Build.0 = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.ActiveCfg = Release|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.ActiveCfg = Debug|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.Build.0 = Debug|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.ActiveCfg = Debug|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.Build.0 = Debug|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.Build.0 = Release|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.ActiveCfg = Debug|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.Build.0 = Debug|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.ActiveCfg = Debug|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.Build.0 = Debug|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.ActiveCfg = Release|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.Build.0 = Release|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.ActiveCfg = Release|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.ActiveCfg = Debug|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.Build.0 = Debug|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.ActiveCfg = Debug|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.Build.0 = Debug|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.Build.0 = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.ActiveCfg = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.Build.0 = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.ActiveCfg = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.Build.0 = Release|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.ActiveCfg = Debug|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.Build.0 = Debug|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.ActiveCfg = Debug|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.Build.0 = Debug|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.ActiveCfg = Release|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.Build.0 = Release|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.ActiveCfg = Release|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.Build.0 = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.ActiveCfg = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.Build.0 = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.ActiveCfg = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.Build.0 = Release|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.ActiveCfg = Debug|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.Build.0 = Debug|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.ActiveCfg = Debug|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.Build.0 = Debug|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.ActiveCfg = Release|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.Build.0 = Release|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.ActiveCfg = Release|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.Build.0 = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.ActiveCfg = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.Build.0 = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.ActiveCfg = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.Build.0 = Release|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.ActiveCfg = Debug|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.Build.0 = Debug|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.ActiveCfg = Debug|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.Build.0 = Debug|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.ActiveCfg = Release|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.Build.0 = Release|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.ActiveCfg = Release|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.ActiveCfg = Debug|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.Build.0 = Debug|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.ActiveCfg = Debug|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.Build.0 = Debug|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.Build.0 = Release|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.ActiveCfg = Debug|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.Build.0 = Debug|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.ActiveCfg = Debug|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.Build.0 = Debug|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.ActiveCfg = Release|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.Build.0 = Release|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.ActiveCfg = Release|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.Build.0 = Release|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.ActiveCfg = Debug|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.Build.0 = Debug|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.ActiveCfg = Debug|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.Build.0 = Debug|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.ActiveCfg = Release|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.Build.0 = Release|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.ActiveCfg = Release|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.Build.0 = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.ActiveCfg = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.Build.0 = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.ActiveCfg = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.Build.0 = Release|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.ActiveCfg = Debug|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.Build.0 = Debug|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.ActiveCfg = Debug|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.Build.0 = Debug|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.ActiveCfg = Release|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.Build.0 = Release|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.ActiveCfg = Release|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.Build.0 = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.ActiveCfg = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.Build.0 = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.ActiveCfg = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.Build.0 = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.ActiveCfg = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.Build.0 = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.ActiveCfg = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.Build.0 = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|Win32.ActiveCfg = Debug|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|x64.ActiveCfg = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.ActiveCfg = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.Build.0 = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.ActiveCfg = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.Build.0 = Release|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.ActiveCfg = Debug|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.Build.0 = Debug|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.ActiveCfg = Debug|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.Build.0 = Debug|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.ActiveCfg = Release|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.Build.0 = Release|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.ActiveCfg = Release|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.Build.0 = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.ActiveCfg = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.Build.0 = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.ActiveCfg = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.Build.0 = Release|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.ActiveCfg = Release|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.Build.0 = Release|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.ActiveCfg = Release|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.Build.0 = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|Win32.ActiveCfg = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.Build.0 = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.ActiveCfg = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.Build.0 = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.ActiveCfg = Debug|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.Build.0 = Debug|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.ActiveCfg = Debug|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.Build.0 = Debug|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.ActiveCfg = Debug|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.Build.0 = Debug|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.ActiveCfg = Debug|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.Build.0 = Debug|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.ActiveCfg = Debug|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.Build.0 = Debug|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.ActiveCfg = Debug|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.Build.0 = Debug|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.ActiveCfg = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.Build.0 = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.ActiveCfg = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.Build.0 = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.ActiveCfg = Debug|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.Build.0 = Debug|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.ActiveCfg = Debug|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.Build.0 = Debug|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.Build.0 = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.Build.0 = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.Build.0 = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.Build.0 = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.ActiveCfg = Debug|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.Build.0 = Debug|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.ActiveCfg = Debug|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.Build.0 = Debug|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.Build.0 = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.Build.0 = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.Build.0 = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.ActiveCfg = Debug|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.Build.0 = Debug|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.ActiveCfg = Debug|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.Build.0 = Debug|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.ActiveCfg = Debug|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.Build.0 = Debug|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.ActiveCfg = Debug|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.Build.0 = Debug|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.ActiveCfg = Debug|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.Build.0 = Debug|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.ActiveCfg = Debug|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.Build.0 = Debug|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.ActiveCfg = Debug|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.Build.0 = Debug|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.ActiveCfg = Debug|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.Build.0 = Debug|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.ActiveCfg = Debug|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.Build.0 = Debug|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.ActiveCfg = Debug|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.Build.0 = Debug|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.Build.0 = Release|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.Build.0 = Debug|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.ActiveCfg = Release|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.ActiveCfg = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.Build.0 = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.ActiveCfg = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.Build.0 = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/PCbuild/prepare_ssl.bat b/PCbuild/prepare_ssl.bat index 2f41ae847da5ae..6d9ac87a727076 100644 --- a/PCbuild/prepare_ssl.bat +++ b/PCbuild/prepare_ssl.bat @@ -1,12 +1,12 @@ -@echo off -if not defined HOST_PYTHON ( - if "%1" EQU "Debug" ( - shift - set HOST_PYTHON=python_d.exe - if not exist python36_d.dll exit 1 - ) ELSE ( - set HOST_PYTHON=python.exe - if not exist python36.dll exit 1 - ) -) -%HOST_PYTHON% "%~dp0prepare_ssl.py" %1 +@echo off +if not defined HOST_PYTHON ( + if "%1" EQU "Debug" ( + shift + set HOST_PYTHON=python_d.exe + if not exist python36_d.dll exit 1 + ) ELSE ( + set HOST_PYTHON=python.exe + if not exist python36.dll exit 1 + ) +) +%HOST_PYTHON% "%~dp0prepare_ssl.py" %1 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 8924b3fcfa2e80..7c87cbb1ae3a96 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -1,300 +1,300 @@ -Quick Start Guide ------------------ - -1. Install Microsoft Visual Studio 2015, any edition. -2. Install Subversion, and make sure 'svn.exe' is on your PATH. -3. Run "build.bat -e" to build Python in 32-bit Release configuration. -4. (Optional, but recommended) Run the test suite with "rt.bat -q". - - -Building Python using Microsoft Visual C++ ------------------------------------------- - -This directory is used to build CPython for Microsoft Windows NT version -6.0 or higher (Windows Vista, Windows Server 2008, or later) on 32 and 64 -bit platforms. Using this directory requires an installation of -Microsoft Visual C++ 2015 (MSVC 14.0) of any edition. The specific -requirements are as follows: - -Visual Studio Express 2015 for Desktop -Visual Studio Professional 2015 - Either edition is sufficient for building all configurations except - for Profile Guided Optimization. - The Python build solution pcbuild.sln makes use of Solution Folders, - which this edition does not support. Any time pcbuild.sln is opened - or reloaded by Visual Studio, a warning about Solution Folders will - be displayed, which can be safely dismissed with no impact on your - ability to build Python. - Required for building 64-bit Debug and Release configuration builds -Visual Studio Premium 2015 - Required for building Release configuration builds that make use of - Profile Guided Optimization (PGO), on either platform. - -All you need to do to build is open the solution "pcbuild.sln" in Visual -Studio, select the desired combination of configuration and platform, -then build with "Build Solution". You can also build from the command -line using the "build.bat" script in this directory; see below for -details. The solution is configured to build the projects in the correct -order. - -The solution currently supports two platforms. The Win32 platform is -used to build standard x86-compatible 32-bit binaries, output into the -win32 sub-directory. The x64 platform is used for building 64-bit AMD64 -(aka x86_64 or EM64T) binaries, output into the amd64 sub-directory. -The Itanium (IA-64) platform is no longer supported. - -Four configuration options are supported by the solution: -Debug - Used to build Python with extra debugging capabilities, equivalent - to using ./configure --with-pydebug on UNIX. All binaries built - using this configuration have "_d" added to their name: - python36_d.dll, python_d.exe, parser_d.pyd, and so on. Both the - build and rt (run test) batch files in this directory accept a -d - option for debug builds. If you are building Python to help with - development of CPython, you will most likely use this configuration. -PGInstrument, PGUpdate - Used to build Python in Release configuration using PGO, which - requires Premium Edition of Visual Studio. See the "Profile - Guided Optimization" section below for more information. Build - output from each of these configurations lands in its own - sub-directory of this directory. The official Python releases may - be built using these configurations. -Release - Used to build Python as it is meant to be used in production - settings, though without PGO. - - -Building Python using the build.bat script ----------------------------------------------- - -In this directory you can find build.bat, a script designed to make -building Python on Windows simpler. This script will use the env.bat -script to detect one of Visual Studio 2015, 2013, 2012, or 2010, any of -which may be used to build Python, though only Visual Studio 2015 is -officially supported. - -By default, build.bat will build Python in Release configuration for -the 32-bit Win32 platform. It accepts several arguments to change -this behavior, try `build.bat -h` to learn more. - - -C Runtime ---------- - -Visual Studio 2015 uses version 14 of the C runtime (MSVCRT14). The -executables no longer use the "Side by Side" assemblies used in previous -versions of the compiler. This simplifies distribution of applications. - -The run time libraries are available under the VC/Redist folder of your -Visual Studio distribution. For more info, see the Readme in the -VC/Redist folder. - - -Sub-Projects ------------- - -The CPython project is split up into several smaller sub-projects which -are managed by the pcbuild.sln solution file. Each sub-project is -represented by a .vcxproj and a .vcxproj.filters file starting with the -name of the sub-project. These sub-projects fall into a few general -categories: - -The following sub-projects represent the bare minimum required to build -a functioning CPython interpreter. If nothing else builds but these, -you'll have a very limited but usable python.exe: -pythoncore - .dll and .lib -python - .exe - -These sub-projects provide extra executables that are useful for running -CPython in different ways: -pythonw - pythonw.exe, a variant of python.exe that doesn't open a Command - Prompt window -pylauncher - py.exe, the Python Launcher for Windows, see - http://docs.python.org/3/using/windows.html#launcher -pywlauncher - pyw.exe, a variant of py.exe that doesn't open a Command Prompt - window -_testembed - _testembed.exe, a small program that embeds Python for testing - purposes, used by test_capi.py - -These are miscellaneous sub-projects that don't really fit the other -categories: -_freeze_importlib - _freeze_importlib.exe, used to regenerate Python\importlib.h after - changes have been made to Lib\importlib\_bootstrap.py -python3dll - python3.dll, the PEP 384 Stable ABI dll -xxlimited - builds an example module that makes use of the PEP 384 Stable ABI, - see Modules\xxlimited.c - -The following sub-projects are for individual modules of the standard -library which are implemented in C; each one builds a DLL (renamed to -.pyd) of the same name as the project: -_ctypes -_ctypes_test -_decimal -_elementtree -_hashlib -_msi -_multiprocessing -_overlapped -_socket -_testcapi -_testbuffer -_testimportmultiple -pyexpat -select -unicodedata -winsound - -The following Python-controlled sub-projects wrap external projects. -Note that these external libraries are not necessary for a working -interpreter, but they do implement several major features. See the -"Getting External Sources" section below for additional information -about getting the source for building these libraries. The sub-projects -are: -_bz2 - Python wrapper for version 1.0.6 of the libbzip2 compression library - Homepage: - http://www.bzip.org/ -_lzma - Python wrapper for the liblzma compression library, using pre-built - binaries of XZ Utils version 5.0.5 - Homepage: - http://tukaani.org/xz/ -_ssl - Python wrapper for version 1.0.2k of the OpenSSL secure sockets - library, which is built by ssl.vcxproj - Homepage: - http://www.openssl.org/ - - Building OpenSSL requires nasm.exe (the Netwide Assembler), version - 2.10 or newer from - http://www.nasm.us/ - to be somewhere on your PATH. More recent versions of OpenSSL may - need a later version of NASM. If OpenSSL's self tests don't pass, - you should first try to update NASM and do a full rebuild of - OpenSSL. If you use the PCbuild\get_externals.bat method - for getting sources, it also downloads a version of NASM which the - libeay/ssleay sub-projects use. - - The libeay/ssleay sub-projects expect your OpenSSL sources to have - already been configured and be ready to build. If you get your sources - from svn.python.org as suggested in the "Getting External Sources" - section below, the OpenSSL source will already be ready to go. If - you want to build a different version, you will need to run - - PCbuild\prepare_ssl.py path\to\openssl-source-dir - - That script will prepare your OpenSSL sources in the same way that - those available on svn.python.org have been prepared. Note that - Perl must be installed and available on your PATH to configure - OpenSSL. ActivePerl is recommended and is available from - http://www.activestate.com/activeperl/ - - The libeay and ssleay sub-projects will build the modules of OpenSSL - required by _ssl and _hashlib and may need to be manually updated when - upgrading to a newer version of OpenSSL or when adding new - functionality to _ssl or _hashlib. They will not clean up their output - with the normal Clean target; CleanAll should be used instead. -_sqlite3 - Wraps SQLite 3.14.2.0, which is itself built by sqlite3.vcxproj - Homepage: - http://www.sqlite.org/ -_tkinter - Wraps version 8.6.6 of the Tk windowing system. - Homepage: - http://www.tcl.tk/ - - Tkinter's dependencies are built by the tcl.vcxproj and tk.vcxproj - projects. The tix.vcxproj project also builds the Tix extended - widget set for use with Tkinter. - - Those three projects install their respective components in a - directory alongside the source directories called "tcltk" on - Win32 and "tcltk64" on x64. They also copy the Tcl and Tk DLLs - into the current output directory, which should ensure that Tkinter - is able to load Tcl/Tk without having to change your PATH. - - The tcl, tk, and tix sub-projects do not clean their builds with - the normal Clean target; if you need to rebuild, you should use the - CleanAll target or manually delete their builds. - - -Getting External Sources ------------------------- - -The last category of sub-projects listed above wrap external projects -Python doesn't control, and as such a little more work is required in -order to download the relevant source files for each project before they -can be built. However, a simple script is provided to make this as -painless as possible, called "get_externals.bat" and located in this -directory. This script extracts all the external sub-projects from - http://svn.python.org/projects/external -via Subversion (so you'll need svn.exe on your PATH) and places them -in ..\externals (relative to this directory). - -It is also possible to download sources from each project's homepage, -though you may have to change folder names or pass the names to MSBuild -as the values of certain properties in order for the build solution to -find them. This is an advanced topic and not necessarily fully -supported. - -The get_externals.bat script is called automatically by build.bat when -you pass the '-e' option to it. - - -Profile Guided Optimization ---------------------------- - -The solution has two configurations for PGO. The PGInstrument -configuration must be built first. The PGInstrument binaries are linked -against a profiling library and contain extra debug information. The -PGUpdate configuration takes the profiling data and generates optimized -binaries. - -The build_pgo.bat script automates the creation of optimized binaries. -It creates the PGI files, runs the unit test suite or PyBench with the -PGI python, and finally creates the optimized files. - -See - http://msdn.microsoft.com/en-us/library/e7k32f4k(VS.140).aspx -for more on this topic. - - -Static library --------------- - -The solution has no configuration for static libraries. However it is -easy to build a static library instead of a DLL. You simply have to set -the "Configuration Type" to "Static Library (.lib)" and alter the -preprocessor macro "Py_ENABLE_SHARED" to "Py_NO_ENABLE_SHARED". You may -also have to change the "Runtime Library" from "Multi-threaded DLL -(/MD)" to "Multi-threaded (/MT)". - - -Visual Studio properties ------------------------- - -The PCbuild solution makes use of Visual Studio property files (*.props) -to simplify each project. The properties can be viewed in the Property -Manager (View -> Other Windows -> Property Manager) but should be -carefully modified by hand. - -The property files used are: - * python (versions, directories and build names) - * pyproject (base settings for all projects) - * openssl (used by libeay and ssleay projects) - * tcltk (used by _tkinter, tcl, tk and tix projects) - -The pyproject property file defines all of the build settings for each -project, with some projects overriding certain specific values. The GUI -doesn't always reflect the correct settings and may confuse the user -with false information, especially for settings that automatically adapt -for diffirent configurations. +Quick Start Guide +----------------- + +1. Install Microsoft Visual Studio 2015, any edition. +2. Install Subversion, and make sure 'svn.exe' is on your PATH. +3. Run "build.bat -e" to build Python in 32-bit Release configuration. +4. (Optional, but recommended) Run the test suite with "rt.bat -q". + + +Building Python using Microsoft Visual C++ +------------------------------------------ + +This directory is used to build CPython for Microsoft Windows NT version +6.0 or higher (Windows Vista, Windows Server 2008, or later) on 32 and 64 +bit platforms. Using this directory requires an installation of +Microsoft Visual C++ 2015 (MSVC 14.0) of any edition. The specific +requirements are as follows: + +Visual Studio Express 2015 for Desktop +Visual Studio Professional 2015 + Either edition is sufficient for building all configurations except + for Profile Guided Optimization. + The Python build solution pcbuild.sln makes use of Solution Folders, + which this edition does not support. Any time pcbuild.sln is opened + or reloaded by Visual Studio, a warning about Solution Folders will + be displayed, which can be safely dismissed with no impact on your + ability to build Python. + Required for building 64-bit Debug and Release configuration builds +Visual Studio Premium 2015 + Required for building Release configuration builds that make use of + Profile Guided Optimization (PGO), on either platform. + +All you need to do to build is open the solution "pcbuild.sln" in Visual +Studio, select the desired combination of configuration and platform, +then build with "Build Solution". You can also build from the command +line using the "build.bat" script in this directory; see below for +details. The solution is configured to build the projects in the correct +order. + +The solution currently supports two platforms. The Win32 platform is +used to build standard x86-compatible 32-bit binaries, output into the +win32 sub-directory. The x64 platform is used for building 64-bit AMD64 +(aka x86_64 or EM64T) binaries, output into the amd64 sub-directory. +The Itanium (IA-64) platform is no longer supported. + +Four configuration options are supported by the solution: +Debug + Used to build Python with extra debugging capabilities, equivalent + to using ./configure --with-pydebug on UNIX. All binaries built + using this configuration have "_d" added to their name: + python36_d.dll, python_d.exe, parser_d.pyd, and so on. Both the + build and rt (run test) batch files in this directory accept a -d + option for debug builds. If you are building Python to help with + development of CPython, you will most likely use this configuration. +PGInstrument, PGUpdate + Used to build Python in Release configuration using PGO, which + requires Premium Edition of Visual Studio. See the "Profile + Guided Optimization" section below for more information. Build + output from each of these configurations lands in its own + sub-directory of this directory. The official Python releases may + be built using these configurations. +Release + Used to build Python as it is meant to be used in production + settings, though without PGO. + + +Building Python using the build.bat script +---------------------------------------------- + +In this directory you can find build.bat, a script designed to make +building Python on Windows simpler. This script will use the env.bat +script to detect one of Visual Studio 2015, 2013, 2012, or 2010, any of +which may be used to build Python, though only Visual Studio 2015 is +officially supported. + +By default, build.bat will build Python in Release configuration for +the 32-bit Win32 platform. It accepts several arguments to change +this behavior, try `build.bat -h` to learn more. + + +C Runtime +--------- + +Visual Studio 2015 uses version 14 of the C runtime (MSVCRT14). The +executables no longer use the "Side by Side" assemblies used in previous +versions of the compiler. This simplifies distribution of applications. + +The run time libraries are available under the VC/Redist folder of your +Visual Studio distribution. For more info, see the Readme in the +VC/Redist folder. + + +Sub-Projects +------------ + +The CPython project is split up into several smaller sub-projects which +are managed by the pcbuild.sln solution file. Each sub-project is +represented by a .vcxproj and a .vcxproj.filters file starting with the +name of the sub-project. These sub-projects fall into a few general +categories: + +The following sub-projects represent the bare minimum required to build +a functioning CPython interpreter. If nothing else builds but these, +you'll have a very limited but usable python.exe: +pythoncore + .dll and .lib +python + .exe + +These sub-projects provide extra executables that are useful for running +CPython in different ways: +pythonw + pythonw.exe, a variant of python.exe that doesn't open a Command + Prompt window +pylauncher + py.exe, the Python Launcher for Windows, see + http://docs.python.org/3/using/windows.html#launcher +pywlauncher + pyw.exe, a variant of py.exe that doesn't open a Command Prompt + window +_testembed + _testembed.exe, a small program that embeds Python for testing + purposes, used by test_capi.py + +These are miscellaneous sub-projects that don't really fit the other +categories: +_freeze_importlib + _freeze_importlib.exe, used to regenerate Python\importlib.h after + changes have been made to Lib\importlib\_bootstrap.py +python3dll + python3.dll, the PEP 384 Stable ABI dll +xxlimited + builds an example module that makes use of the PEP 384 Stable ABI, + see Modules\xxlimited.c + +The following sub-projects are for individual modules of the standard +library which are implemented in C; each one builds a DLL (renamed to +.pyd) of the same name as the project: +_ctypes +_ctypes_test +_decimal +_elementtree +_hashlib +_msi +_multiprocessing +_overlapped +_socket +_testcapi +_testbuffer +_testimportmultiple +pyexpat +select +unicodedata +winsound + +The following Python-controlled sub-projects wrap external projects. +Note that these external libraries are not necessary for a working +interpreter, but they do implement several major features. See the +"Getting External Sources" section below for additional information +about getting the source for building these libraries. The sub-projects +are: +_bz2 + Python wrapper for version 1.0.6 of the libbzip2 compression library + Homepage: + http://www.bzip.org/ +_lzma + Python wrapper for the liblzma compression library, using pre-built + binaries of XZ Utils version 5.0.5 + Homepage: + http://tukaani.org/xz/ +_ssl + Python wrapper for version 1.0.2k of the OpenSSL secure sockets + library, which is built by ssl.vcxproj + Homepage: + http://www.openssl.org/ + + Building OpenSSL requires nasm.exe (the Netwide Assembler), version + 2.10 or newer from + http://www.nasm.us/ + to be somewhere on your PATH. More recent versions of OpenSSL may + need a later version of NASM. If OpenSSL's self tests don't pass, + you should first try to update NASM and do a full rebuild of + OpenSSL. If you use the PCbuild\get_externals.bat method + for getting sources, it also downloads a version of NASM which the + libeay/ssleay sub-projects use. + + The libeay/ssleay sub-projects expect your OpenSSL sources to have + already been configured and be ready to build. If you get your sources + from svn.python.org as suggested in the "Getting External Sources" + section below, the OpenSSL source will already be ready to go. If + you want to build a different version, you will need to run + + PCbuild\prepare_ssl.py path\to\openssl-source-dir + + That script will prepare your OpenSSL sources in the same way that + those available on svn.python.org have been prepared. Note that + Perl must be installed and available on your PATH to configure + OpenSSL. ActivePerl is recommended and is available from + http://www.activestate.com/activeperl/ + + The libeay and ssleay sub-projects will build the modules of OpenSSL + required by _ssl and _hashlib and may need to be manually updated when + upgrading to a newer version of OpenSSL or when adding new + functionality to _ssl or _hashlib. They will not clean up their output + with the normal Clean target; CleanAll should be used instead. +_sqlite3 + Wraps SQLite 3.14.2.0, which is itself built by sqlite3.vcxproj + Homepage: + http://www.sqlite.org/ +_tkinter + Wraps version 8.6.6 of the Tk windowing system. + Homepage: + http://www.tcl.tk/ + + Tkinter's dependencies are built by the tcl.vcxproj and tk.vcxproj + projects. The tix.vcxproj project also builds the Tix extended + widget set for use with Tkinter. + + Those three projects install their respective components in a + directory alongside the source directories called "tcltk" on + Win32 and "tcltk64" on x64. They also copy the Tcl and Tk DLLs + into the current output directory, which should ensure that Tkinter + is able to load Tcl/Tk without having to change your PATH. + + The tcl, tk, and tix sub-projects do not clean their builds with + the normal Clean target; if you need to rebuild, you should use the + CleanAll target or manually delete their builds. + + +Getting External Sources +------------------------ + +The last category of sub-projects listed above wrap external projects +Python doesn't control, and as such a little more work is required in +order to download the relevant source files for each project before they +can be built. However, a simple script is provided to make this as +painless as possible, called "get_externals.bat" and located in this +directory. This script extracts all the external sub-projects from + http://svn.python.org/projects/external +via Subversion (so you'll need svn.exe on your PATH) and places them +in ..\externals (relative to this directory). + +It is also possible to download sources from each project's homepage, +though you may have to change folder names or pass the names to MSBuild +as the values of certain properties in order for the build solution to +find them. This is an advanced topic and not necessarily fully +supported. + +The get_externals.bat script is called automatically by build.bat when +you pass the '-e' option to it. + + +Profile Guided Optimization +--------------------------- + +The solution has two configurations for PGO. The PGInstrument +configuration must be built first. The PGInstrument binaries are linked +against a profiling library and contain extra debug information. The +PGUpdate configuration takes the profiling data and generates optimized +binaries. + +The build_pgo.bat script automates the creation of optimized binaries. +It creates the PGI files, runs the unit test suite or PyBench with the +PGI python, and finally creates the optimized files. + +See + http://msdn.microsoft.com/en-us/library/e7k32f4k(VS.140).aspx +for more on this topic. + + +Static library +-------------- + +The solution has no configuration for static libraries. However it is +easy to build a static library instead of a DLL. You simply have to set +the "Configuration Type" to "Static Library (.lib)" and alter the +preprocessor macro "Py_ENABLE_SHARED" to "Py_NO_ENABLE_SHARED". You may +also have to change the "Runtime Library" from "Multi-threaded DLL +(/MD)" to "Multi-threaded (/MT)". + + +Visual Studio properties +------------------------ + +The PCbuild solution makes use of Visual Studio property files (*.props) +to simplify each project. The properties can be viewed in the Property +Manager (View -> Other Windows -> Property Manager) but should be +carefully modified by hand. + +The property files used are: + * python (versions, directories and build names) + * pyproject (base settings for all projects) + * openssl (used by libeay and ssleay projects) + * tcltk (used by _tkinter, tcl, tk and tix projects) + +The pyproject property file defines all of the build settings for each +project, with some projects overriding certain specific values. The GUI +doesn't always reflect the correct settings and may confuse the user +with false information, especially for settings that automatically adapt +for diffirent configurations. diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index e73ac0457646e9..808102f826d370 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -1,63 +1,63 @@ -@echo off -rem Run Tests. Run the regression test suite. -rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args -rem -d Run Debug build (python_d.exe). Else release build. -rem -O Run python.exe or python_d.exe (see -d) with -O. -rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .pyc files reachable from Lib/. -rem -q runs the tests just once, and without deleting .pyc files. -rem -x64 Run the 64-bit build of python (or python_d if -d was specified) -rem from the 'amd64' dir instead of the 32-bit build in this dir. -rem All leading instances of these switches are shifted off, and -rem whatever remains (up to 9 arguments) is passed to regrtest.py. -rem For example, -rem rt -O -d -x test_thread -rem runs -rem python_d -O ../lib/test/regrtest.py -x test_thread -rem twice, and -rem rt -q -g test_binascii -rem runs -rem python_d ../lib/test/regrtest.py -g test_binascii -rem to generate the expected-output file for binascii quickly. -rem -rem Confusing: if you want to pass a comma-separated list, like -rem -u network,largefile -rem then you have to quote it on the rt line, like -rem rt -u "network,largefile" - -setlocal - -set pcbuild=%~dp0 -set prefix=%pcbuild%win32\ -set suffix= -set qmode= -set dashO= -set regrtestargs= - -:CheckOpts -if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts -if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts -if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts -if "%1"=="-x64" (set prefix=%pcbuild%amd64\) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts - -set exe=%prefix%python%suffix%.exe -set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% -if defined qmode goto Qmode - -echo Deleting .pyc files ... -"%exe%" "%pcbuild%rmpyc.py" - -echo Cleaning _pth files ... -if exist %prefix%*._pth del %prefix%*._pth - -echo on -%cmd% -@echo off - -echo About to run again without deleting .pyc first: -pause - -:Qmode -echo on -%cmd% +@echo off +rem Run Tests. Run the regression test suite. +rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args +rem -d Run Debug build (python_d.exe). Else release build. +rem -O Run python.exe or python_d.exe (see -d) with -O. +rem -q "quick" -- normally the tests are run twice, the first time +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. +rem -x64 Run the 64-bit build of python (or python_d if -d was specified) +rem from the 'amd64' dir instead of the 32-bit build in this dir. +rem All leading instances of these switches are shifted off, and +rem whatever remains (up to 9 arguments) is passed to regrtest.py. +rem For example, +rem rt -O -d -x test_thread +rem runs +rem python_d -O ../lib/test/regrtest.py -x test_thread +rem twice, and +rem rt -q -g test_binascii +rem runs +rem python_d ../lib/test/regrtest.py -g test_binascii +rem to generate the expected-output file for binascii quickly. +rem +rem Confusing: if you want to pass a comma-separated list, like +rem -u network,largefile +rem then you have to quote it on the rt line, like +rem rt -u "network,largefile" + +setlocal + +set pcbuild=%~dp0 +set prefix=%pcbuild%win32\ +set suffix= +set qmode= +set dashO= +set regrtestargs= + +:CheckOpts +if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts +if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts +if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +if "%1"=="-x64" (set prefix=%pcbuild%amd64\) & shift & goto CheckOpts +if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts + +set exe=%prefix%python%suffix%.exe +set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% +if defined qmode goto Qmode + +echo Deleting .pyc files ... +"%exe%" "%pcbuild%rmpyc.py" + +echo Cleaning _pth files ... +if exist %prefix%*._pth del %prefix%*._pth + +echo on +%cmd% +@echo off + +echo About to run again without deleting .pyc first: +pause + +:Qmode +echo on +%cmd% diff --git a/Tools/buildbot/build.bat b/Tools/buildbot/build.bat index 5e840cc7eaca84..9af511a81dbd8e 100644 --- a/Tools/buildbot/build.bat +++ b/Tools/buildbot/build.bat @@ -1,17 +1,17 @@ -@rem Used by the buildbot "compile" step. - -@rem Clean up -call "%~dp0clean.bat" %* - -@rem If you need the buildbots to start fresh (such as when upgrading to -@rem a new version of an external library, especially Tcl/Tk): -@rem 1) uncomment the following line: - -@rem call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only - -@rem 2) commit and push -@rem 3) wait for all Windows bots to start a build with that changeset -@rem 4) re-comment, commit and push again - -@rem Do the build -call "%~dp0..\..\PCbuild\build.bat" -e -d -k -v %* +@rem Used by the buildbot "compile" step. + +@rem Clean up +call "%~dp0clean.bat" %* + +@rem If you need the buildbots to start fresh (such as when upgrading to +@rem a new version of an external library, especially Tcl/Tk): +@rem 1) uncomment the following line: + +@rem call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only + +@rem 2) commit and push +@rem 3) wait for all Windows bots to start a build with that changeset +@rem 4) re-comment, commit and push again + +@rem Do the build +call "%~dp0..\..\PCbuild\build.bat" -e -d -k -v %* diff --git a/Tools/buildbot/buildmsi.bat b/Tools/buildbot/buildmsi.bat index e3c2dbd73d00df..6804d794799509 100644 --- a/Tools/buildbot/buildmsi.bat +++ b/Tools/buildbot/buildmsi.bat @@ -1,9 +1,9 @@ -@rem Used by the buildbot "buildmsi" step. -setlocal - -pushd - -@rem build both snapshot MSIs -call "%~dp0..\msi\build.bat" -x86 -x64 - +@rem Used by the buildbot "buildmsi" step. +setlocal + +pushd + +@rem build both snapshot MSIs +call "%~dp0..\msi\build.bat" -x86 -x64 + popd \ No newline at end of file diff --git a/Tools/buildbot/clean.bat b/Tools/buildbot/clean.bat index 13e667991b6ee9..fe252a916b1c24 100644 --- a/Tools/buildbot/clean.bat +++ b/Tools/buildbot/clean.bat @@ -1,17 +1,17 @@ -@echo off -rem Used by the buildbot "clean" step. - -setlocal -set root=%~dp0..\.. -set pcbuild=%root%\PCbuild - -echo Deleting build -call "%pcbuild%\build.bat" -t Clean -k %* -call "%pcbuild%\build.bat" -t Clean -k -d %* - -echo Deleting .pyc/.pyo files ... -del /s "%root%\Lib\*.pyc" "%root%\Lib\*.pyo" - -echo Deleting test leftovers ... -rmdir /s /q "%root%\build" -del /s "%pcbuild%\python*.zip" +@echo off +rem Used by the buildbot "clean" step. + +setlocal +set root=%~dp0..\.. +set pcbuild=%root%\PCbuild + +echo Deleting build +call "%pcbuild%\build.bat" -t Clean -k %* +call "%pcbuild%\build.bat" -t Clean -k -d %* + +echo Deleting .pyc/.pyo files ... +del /s "%root%\Lib\*.pyc" "%root%\Lib\*.pyo" + +echo Deleting test leftovers ... +rmdir /s /q "%root%\build" +del /s "%pcbuild%\python*.zip" diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index a32d38b5ec8bf5..0b3a3091c1e46e 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -1,19 +1,19 @@ -@echo off -rem Used by the buildbot "test" step. -setlocal - -set here=%~dp0 -set rt_opts=-q -d -set regrtest_args=-j1 - -:CheckOpts -if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts -if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts - -echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% +@echo off +rem Used by the buildbot "test" step. +setlocal + +set here=%~dp0 +set rt_opts=-q -d +set regrtest_args=-j1 + +:CheckOpts +if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts + +echo on +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index 69f00c0bb43f84..59e0261c6d5236 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -1,79 +1,79 @@ -@echo off -setlocal -set D=%~dp0 -set PCBUILD=%D%..\..\PCBuild\ - -set BUILDX86= -set BUILDX64= -set BUILDDOC= -set BUILDTEST=--test-marker -set BUILDPACK= -set REBUILD= - -:CheckOpts -if "%~1" EQU "-h" goto Help -if "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if "%~1" EQU "--doc" (set BUILDDOC=1) && shift && goto CheckOpts -if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts -if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts -if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts - -if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) - -call "%D%get_externals.bat" - -call "%PCBUILD%env.bat" x86 - -if defined BUILDX86 ( - call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% - if errorlevel 1 goto :eof - call "%PCBUILD%build.bat" -e %REBUILD% %BUILDTEST% - if errorlevel 1 goto :eof -) -if defined BUILDX64 ( - call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% - if errorlevel 1 goto :eof - call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% - if errorlevel 1 goto :eof -) - -if defined BUILDDOC ( - call "%PCBUILD%..\Doc\make.bat" htmlhelp - if errorlevel 1 goto :eof -) - -rem Build the launcher MSI separately -msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 - -set BUILD_CMD="%D%bundle\snapshot.wixproj" -if defined BUILDTEST ( - set BUILD_CMD=%BUILD_CMD% /p:UseTestMarker=true -) -if defined BUILDPACK ( - set BUILD_CMD=%BUILD_CMD% /p:Pack=true -) -if defined REBUILD ( - set BUILD_CMD=%BUILD_CMD% /t:Rebuild -) - -if defined BUILDX86 ( - msbuild %BUILD_CMD% - if errorlevel 1 goto :eof -) -if defined BUILDX64 ( - msbuild /p:Platform=x64 %BUILD_CMD% - if errorlevel 1 goto :eof -) - -exit /B 0 - -:Help -echo build.bat [-x86] [-x64] [--doc] [-h] [--no-test-marker] [--pack] [-r] -echo. -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo --doc Build CHM documentation -echo --no-test-marker Build without test markers -echo --pack Embed core MSIs into installer -echo -r Rebuild rather than incremental build +@echo off +setlocal +set D=%~dp0 +set PCBUILD=%D%..\..\PCBuild\ + +set BUILDX86= +set BUILDX64= +set BUILDDOC= +set BUILDTEST=--test-marker +set BUILDPACK= +set REBUILD= + +:CheckOpts +if "%~1" EQU "-h" goto Help +if "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts +if "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts +if "%~1" EQU "--doc" (set BUILDDOC=1) && shift && goto CheckOpts +if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts +if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts +if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts + +if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) + +call "%D%get_externals.bat" + +call "%PCBUILD%env.bat" x86 + +if defined BUILDX86 ( + call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% + if errorlevel 1 goto :eof + call "%PCBUILD%build.bat" -e %REBUILD% %BUILDTEST% + if errorlevel 1 goto :eof +) +if defined BUILDX64 ( + call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% + if errorlevel 1 goto :eof + call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% + if errorlevel 1 goto :eof +) + +if defined BUILDDOC ( + call "%PCBUILD%..\Doc\make.bat" htmlhelp + if errorlevel 1 goto :eof +) + +rem Build the launcher MSI separately +msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 + +set BUILD_CMD="%D%bundle\snapshot.wixproj" +if defined BUILDTEST ( + set BUILD_CMD=%BUILD_CMD% /p:UseTestMarker=true +) +if defined BUILDPACK ( + set BUILD_CMD=%BUILD_CMD% /p:Pack=true +) +if defined REBUILD ( + set BUILD_CMD=%BUILD_CMD% /t:Rebuild +) + +if defined BUILDX86 ( + msbuild %BUILD_CMD% + if errorlevel 1 goto :eof +) +if defined BUILDX64 ( + msbuild /p:Platform=x64 %BUILD_CMD% + if errorlevel 1 goto :eof +) + +exit /B 0 + +:Help +echo build.bat [-x86] [-x64] [--doc] [-h] [--no-test-marker] [--pack] [-r] +echo. +echo -x86 Build x86 installers +echo -x64 Build x64 installers +echo --doc Build CHM documentation +echo --no-test-marker Build without test markers +echo --pack Embed core MSIs into installer +echo -r Rebuild rather than incremental build diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 81a3f86cab36b0..ed930c653c11ea 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -1,235 +1,235 @@ -@setlocal -@echo off - -rem This script is intended for building official releases of Python. -rem To use it to build alternative releases, you should clone this file -rem and modify the following three URIs. - -rem These two will ensure that your release can be installed -rem alongside an official Python release, by modifying the GUIDs used -rem for all components. -rem -rem The following substitutions will be applied to the release URI: -rem Variable Description Example -rem {arch} architecture amd64, win32 -set RELEASE_URI=http://www.python.org/{arch} - -rem This is the URL that will be used to download installation files. -rem The files available from the default URL *will* conflict with your -rem installer. Trust me, you don't want them, even if it seems like a -rem good idea. -rem -rem The following substitutions will be applied to the download URL: -rem Variable Description Example -rem {version} version number 3.5.0 -rem {arch} architecture amd64, win32 -rem {releasename} release name a1, b2, rc3 (or blank for final) -rem {msi} MSI filename core.msi -set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} - -set D=%~dp0 -set PCBUILD=%D%..\..\PCBuild\ -set EXTERNALS=%D%..\..\externals\windows-installer\ - -set BUILDX86= -set BUILDX64= -set TARGET=Rebuild -set TESTTARGETDIR= -set PGO=-m test -q --pgo -set BUILDNUGET=1 -set BUILDZIP=1 - - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-c" (set CERTNAME=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--certificate" (set CERTNAME=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-o" (set OUTDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--out" (set OUTDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-D" (set SKIPDOC=1) && shift && goto CheckOpts -if "%1" EQU "--skip-doc" (set SKIPDOC=1) && shift && goto CheckOpts -if "%1" EQU "-B" (set SKIPBUILD=1) && shift && goto CheckOpts -if "%1" EQU "--skip-build" (set SKIPBUILD=1) && shift && goto CheckOpts -if "%1" EQU "--download" (set DOWNLOAD_URL=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--test" (set TESTTARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-b" (set TARGET=Build) && shift && goto CheckOpts -if "%1" EQU "--build" (set TARGET=Build) && shift && goto CheckOpts -if "%1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if "%1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if "%1" EQU "--pgo" (set PGO=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--skip-pgo" (set PGO=) && shift && goto CheckOpts -if "%1" EQU "--skip-nuget" (set BUILDNUGET=) && shift && goto CheckOpts -if "%1" EQU "--skip-zip" (set BUILDZIP=) && shift && goto CheckOpts - -if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 - -if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) - -if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" -if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 - -call "%D%get_externals.bat" - -:builddoc -if "%SKIPBUILD%" EQU "1" goto skipdoc -if "%SKIPDOC%" EQU "1" goto skipdoc - -if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1 -if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1 - -call "%D%..\..\doc\make.bat" htmlhelp -if errorlevel 1 goto :eof -:skipdoc - -where dlltool /q && goto skipdlltoolsearch -set _DLLTOOL_PATH= -where /R "%EXTERNALS%\" dlltool > "%TEMP%\dlltool.loc" 2> nul && set /P _DLLTOOL_PATH= < "%TEMP%\dlltool.loc" & del "%TEMP%\dlltool.loc" -if not exist "%_DLLTOOL_PATH%" echo Cannot find binutils on PATH or in external && exit /B 1 -for %%f in (%_DLLTOOL_PATH%) do set PATH=%PATH%;%%~dpf -set _DLLTOOL_PATH= -:skipdlltoolsearch - -if defined BUILDX86 ( - call :build x86 - if errorlevel 1 exit /B -) - -if defined BUILDX64 ( - call :build x64 "%PGO%" - if errorlevel 1 exit /B -) - -if defined TESTTARGETDIR ( - call "%D%testrelease.bat" -t "%TESTTARGETDIR%" -) - -exit /B 0 - -:build -@setlocal -@echo off - -if "%1" EQU "x86" ( - set PGO= - set BUILD=%PCBUILD%win32\ - set BUILD_PLAT=Win32 - set OUTDIR_PLAT=win32 - set OBJDIR_PLAT=x86 -) else ( - set BUILD=%PCBUILD%amd64\ - set PGO=%~2 - set BUILD_PLAT=x64 - set OUTDIR_PLAT=amd64 - set OBJDIR_PLAT=x64 -) - -if exist "%BUILD%en-us" ( - echo Deleting %BUILD%en-us - rmdir /q/s "%BUILD%en-us" - if errorlevel 1 exit /B -) - -if exist "%D%obj\Debug_%OBJDIR_PLAT%" ( - echo Deleting "%D%obj\Debug_%OBJDIR_PLAT%" - rmdir /q/s "%D%obj\Debug_%OBJDIR_PLAT%" - if errorlevel 1 exit /B -) - -if exist "%D%obj\Release_%OBJDIR_PLAT%" ( - echo Deleting "%D%obj\Release_%OBJDIR_PLAT%" - rmdir /q/s "%D%obj\Release_%OBJDIR_PLAT%" - if errorlevel 1 exit /B -) - -if not "%CERTNAME%" EQU "" ( - set CERTOPTS="/p:SigningCertificate=%CERTNAME%" -) else ( - set CERTOPTS= -) -if not "%PGO%" EQU "" ( - set PGOOPTS=--pgo-job "%PGO%" -) else ( - set PGOOPTS= -) -if not "%SKIPBUILD%" EQU "1" ( - @echo call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% - @call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% - @if errorlevel 1 exit /B - @rem build.bat turns echo back on, so we disable it again - @echo off - - @echo call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% - @call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% - @if errorlevel 1 exit /B - @rem build.bat turns echo back on, so we disable it again - @echo off -) - -call "%PCBUILD%env.bat" -if "%OUTDIR_PLAT%" EQU "win32" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% - if errorlevel 1 exit /B -) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% - if errorlevel 1 exit /B -) - -set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true -if errorlevel 1 exit /B -msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false -if errorlevel 1 exit /B - -if defined BUILDZIP ( - msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" - if errorlevel 1 exit /B -) - -if defined BUILDNUGET ( - msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" - if errorlevel 1 exit /B -) - -if not "%OUTDIR%" EQU "" ( - mkdir "%OUTDIR%\%OUTDIR_PLAT%" - mkdir "%OUTDIR%\%OUTDIR_PLAT%\binaries" - mkdir "%OUTDIR%\%OUTDIR_PLAT%\symbols" - robocopy "%BUILD%en-us" "%OUTDIR%\%OUTDIR_PLAT%" /XF "*.wixpdb" - robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\binaries" *.exe *.dll *.pyd /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" - robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\symbols" *.pdb /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" -) - -exit /B 0 - -:Help -echo buildrelease.bat [--out DIR] [-x86] [-x64] [--certificate CERTNAME] [--build] [--pgo COMMAND] -echo [--skip-build] [--skip-doc] [--skip-nuget] [--skip-zip] [--skip-pgo] -echo [--download DOWNLOAD URL] [--test TARGETDIR] -echo [-h] -echo. -echo --out (-o) Specify an additional output directory for installers -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo --build (-b) Incrementally build Python rather than rebuilding -echo --skip-build (-B) Do not build Python (just do the installers) -echo --skip-doc (-D) Do not build documentation -echo --pgo Specify PGO command for x64 installers -echo --skip-pgo Build x64 installers without using PGO -echo --skip-nuget Do not build Nuget packages -echo --skip-zip Do not build embeddable package -echo --download Specify the full download URL for MSIs -echo --test Specify the test directory to run the installer tests -echo -h Display this help information -echo. -echo If no architecture is specified, all architectures will be built. -echo If --test is not specified, the installer tests are not run. -echo. -echo For the --pgo option, any Python command line can be used, or 'default' to -echo use the default task (-m test --pgo). -echo. -echo The following substitutions will be applied to the download URL: -echo Variable Description Example -echo {version} version number 3.5.0 -echo {arch} architecture amd64, win32 -echo {releasename} release name a1, b2, rc3 (or blank for final) -echo {msi} MSI filename core.msi +@setlocal +@echo off + +rem This script is intended for building official releases of Python. +rem To use it to build alternative releases, you should clone this file +rem and modify the following three URIs. + +rem These two will ensure that your release can be installed +rem alongside an official Python release, by modifying the GUIDs used +rem for all components. +rem +rem The following substitutions will be applied to the release URI: +rem Variable Description Example +rem {arch} architecture amd64, win32 +set RELEASE_URI=http://www.python.org/{arch} + +rem This is the URL that will be used to download installation files. +rem The files available from the default URL *will* conflict with your +rem installer. Trust me, you don't want them, even if it seems like a +rem good idea. +rem +rem The following substitutions will be applied to the download URL: +rem Variable Description Example +rem {version} version number 3.5.0 +rem {arch} architecture amd64, win32 +rem {releasename} release name a1, b2, rc3 (or blank for final) +rem {msi} MSI filename core.msi +set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + +set D=%~dp0 +set PCBUILD=%D%..\..\PCBuild\ +set EXTERNALS=%D%..\..\externals\windows-installer\ + +set BUILDX86= +set BUILDX64= +set TARGET=Rebuild +set TESTTARGETDIR= +set PGO=-m test -q --pgo +set BUILDNUGET=1 +set BUILDZIP=1 + + +:CheckOpts +if "%1" EQU "-h" goto Help +if "%1" EQU "-c" (set CERTNAME=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--certificate" (set CERTNAME=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-o" (set OUTDIR=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--out" (set OUTDIR=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-D" (set SKIPDOC=1) && shift && goto CheckOpts +if "%1" EQU "--skip-doc" (set SKIPDOC=1) && shift && goto CheckOpts +if "%1" EQU "-B" (set SKIPBUILD=1) && shift && goto CheckOpts +if "%1" EQU "--skip-build" (set SKIPBUILD=1) && shift && goto CheckOpts +if "%1" EQU "--download" (set DOWNLOAD_URL=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--test" (set TESTTARGETDIR=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-b" (set TARGET=Build) && shift && goto CheckOpts +if "%1" EQU "--build" (set TARGET=Build) && shift && goto CheckOpts +if "%1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts +if "%1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts +if "%1" EQU "--pgo" (set PGO=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--skip-pgo" (set PGO=) && shift && goto CheckOpts +if "%1" EQU "--skip-nuget" (set BUILDNUGET=) && shift && goto CheckOpts +if "%1" EQU "--skip-zip" (set BUILDZIP=) && shift && goto CheckOpts + +if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 + +if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) + +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 + +call "%D%get_externals.bat" + +:builddoc +if "%SKIPBUILD%" EQU "1" goto skipdoc +if "%SKIPDOC%" EQU "1" goto skipdoc + +if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1 +if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1 + +call "%D%..\..\doc\make.bat" htmlhelp +if errorlevel 1 goto :eof +:skipdoc + +where dlltool /q && goto skipdlltoolsearch +set _DLLTOOL_PATH= +where /R "%EXTERNALS%\" dlltool > "%TEMP%\dlltool.loc" 2> nul && set /P _DLLTOOL_PATH= < "%TEMP%\dlltool.loc" & del "%TEMP%\dlltool.loc" +if not exist "%_DLLTOOL_PATH%" echo Cannot find binutils on PATH or in external && exit /B 1 +for %%f in (%_DLLTOOL_PATH%) do set PATH=%PATH%;%%~dpf +set _DLLTOOL_PATH= +:skipdlltoolsearch + +if defined BUILDX86 ( + call :build x86 + if errorlevel 1 exit /B +) + +if defined BUILDX64 ( + call :build x64 "%PGO%" + if errorlevel 1 exit /B +) + +if defined TESTTARGETDIR ( + call "%D%testrelease.bat" -t "%TESTTARGETDIR%" +) + +exit /B 0 + +:build +@setlocal +@echo off + +if "%1" EQU "x86" ( + set PGO= + set BUILD=%PCBUILD%win32\ + set BUILD_PLAT=Win32 + set OUTDIR_PLAT=win32 + set OBJDIR_PLAT=x86 +) else ( + set BUILD=%PCBUILD%amd64\ + set PGO=%~2 + set BUILD_PLAT=x64 + set OUTDIR_PLAT=amd64 + set OBJDIR_PLAT=x64 +) + +if exist "%BUILD%en-us" ( + echo Deleting %BUILD%en-us + rmdir /q/s "%BUILD%en-us" + if errorlevel 1 exit /B +) + +if exist "%D%obj\Debug_%OBJDIR_PLAT%" ( + echo Deleting "%D%obj\Debug_%OBJDIR_PLAT%" + rmdir /q/s "%D%obj\Debug_%OBJDIR_PLAT%" + if errorlevel 1 exit /B +) + +if exist "%D%obj\Release_%OBJDIR_PLAT%" ( + echo Deleting "%D%obj\Release_%OBJDIR_PLAT%" + rmdir /q/s "%D%obj\Release_%OBJDIR_PLAT%" + if errorlevel 1 exit /B +) + +if not "%CERTNAME%" EQU "" ( + set CERTOPTS="/p:SigningCertificate=%CERTNAME%" +) else ( + set CERTOPTS= +) +if not "%PGO%" EQU "" ( + set PGOOPTS=--pgo-job "%PGO%" +) else ( + set PGOOPTS= +) +if not "%SKIPBUILD%" EQU "1" ( + @echo call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% + @call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% + @if errorlevel 1 exit /B + @rem build.bat turns echo back on, so we disable it again + @echo off + + @echo call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% + @call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% + @if errorlevel 1 exit /B + @rem build.bat turns echo back on, so we disable it again + @echo off +) + +call "%PCBUILD%env.bat" +if "%OUTDIR_PLAT%" EQU "win32" ( + msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + if errorlevel 1 exit /B +) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( + msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + if errorlevel 1 exit /B +) + +set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% +msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true +if errorlevel 1 exit /B +msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false +if errorlevel 1 exit /B + +if defined BUILDZIP ( + msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" + if errorlevel 1 exit /B +) + +if defined BUILDNUGET ( + msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" + if errorlevel 1 exit /B +) + +if not "%OUTDIR%" EQU "" ( + mkdir "%OUTDIR%\%OUTDIR_PLAT%" + mkdir "%OUTDIR%\%OUTDIR_PLAT%\binaries" + mkdir "%OUTDIR%\%OUTDIR_PLAT%\symbols" + robocopy "%BUILD%en-us" "%OUTDIR%\%OUTDIR_PLAT%" /XF "*.wixpdb" + robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\binaries" *.exe *.dll *.pyd /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" + robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\symbols" *.pdb /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" +) + +exit /B 0 + +:Help +echo buildrelease.bat [--out DIR] [-x86] [-x64] [--certificate CERTNAME] [--build] [--pgo COMMAND] +echo [--skip-build] [--skip-doc] [--skip-nuget] [--skip-zip] [--skip-pgo] +echo [--download DOWNLOAD URL] [--test TARGETDIR] +echo [-h] +echo. +echo --out (-o) Specify an additional output directory for installers +echo -x86 Build x86 installers +echo -x64 Build x64 installers +echo --build (-b) Incrementally build Python rather than rebuilding +echo --skip-build (-B) Do not build Python (just do the installers) +echo --skip-doc (-D) Do not build documentation +echo --pgo Specify PGO command for x64 installers +echo --skip-pgo Build x64 installers without using PGO +echo --skip-nuget Do not build Nuget packages +echo --skip-zip Do not build embeddable package +echo --download Specify the full download URL for MSIs +echo --test Specify the test directory to run the installer tests +echo -h Display this help information +echo. +echo If no architecture is specified, all architectures will be built. +echo If --test is not specified, the installer tests are not run. +echo. +echo For the --pgo option, any Python command line can be used, or 'default' to +echo use the default task (-m test --pgo). +echo. +echo The following substitutions will be applied to the download URL: +echo Variable Description Example +echo {version} version number 3.5.0 +echo {arch} architecture amd64, win32 +echo {releasename} release name a1, b2, rc3 (or blank for final) +echo {msi} MSI filename core.msi diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index 4ead75e757233e..e1d74de6ac78f5 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -1,27 +1,27 @@ -@echo off -setlocal -rem Simple script to fetch source for external tools - -where /Q svn -if ERRORLEVEL 1 ( - echo.svn.exe must be on your PATH to get external tools. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 -) - -if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals" -pushd "%~dp0..\..\externals" - -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ - -if not exist "windows-installer\.svn" ( - echo.Checking out installer dependencies to %CD%\windows-installer - svn co %SVNROOT%windows-installer -) else ( - echo.Updating installer dependencies in %CD%\windows-installer - svn up windows-installer -) - -popd +@echo off +setlocal +rem Simple script to fetch source for external tools + +where /Q svn +if ERRORLEVEL 1 ( + echo.svn.exe must be on your PATH to get external tools. + echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the + echo.command line tools option. + popd + exit /b 1 +) + +if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals" +pushd "%~dp0..\..\externals" + +if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ + +if not exist "windows-installer\.svn" ( + echo.Checking out installer dependencies to %CD%\windows-installer + svn co %SVNROOT%windows-installer +) else ( + echo.Updating installer dependencies in %CD%\windows-installer + svn up windows-installer +) + +popd diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat index a989575ed6f032..96fdf5eb86e480 100644 --- a/Tools/msi/testrelease.bat +++ b/Tools/msi/testrelease.bat @@ -1,117 +1,117 @@ -@setlocal enableextensions -@echo off - -set D=%~dp0 -set PCBUILD=%D%..\..\PCBuild\ - -set TARGETDIR=%TEMP% -set TESTX86= -set TESTX64= -set TESTALLUSER= -set TESTPERUSER= - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-x86" (set TESTX86=1) && shift && goto CheckOpts -if "%1" EQU "-x64" (set TESTX64=1) && shift && goto CheckOpts -if "%1" EQU "-t" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--target" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-a" (set TESTALLUSER=1) && shift && goto CheckOpts -if "%1" EQU "--alluser" (set TESTALLUSER=1) && shift && goto CheckOpts -if "%1" EQU "-p" (set TESTPERUSER=1) && shift && goto CheckOpts -if "%1" EQU "--peruser" (set TESTPERUSER=1) && shift && goto CheckOpts - -if not defined TESTX86 if not defined TESTX64 (set TESTX86=1) && (set TESTX64=1) -if not defined TESTALLUSER if not defined TESTPERUSER (set TESTALLUSER=1) && (set TESTPERUSER=1) - - -if defined TESTX86 ( - for %%f in ("%PCBUILD%win32\en-us\*.exe") do ( - if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" - if errorlevel 1 exit /B - if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" - if errorlevel 1 exit /B - ) -) - -if defined TESTX64 ( - for %%f in ("%PCBUILD%amd64\en-us\*.exe") do ( - if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" - if errorlevel 1 exit /B - if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" - if errorlevel 1 exit /B - ) -) - -exit /B 0 - -:test -@setlocal -@echo on - -@if not exist "%~1" exit /B 1 - -@set EXE=%~1 -@if not "%EXE:embed=%"=="%EXE%" exit /B 0 - -@set EXITCODE=0 -@echo Installing %1 into %2 -"%~1" /passive /log "%~2\install\log.txt" TargetDir="%~2\Python" Include_debug=1 Include_symbols=1 %~3 - -@if not errorlevel 1 ( - @echo Printing version - "%~2\Python\python.exe" -c "import sys; print(sys.version)" > "%~2\version.txt" 2>&1 -) - -@if not errorlevel 1 ( - @echo Capturing Start Menu - @dir /s/b "%PROGRAMDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" > "%~2\startmenu.txt" 2>&1 - @dir /s/b "%APPDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" >> "%~2\startmenu.txt" 2>&1 - - @echo Capturing registry - @for /F "usebackq" %%f in (`reg query HKCR /s /f python /k`) do @( - echo %%f >> "%~2\hkcr.txt" - reg query "%%f" /s >> "%~2\hkcr.txt" 2>&1 - ) - @reg query HKCU\Software\Python /s > "%~2\hkcu.txt" 2>&1 - @reg query HKLM\Software\Python /reg:32 /s > "%~2\hklm.txt" 2>&1 - @reg query HKLM\Software\Python /reg:64 /s >> "%~2\hklm.txt" 2>&1 - cmd /k exit 0 -) - -@if not errorlevel 1 ( - @echo Installing package - "%~2\Python\python.exe" -m pip install "azure<0.10" > "%~2\pip.txt" 2>&1 - @if not errorlevel 1 ( - "%~2\Python\python.exe" -m pip uninstall -y azure python-dateutil six >> "%~2\pip.txt" 2>&1 - ) -) -@if not errorlevel 1 ( - @echo Testing Tcl/tk - @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 - "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 - @set TCL_LIBRARY= -) - -@set EXITCODE=%ERRORLEVEL% - -@echo Result was %EXITCODE% -@echo Removing %1 -"%~1" /passive /uninstall /log "%~2\uninstall\log.txt" - -@echo off -exit /B %EXITCODE% - -:Help -echo testrelease.bat [--target TARGET] [-x86] [-x64] [--alluser] [--peruser] [-h] -echo. -echo --target (-t) Specify the target directory for installs and logs -echo -x86 Run tests for x86 installers -echo -x64 Run tests for x64 installers -echo --alluser (-a) Run tests for all-user installs (requires Administrator) -echo --peruser (-p) Run tests for per-user installs -echo -h Display this help information -echo. -echo If no test architecture is specified, all architectures will be tested. -echo If no install type is selected, all install types will be tested. -echo. +@setlocal enableextensions +@echo off + +set D=%~dp0 +set PCBUILD=%D%..\..\PCBuild\ + +set TARGETDIR=%TEMP% +set TESTX86= +set TESTX64= +set TESTALLUSER= +set TESTPERUSER= + +:CheckOpts +if "%1" EQU "-h" goto Help +if "%1" EQU "-x86" (set TESTX86=1) && shift && goto CheckOpts +if "%1" EQU "-x64" (set TESTX64=1) && shift && goto CheckOpts +if "%1" EQU "-t" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--target" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-a" (set TESTALLUSER=1) && shift && goto CheckOpts +if "%1" EQU "--alluser" (set TESTALLUSER=1) && shift && goto CheckOpts +if "%1" EQU "-p" (set TESTPERUSER=1) && shift && goto CheckOpts +if "%1" EQU "--peruser" (set TESTPERUSER=1) && shift && goto CheckOpts + +if not defined TESTX86 if not defined TESTX64 (set TESTX86=1) && (set TESTX64=1) +if not defined TESTALLUSER if not defined TESTPERUSER (set TESTALLUSER=1) && (set TESTPERUSER=1) + + +if defined TESTX86 ( + for %%f in ("%PCBUILD%win32\en-us\*.exe") do ( + if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" + if errorlevel 1 exit /B + if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" + if errorlevel 1 exit /B + ) +) + +if defined TESTX64 ( + for %%f in ("%PCBUILD%amd64\en-us\*.exe") do ( + if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" + if errorlevel 1 exit /B + if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" + if errorlevel 1 exit /B + ) +) + +exit /B 0 + +:test +@setlocal +@echo on + +@if not exist "%~1" exit /B 1 + +@set EXE=%~1 +@if not "%EXE:embed=%"=="%EXE%" exit /B 0 + +@set EXITCODE=0 +@echo Installing %1 into %2 +"%~1" /passive /log "%~2\install\log.txt" TargetDir="%~2\Python" Include_debug=1 Include_symbols=1 %~3 + +@if not errorlevel 1 ( + @echo Printing version + "%~2\Python\python.exe" -c "import sys; print(sys.version)" > "%~2\version.txt" 2>&1 +) + +@if not errorlevel 1 ( + @echo Capturing Start Menu + @dir /s/b "%PROGRAMDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" > "%~2\startmenu.txt" 2>&1 + @dir /s/b "%APPDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" >> "%~2\startmenu.txt" 2>&1 + + @echo Capturing registry + @for /F "usebackq" %%f in (`reg query HKCR /s /f python /k`) do @( + echo %%f >> "%~2\hkcr.txt" + reg query "%%f" /s >> "%~2\hkcr.txt" 2>&1 + ) + @reg query HKCU\Software\Python /s > "%~2\hkcu.txt" 2>&1 + @reg query HKLM\Software\Python /reg:32 /s > "%~2\hklm.txt" 2>&1 + @reg query HKLM\Software\Python /reg:64 /s >> "%~2\hklm.txt" 2>&1 + cmd /k exit 0 +) + +@if not errorlevel 1 ( + @echo Installing package + "%~2\Python\python.exe" -m pip install "azure<0.10" > "%~2\pip.txt" 2>&1 + @if not errorlevel 1 ( + "%~2\Python\python.exe" -m pip uninstall -y azure python-dateutil six >> "%~2\pip.txt" 2>&1 + ) +) +@if not errorlevel 1 ( + @echo Testing Tcl/tk + @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 + "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 + @set TCL_LIBRARY= +) + +@set EXITCODE=%ERRORLEVEL% + +@echo Result was %EXITCODE% +@echo Removing %1 +"%~1" /passive /uninstall /log "%~2\uninstall\log.txt" + +@echo off +exit /B %EXITCODE% + +:Help +echo testrelease.bat [--target TARGET] [-x86] [-x64] [--alluser] [--peruser] [-h] +echo. +echo --target (-t) Specify the target directory for installs and logs +echo -x86 Run tests for x86 installers +echo -x64 Run tests for x64 installers +echo --alluser (-a) Run tests for all-user installs (requires Administrator) +echo --peruser (-p) Run tests for per-user installs +echo -h Display this help information +echo. +echo If no test architecture is specified, all architectures will be tested. +echo If no install type is selected, all install types will be tested. +echo. diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 670836be8c1c8b..0873d55589f0db 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -1,76 +1,76 @@ -@setlocal -@echo off - -set D=%~dp0 -set PCBUILD=%D%..\..\PCBuild\ - -set HOST= -set USER= -set TARGET= -set DRYRUN=false -set NOGPG= -set PURGE_OPTION=/p:Purge=true -set NOTEST= - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-o" (set HOST=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--host" (set HOST=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-u" (set USER=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts -if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts -if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts -if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts -if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 - -if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles(x86)%" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK echo Cannot locate plink.exe & exit /B 1 -echo Found plink.exe at %PLINK% - -if not defined PSCP where pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles(x86)%\PuTTY" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles(x86)%" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP echo Cannot locate pscp.exe & exit /B 1 -echo Found pscp.exe at %PSCP% - -if defined NOGPG ( - set GPG= - echo Skipping GPG signature generation because of --skip-gpg -) else ( - if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" - if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" - if not defined GPG echo Cannot locate gpg2.exe. Signatures will not be uploaded & pause - echo Found gpg2.exe at %GPG% -) - -call "%PCBUILD%env.bat" > nul 2> nul -pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% -if not defined NOTEST ( - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 -) -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false -popd -exit /B 0 - -:Help -echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] -echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo --skip-gpg Does not generate GPG signatures before uploading -echo --skip-purge Does not perform CDN purge after uploading -echo --skip-test (-T) Does not perform post-upload tests -echo -h Display this help information -echo. +@setlocal +@echo off + +set D=%~dp0 +set PCBUILD=%D%..\..\PCBuild\ + +set HOST= +set USER= +set TARGET= +set DRYRUN=false +set NOGPG= +set PURGE_OPTION=/p:Purge=true +set NOTEST= + +:CheckOpts +if "%1" EQU "-h" goto Help +if "%1" EQU "-o" (set HOST=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--host" (set HOST=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-u" (set USER=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts +if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts +if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts +if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts +if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts +if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 + +if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" +if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" +if not defined PLINK where /R "%ProgramFiles(x86)%" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" +if not defined PLINK echo Cannot locate plink.exe & exit /B 1 +echo Found plink.exe at %PLINK% + +if not defined PSCP where pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" +if not defined PSCP where /R "%ProgramFiles(x86)%\PuTTY" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" +if not defined PSCP where /R "%ProgramFiles(x86)%" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" +if not defined PSCP echo Cannot locate pscp.exe & exit /B 1 +echo Found pscp.exe at %PSCP% + +if defined NOGPG ( + set GPG= + echo Skipping GPG signature generation because of --skip-gpg +) else ( + if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" + if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" + if not defined GPG echo Cannot locate gpg2.exe. Signatures will not be uploaded & pause + echo Found gpg2.exe at %GPG% +) + +call "%PCBUILD%env.bat" > nul 2> nul +pushd "%D%" +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOTEST ( + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 +) +msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +popd +exit /B 0 + +:Help +echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] +echo. +echo --host (-o) Specify the upload host (required) +echo --user (-u) Specify the user on the host (required) +echo --target (-t) Specify the target directory on the host +echo --dry-run Display commands and filenames without executing them +echo --skip-gpg Does not generate GPG signatures before uploading +echo --skip-purge Does not perform CDN purge after uploading +echo --skip-test (-T) Does not perform post-upload tests +echo -h Display this help information +echo. diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 120b38cb38ce61..5068c17244456e 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -1,55 +1,55 @@ -@echo off -setlocal -set D=%~dp0 -set PCBUILD=%D%..\..\PCBuild\ - -set BUILDX86= -set BUILDX64= -set REBUILD= -set OUTPUT= -set PACKAGES= - -:CheckOpts -if "%~1" EQU "-h" goto Help -if "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts -if "%~1" EQU "-o" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts -if "%~1" EQU "--out" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts -if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckOpts - -if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) - -call "%D%..\msi\get_externals.bat" -call "%PCBUILD%env.bat" x86 - -if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" - -if defined BUILDX86 ( - if defined REBUILD ( call "%PCBUILD%build.bat" -e -r - ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e - if errorlevel 1 goto :eof - - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% - if errorlevel 1 goto :eof -) - -if defined BUILDX64 ( - if defined REBUILD ( call "%PCBUILD%build.bat" -p x64 -e -r - ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e - if errorlevel 1 goto :eof - - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% - if errorlevel 1 goto :eof -) - -exit /B 0 - -:Help -echo build.bat [-x86] [-x64] [--out DIR] [-r] [-h] -echo. -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo -r Rebuild rather than incremental build -echo --out [DIR] Override output directory -echo -h Show usage +@echo off +setlocal +set D=%~dp0 +set PCBUILD=%D%..\..\PCBuild\ + +set BUILDX86= +set BUILDX64= +set REBUILD= +set OUTPUT= +set PACKAGES= + +:CheckOpts +if "%~1" EQU "-h" goto Help +if "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts +if "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts +if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts +if "%~1" EQU "-o" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts +if "%~1" EQU "--out" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts +if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckOpts + +if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) + +call "%D%..\msi\get_externals.bat" +call "%PCBUILD%env.bat" x86 + +if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" + +if defined BUILDX86 ( + if defined REBUILD ( call "%PCBUILD%build.bat" -e -r + ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e + if errorlevel 1 goto :eof + + msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% + if errorlevel 1 goto :eof +) + +if defined BUILDX64 ( + if defined REBUILD ( call "%PCBUILD%build.bat" -p x64 -e -r + ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e + if errorlevel 1 goto :eof + + msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% + if errorlevel 1 goto :eof +) + +exit /B 0 + +:Help +echo build.bat [-x86] [-x64] [--out DIR] [-r] [-h] +echo. +echo -x86 Build x86 installers +echo -x64 Build x64 installers +echo -r Rebuild rather than incremental build +echo --out [DIR] Override output directory +echo -h Show usage diff --git a/Tools/unicode/genwincodecs.bat b/Tools/unicode/genwincodecs.bat index ad45c6c4438014..43cab0d6f8e71a 100644 --- a/Tools/unicode/genwincodecs.bat +++ b/Tools/unicode/genwincodecs.bat @@ -1,7 +1,7 @@ -@rem Recreate some python charmap codecs from the Windows function -@rem MultiByteToWideChar. - -@cd /d %~dp0 -@mkdir build -@rem Arabic DOS code page -c:\python30\python genwincodec.py 720 > build/cp720.py +@rem Recreate some python charmap codecs from the Windows function +@rem MultiByteToWideChar. + +@cd /d %~dp0 +@mkdir build +@rem Arabic DOS code page +c:\python30\python genwincodec.py 720 > build/cp720.py From 8d15b19be667e30f8d36aefb680859bdf23866cf Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 10 Jun 2017 14:41:00 -0700 Subject: [PATCH 0399/1345] bpo-28556: Updates to typing module (GH-2076) (GH-2087) This PR contains two updates to typing module: - Support ContextManager on all versions (original PR by Jelle Zijlstra). - Add generic AsyncContextManager.. (cherry picked from commit 29fda8db16e0edab92841277fa223f844f5a92cc) --- Lib/test/test_typing.py | 39 ++++++++++++++++++++++++-- Lib/typing.py | 61 +++++++++++++++++++++++++++++++++++++++-- Misc/NEWS | 4 +++ 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b3cabda394497e..6d0b2659db22b7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1552,6 +1552,12 @@ def __anext__(self) -> T_a: return data else: raise StopAsyncIteration + +class ACM: + async def __aenter__(self) -> int: + return 42 + async def __aexit__(self, etype, eval, tb): + return None """ if ASYNCIO: @@ -1562,12 +1568,13 @@ def __anext__(self) -> T_a: else: # fake names for the sake of static analysis asyncio = None - AwaitableWrapper = AsyncIteratorWrapper = object + AwaitableWrapper = AsyncIteratorWrapper = ACM = object PY36 = sys.version_info[:2] >= (3, 6) PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 +from typing import AsyncContextManager class A: y: float @@ -1604,6 +1611,16 @@ def __str__(self): return f'{self.x} -> {self.y}' def __add__(self, other): return 0 + +async def g_with(am: AsyncContextManager[int]): + x: int + async with am as x: + return x + +try: + g_with(ACM()).send(None) +except StopIteration as e: + assert e.args[0] == 42 """ if PY36: @@ -2156,8 +2173,6 @@ class B: ... class OtherABCTests(BaseTestCase): - @skipUnless(hasattr(typing, 'ContextManager'), - 'requires typing.ContextManager') def test_contextmanager(self): @contextlib.contextmanager def manager(): @@ -2167,6 +2182,24 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + @skipUnless(ASYNCIO, 'Python 3.5 required') + def test_async_contextmanager(self): + class NotACM: + pass + self.assertIsInstance(ACM(), typing.AsyncContextManager) + self.assertNotIsInstance(NotACM(), typing.AsyncContextManager) + @contextlib.contextmanager + def manager(): + yield 42 + + cm = manager() + self.assertNotIsInstance(cm, typing.AsyncContextManager) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + with self.assertRaises(TypeError): + isinstance(42, typing.AsyncContextManager[int]) + with self.assertRaises(TypeError): + typing.AsyncContextManager[int, str] + class TypeTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 645bc6f8ae0edd..c487afcb5b42f8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,8 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +if sys.version_info[:2] >= (3, 6): + import _collections_abc # Needed for private function _check_methods # noqa try: from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType except ImportError: @@ -37,6 +39,7 @@ # for 'Generic' and ABCs below. 'ByteString', 'Container', + 'ContextManager', 'Hashable', 'ItemsView', 'Iterable', @@ -57,8 +60,8 @@ # AsyncIterable, # Coroutine, # Collection, - # ContextManager, # AsyncGenerator, + # AsyncContextManager # Structural checks, a.k.a. protocols. 'Reversible', @@ -1949,7 +1952,61 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): if hasattr(contextlib, 'AbstractContextManager'): class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): __slots__ = () - __all__.append('ContextManager') +else: + class ContextManager(Generic[T_co]): + __slots__ = () + + def __enter__(self): + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is ContextManager: + # In Python 3.6+, it is possible to set a method to None to + # explicitly indicate that the class does not implement an ABC + # (https://bugs.python.org/issue25958), but we do not support + # that pattern here because this fallback class is only used + # in Python 3.5 and earlier. + if (any("__enter__" in B.__dict__ for B in C.__mro__) and + any("__exit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + + +if hasattr(contextlib, 'AbstractAsyncContextManager'): + class AsyncContextManager(Generic[T_co], + extra=contextlib.AbstractAsyncContextManager): + __slots__ = () + + __all__.append('AsyncContextManager') +elif sys.version_info[:2] >= (3, 5): + exec(""" +class AsyncContextManager(Generic[T_co]): + __slots__ = () + + async def __aenter__(self): + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncContextManager: + if sys.version_info[:2] >= (3, 6): + return _collections_abc._check_methods(C, "__aenter__", "__aexit__") + if (any("__aenter__" in B.__dict__ for B in C.__mro__) and + any("__aexit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + +__all__.append('AsyncContextManager') +""") class Dict(dict, MutableMapping[KT, VT], extra=dict): diff --git a/Misc/NEWS b/Misc/NEWS index 9d4f5d432a1ec4..b7fa701610bc80 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,10 @@ Core and Builtins Library ------- +- bpo-28556: Updates to typing module: Add generic AsyncContextManager, add + support for ContextManager on all versions. Original PRs by Jelle Zijlstra + and Ivan Levkivskyi + - bpo-29406: asyncio SSL contexts leak sockets after calling close with certain servers. Patch by Nikolay Kim From 82e606785e797be6067b1e485d8b63b58fa7ad19 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sat, 10 Jun 2017 23:15:08 -0400 Subject: [PATCH 0400/1345] [3.6] Add IDLE news items (GH-2090) (#2096) (cherry-pick from c94c41c) --- Misc/NEWS | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index b7fa701610bc80..d31ae7ee9b6ffe 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -199,6 +199,17 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +IDLE +---- + +- bpo-30290: Help-about: use pep8 names and add tests. + Increase coverage to 100%. + Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. + +- bpo-30303: Add _utest option to textview; add new tests. + Increase coverage to 100%. + Patches by Louie Lu and Terry Jan Reedy. + C API ----- @@ -492,6 +503,14 @@ Library now when the grp module cannot be imported, as for example on Android platforms. +IDLE +---- + +- Issue #29071: IDLE colors f-string prefixes (but not invalid ur prefixes). + +- Issue #28572: Add 10% to coverage of IDLE's test_configdialog. + Update and augment description of the configuration system. + Windows ------- From 556cddba770fc9f85f133e52993dc618caa347c0 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sat, 10 Jun 2017 23:48:00 -0400 Subject: [PATCH 0401/1345] [3.6] Update idlelib NEWS for 3.6 (GH-2089) (#2097) (cherry-picked from 503bc63) --- Lib/idlelib/NEWS.txt | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index a3fc5012fb3014..ee2cbacbea646f 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,6 +1,29 @@ -What's New in IDLE 3.6.0? -=========================== -*Release date: 2016-12-16?* +What's New in IDLE 3.6.2 +Released on 2017-06-23? +======================== + +bpo-30290: Help-about: use pep8 names and add tests. +Increase coverage to 100%. +Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. + +bpo-30303: Add _utest option to textview; add new tests. +Increase coverage to 100%. +Patches by Louie Lu and Terry Jan Reedy. + + +What's New in IDLE 3.6.1 +Released on 2017-03-21 +======================== + +Issue #29071: IDLE colors f-string prefixes (but not invalid ur prefixes). + +Issue #28572: Add 10% to coverage of IDLE's test_configdialog. +Update and augment description of the configuration system. + + +What's New in IDLE 3.6.0 +Released on 2016-12-23 +======================== - Issue #15308: Add 'interrupt execution' (^C) to Shell menu. Patch by Roger Serwy, updated by Bayard Randel. From c0ef607c52e58e94693fbedb27f2813bc3fed8fa Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 00:34:20 -0400 Subject: [PATCH 0402/1345] [3.6] bpo-30166: backport pyshell changes (GH 1293) (#2098) (cherry-pick IDLE pyshell change from 7e4db2f) --- Lib/idlelib/pyshell.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index dd6c997abc587b..5b0e5b267642ab 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -18,12 +18,10 @@ raise SystemExit(1) from code import InteractiveInterpreter -import getopt -import io import linecache import os import os.path -from platform import python_version, system +from platform import python_version import re import socket import subprocess @@ -32,14 +30,12 @@ import tokenize import warnings -from idlelib import testing # bool value from idlelib.colorizer import ColorDelegator from idlelib.config import idleConf from idlelib import debugger from idlelib import debugger_r from idlelib.editor import EditorWindow, fixwordbreaks from idlelib.filelist import FileList -from idlelib import macosx from idlelib.outwin import OutputWindow from idlelib import rpc from idlelib.run import idle_formatwarning, PseudoInputFile, PseudoOutputFile @@ -1372,6 +1368,11 @@ def fix_x11_paste(root): """ def main(): + import getopt + from platform import system + from idlelib import testing # bool value + from idlelib import macosx + global flist, root, use_subprocess capture_warnings(True) From ccccf3156f8f5cfb820c7deac05beba9f51ec51c Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 04:16:55 -0400 Subject: [PATCH 0403/1345] bpo-30495: IDLE: Modernize textview.py with docstrings and PEP8 names (#1839) (#2102) Patch by Cheryl Sabella. (cherry picked from commit 0aa0a06e8b719533aefd175a5716f1698f474052) --- Lib/idlelib/idle_test/test_help_about.py | 11 ++-- Lib/idlelib/idle_test/test_textview.py | 20 +++---- Lib/idlelib/textview.py | 73 +++++++++++++++--------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index b98405df52a0c7..15d1b6b42c797a 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -45,10 +45,10 @@ def test_printer_buttons(self): button.invoke() self.assertEqual( printer._Printer__lines[0], - dialog._current_textview.textView.get('1.0', '1.end')) + dialog._current_textview.text.get('1.0', '1.end')) self.assertEqual( printer._Printer__lines[1], - dialog._current_textview.textView.get('2.0', '2.end')) + dialog._current_textview.text.get('2.0', '2.end')) dialog._current_textview.destroy() def test_file_buttons(self): @@ -64,10 +64,11 @@ def test_file_buttons(self): with open(fn) as f: self.assertEqual( f.readline().strip(), - dialog._current_textview.textView.get('1.0', '1.end')) + dialog._current_textview.text.get('1.0', '1.end')) f.readline() - self.assertEqual(f.readline().strip(), - dialog._current_textview.textView.get('3.0', '3.end')) + self.assertEqual( + f.readline().strip(), + dialog._current_textview.text.get('3.0', '3.end')) dialog._current_textview.destroy() diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index 57b886aba11f31..7a2f7e460f2c7e 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -41,7 +41,7 @@ class TV(tv.TextViewer): # Used in TextViewTest. # Call wrapper class with mock wait_window. class TextViewTest(unittest.TestCase): - + def setUp(self): TV.transient.__init__() TV.grab_set.__init__() @@ -52,19 +52,19 @@ def test_init_modal(self): self.assertTrue(TV.transient.called) self.assertTrue(TV.grab_set.called) self.assertTrue(TV.wait_window.called) - view.Ok() + view.ok() def test_init_nonmodal(self): view = TV(root, 'Title', 'test text', modal=False) self.assertFalse(TV.transient.called) self.assertFalse(TV.grab_set.called) self.assertFalse(TV.wait_window.called) - view.Ok() + view.ok() def test_ok(self): view = TV(root, 'Title', 'test text', modal=False) view.destroy = Func() - view.Ok() + view.ok() self.assertTrue(view.destroy.called) del view.destroy # Unmask real function. view.destroy() @@ -86,13 +86,13 @@ def tearDownClass(cls): def test_view_text(self): view = tv.view_text(root, 'Title', 'test text', modal=False) self.assertIsInstance(view, tv.TextViewer) - view.Ok() + view.ok() def test_view_file(self): view = tv.view_file(root, 'Title', __file__, modal=False) self.assertIsInstance(view, tv.TextViewer) - self.assertIn('Test', view.textView.get('1.0', '1.end')) - view.Ok() + self.assertIn('Test', view.text.get('1.0', '1.end')) + view.ok() def test_bad_file(self): # Mock showerror will be used; view_file will return None. @@ -131,7 +131,7 @@ def _command(): self.assertEqual(self.called, True) self.assertEqual(self.view.title(), 'TITLE_TEXT') - self.assertEqual(self.view.textView.get('1.0', '1.end'), 'COMMAND') + self.assertEqual(self.view.text.get('1.0', '1.end'), 'COMMAND') def test_view_file_bind_with_button(self): def _command(): @@ -144,10 +144,10 @@ def _command(): self.assertEqual(self.called, True) self.assertEqual(self.view.title(), 'TITLE_FILE') with open(__file__) as f: - self.assertEqual(self.view.textView.get('1.0', '1.end'), + self.assertEqual(self.view.text.get('1.0', '1.end'), f.readline().strip()) f.readline() - self.assertEqual(self.view.textView.get('3.0', '3.end'), + self.assertEqual(self.view.text.get('3.0', '3.end'), f.readline().strip()) diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index dc1a7bfc9eff38..ab653a91413252 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -1,7 +1,9 @@ """Simple text browser for IDLE """ -from tkinter import * +from tkinter import Toplevel, Frame, Button, Text +from tkinter import DISABLED, SUNKEN, VERTICAL, WORD +from tkinter import RIGHT, LEFT, TOP, BOTTOM, BOTH, X, Y from tkinter.ttk import Scrollbar from tkinter.messagebox import showerror @@ -16,6 +18,9 @@ def __init__(self, parent, title, text, modal=True, If modal is left True, users cannot interact with other windows until the textview window is closed. + parent - parent of this dialog + title - string which is title of popup dialog + text - text to display in dialog _htest - bool; change box location when running htest. _utest - bool; don't wait_window when running unittest. """ @@ -29,16 +34,16 @@ def __init__(self, parent, title, text, modal=True, self.bg = '#ffffff' self.fg = '#000000' - self.CreateWidgets() + self.create_widgets() self.title(title) - self.protocol("WM_DELETE_WINDOW", self.Ok) + self.protocol("WM_DELETE_WINDOW", self.ok) self.parent = parent - self.textView.focus_set() + self.text.focus_set() # Bind keys for closing this dialog. - self.bind('',self.Ok) - self.bind('',self.Ok) - self.textView.insert(0.0, text) - self.textView.config(state=DISABLED) + self.bind('', self.ok) + self.bind('', self.ok) + self.text.insert(0.0, text) + self.text.config(state=DISABLED) if modal: self.transient(parent) @@ -46,34 +51,48 @@ def __init__(self, parent, title, text, modal=True, if not _utest: self.wait_window() - def CreateWidgets(self): + def create_widgets(self): "Create Frame with Text (with vertical Scrollbar) and Button." - frameText = Frame(self, relief=SUNKEN, height=700) - frameButtons = Frame(self) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok, takefocus=FALSE) - self.scrollbarView = Scrollbar(frameText, orient=VERTICAL, - takefocus=FALSE) - self.textView = Text(frameText, wrap=WORD, highlightthickness=0, + frame = Frame(self, relief=SUNKEN, height=700) + frame_buttons = Frame(self) + self.button_ok = Button(frame_buttons, text='Close', + command=self.ok, takefocus=False) + self.scrollbar = Scrollbar(frame, orient=VERTICAL, takefocus=False) + self.text = Text(frame, wrap=WORD, highlightthickness=0, fg=self.fg, bg=self.bg) - self.scrollbarView.config(command=self.textView.yview) - self.textView.config(yscrollcommand=self.scrollbarView.set) - self.buttonOk.pack() - self.scrollbarView.pack(side=RIGHT,fill=Y) - self.textView.pack(side=LEFT,expand=TRUE,fill=BOTH) - frameButtons.pack(side=BOTTOM,fill=X) - frameText.pack(side=TOP,expand=TRUE,fill=BOTH) - - def Ok(self, event=None): + self.scrollbar.config(command=self.text.yview) + self.text.config(yscrollcommand=self.scrollbar.set) + + self.button_ok.pack() + self.scrollbar.pack(side=RIGHT, fill=Y) + self.text.pack(side=LEFT, expand=True, fill=BOTH) + frame_buttons.pack(side=BOTTOM, fill=X) + frame.pack(side=TOP, expand=True, fill=BOTH) + + def ok(self, event=None): + """Dismiss text viewer dialog.""" self.destroy() def view_text(parent, title, text, modal=True, _utest=False): - "Display text in a TextViewer." + """Create TextViewer for given text. + + parent - parent of this dialog + title - string which is the title of popup dialog + text - text to display in this dialog + modal - controls if users can interact with other windows while this + dialog is displayed + _utest - bool; controls wait_window on unittest + """ return TextViewer(parent, title, text, modal, _utest=_utest) + def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): - "Display file in a TextViever or show error message." + """Create TextViewer for text in filename. + + Return error message if file cannot be read. Otherwise calls view_text + with contents of the file. + """ try: with open(filename, 'r', encoding=encoding) as file: contents = file.read() From d755d19ac4f50be2b11a6efb0f0396290c8d8263 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 04:45:35 -0400 Subject: [PATCH 0404/1345] [3.6] Remove unused imports (#2104) Part of patch by Serhiy Strochaka, 2016-12-16 (no bpo issue) (cherry-pick of 70d28a184c42d107cc8c69a95aa52a4469e7929c) --- Lib/idlelib/idle_test/test_macosx.py | 1 - Lib/idlelib/idle_test/test_tree.py | 1 - Lib/idlelib/macosx.py | 1 - Lib/idlelib/stackviewer.py | 1 - 4 files changed, 4 deletions(-) diff --git a/Lib/idlelib/idle_test/test_macosx.py b/Lib/idlelib/idle_test/test_macosx.py index fae75d8a497320..775697b278d30d 100644 --- a/Lib/idlelib/idle_test/test_macosx.py +++ b/Lib/idlelib/idle_test/test_macosx.py @@ -4,7 +4,6 @@ ''' from idlelib import macosx from test.support import requires -import sys import tkinter as tk import unittest import unittest.mock as mock diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py index 09ba9641af552b..bb597d87ffd104 100644 --- a/Lib/idlelib/idle_test/test_tree.py +++ b/Lib/idlelib/idle_test/test_tree.py @@ -5,7 +5,6 @@ from idlelib import tree from test.support import requires requires('gui') -import os import unittest from tkinter import Tk diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index c225dd9e1a115d..d4566cd815a2dd 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -2,7 +2,6 @@ A number of functions that enhance IDLE on Mac OSX. """ from sys import platform # Used in _init_tk_type, changed by test. -import warnings import tkinter diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 0698def5d4c7f2..400fa632a098cf 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -1,6 +1,5 @@ import linecache import os -import re import sys import tkinter as tk From c6696feb09a8cf13e13e42ed9cc0b8c2e8795800 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 05:53:46 -0400 Subject: [PATCH 0405/1345] [3.6] bpo-29919 Remove unused imports from idlelib [GH-137] (#2105) Part of patch by Victor Stinner. (cherry-pick from d6debb24e06152a827769b0cac24c47deccdeac1) --- Lib/idlelib/config.py | 2 +- Lib/idlelib/editor.py | 1 - Lib/idlelib/idle_test/test_config.py | 2 -- Lib/idlelib/idle_test/test_config_key.py | 2 +- Lib/idlelib/idle_test/test_editor.py | 2 -- Lib/idlelib/idle_test/test_searchbase.py | 2 +- Lib/idlelib/query.py | 2 +- 7 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 358bee4803b0c1..6a9fc61c2b59d7 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -29,7 +29,7 @@ import os import sys -from tkinter.font import Font, nametofont +from tkinter.font import Font class InvalidConfigType(Exception): pass class InvalidConfigSet(Exception): pass diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index ae475cb9f9a6a5..ab4f1a37c168c1 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -26,7 +26,6 @@ from idlelib import query from idlelib import replace from idlelib import search -from idlelib import textview from idlelib import windows # The default tab setting for a Text widget, in average-width characters. diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index a3fa1a341a1642..e678cc6f332670 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -3,9 +3,7 @@ Much is tested by opening config dialog live or in test_configdialog. Coverage: 27% ''' -from sys import modules from test.support import captured_stderr -from tkinter import Tk import unittest from idlelib import config diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index ee3f2c835c98b7..8a24c9632b7224 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -6,7 +6,7 @@ from test.support import requires requires('gui') import unittest -from tkinter import Tk, Text +from tkinter import Tk class GetKeysTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index e9d29d45b82345..64a2a88b7e3765 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -1,7 +1,5 @@ import unittest -from tkinter import Tk, Text from idlelib.editor import EditorWindow -from test.support import requires class Editor_func_test(unittest.TestCase): def test_filename_to_unicode(self): diff --git a/Lib/idlelib/idle_test/test_searchbase.py b/Lib/idlelib/idle_test/test_searchbase.py index d769fa2fc2f0de..27b02fbe54602c 100644 --- a/Lib/idlelib/idle_test/test_searchbase.py +++ b/Lib/idlelib/idle_test/test_searchbase.py @@ -5,7 +5,7 @@ ''' import unittest from test.support import requires -from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar +from tkinter import Tk, Frame ##, BooleanVar, StringVar from idlelib import searchengine as se from idlelib import searchbase as sdb from idlelib.idle_test.mock_idle import Func diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index 3b1f1e25be7f86..593506383c41ab 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -24,7 +24,7 @@ import os from sys import executable, platform # Platform is set for one test. -from tkinter import Toplevel, StringVar, W, E, N, S +from tkinter import Toplevel, StringVar, W, E, S from tkinter.ttk import Frame, Button, Entry, Label from tkinter import filedialog from tkinter.font import Font From a13225e209cfa5f7b458dbcbac19dc4df26feb95 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 06:12:31 -0400 Subject: [PATCH 0406/1345] [3.6]bpo-30144: change idlelib abc import [GH-1263] (#2106) Part of patch by Serhiy Storchaka. (cherry-pick from 2e576f5) --- Lib/idlelib/pyparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py index 6739dfd1a07a6b..536b2d7f5fef73 100644 --- a/Lib/idlelib/pyparse.py +++ b/Lib/idlelib/pyparse.py @@ -1,4 +1,4 @@ -from collections import Mapping +from collections.abc import Mapping import re import sys From 59422a29ee3a95866c4f7e037bdfffd5768afddd Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 06:26:25 -0400 Subject: [PATCH 0407/1345] [3.6]bpo-30022: idlelib.run IOError -> OSError [GH-1051] (#2107) Part of patch by Serhiy Storchaka. (cherry-pick from 55fe1ae9708d81b902b6fe8f6590e2a24b1bd4b0) --- Lib/idlelib/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index afa9744a346c92..12c339f1058b72 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -52,7 +52,7 @@ def idle_showwarning_subproc( try: file.write(idle_formatwarning( message, category, filename, lineno, line)) - except IOError: + except OSError: pass # the file (probably stderr) is invalid - this warning gets lost. _warnings_showwarning = None From 176f2ebdad93f20876c08efabd364a0e6c86de14 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 11 Jun 2017 14:00:14 +0000 Subject: [PATCH 0408/1345] bpo-30508: Don't log exceptions if Task/Future "cancel()" method was called. (#2109) --- Lib/asyncio/futures.py | 1 + Lib/asyncio/tasks.py | 1 + Lib/test/test_asyncio/test_futures.py | 8 ++++++++ Lib/test/test_asyncio/test_tasks.py | 19 +++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 18 +++++++++++++++++- 6 files changed, 49 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d11d289307eaf3..cff9590e4ead5f 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -191,6 +191,7 @@ def cancel(self): change the future's state to cancelled, schedule the callbacks and return True. """ + self._log_traceback = False if self._state != _PENDING: return False self._state = _CANCELLED diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index d7867d128a8afe..9fe2a2fabf076a 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -148,6 +148,7 @@ def cancel(self): terminates with a CancelledError exception (even if cancel() was not called). """ + self._log_traceback = False if self.done(): return False if self._fut_waiter is not None: diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 99336f86ab824e..5d4b2d2aa0efc0 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -318,6 +318,14 @@ def test_tb_logger_abandoned(self, m_log): del fut self.assertFalse(m_log.error.called) + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(Exception()) + fut.cancel() + del fut + self.assertFalse(m_log.error.called) + @mock.patch('asyncio.base_events.logger') def test_tb_logger_result_unretrieved(self, m_log): fut = self._new_future(loop=self.loop) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5462c80ad3013a..195a1ed40770dd 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1865,6 +1865,25 @@ def kill_me(loop): }) mock_handler.reset_mock() + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def coro(): + raise TypeError + + @asyncio.coroutine + def runner(): + task = self.new_task(loop, coro()) + yield from asyncio.sleep(0.05, loop=loop) + task.cancel() + task = None + + loop.run_until_complete(runner()) + self.assertFalse(m_log.error.called) + @mock.patch('asyncio.coroutines.logger') def test_coroutine_never_yielded(self, m_log): with set_coroutine_debug(True): diff --git a/Misc/NEWS b/Misc/NEWS index d31ae7ee9b6ffe..5f44a790f7f9f2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Core and Builtins Library ------- +- bpo-30508: Don't log exceptions if Task/Future "cancel()" method was + called. + - bpo-28556: Updates to typing module: Add generic AsyncContextManager, add support for ContextManager on all versions. Original PRs by Jelle Zijlstra and Ivan Levkivskyi diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 75327fa3014be9..492b98359713df 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -305,6 +305,8 @@ future_add_done_callback(FutureObj *fut, PyObject *arg) static PyObject * future_cancel(FutureObj *fut) { + fut->fut_log_tb = 0; + if (fut->fut_state != STATE_PENDING) { Py_RETURN_FALSE; } @@ -638,6 +640,17 @@ FutureObj_get_log_traceback(FutureObj *fut) } } +static int +FutureObj_set_log_traceback(FutureObj *fut, PyObject *val) +{ + int is_true = PyObject_IsTrue(val); + if (is_true < 0) { + return -1; + } + fut->fut_log_tb = is_true; + return 0; +} + static PyObject * FutureObj_get_loop(FutureObj *fut) { @@ -882,7 +895,8 @@ static PyMethodDef FutureType_methods[] = { {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, NULL, NULL}, \ + {"_log_traceback", (getter)FutureObj_get_log_traceback, \ + (setter)FutureObj_set_log_traceback, NULL}, \ {"_source_traceback", (getter)FutureObj_get_source_traceback, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { @@ -1568,6 +1582,8 @@ static PyObject * _asyncio_Task_cancel_impl(TaskObj *self) /*[clinic end generated code: output=6bfc0479da9d5757 input=13f9bf496695cb52]*/ { + self->task_log_tb = 0; + if (self->task_state != STATE_PENDING) { Py_RETURN_FALSE; } From 83d30bd667924549bacf1428ac3e475cdf9792ae Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 11 Jun 2017 16:46:45 +0200 Subject: [PATCH 0409/1345] Revert "[3.6] bpo-29406: asyncio SSL contexts leak sockets after calling close with certain servers (GH-409) (#2062)" (#2112) This reverts commit 6e14fd2a14cef6ea0709ad234ab41198c2195591. --- Lib/asyncio/sslproto.py | 24 +----------------- Lib/test/test_asyncio/test_sslproto.py | 34 -------------------------- Misc/NEWS | 4 --- 3 files changed, 1 insertion(+), 61 deletions(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 3b1eb993dfc7a8..7948c4c3b4efe6 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -7,7 +7,6 @@ from . import base_events from . import compat -from . import futures from . import protocols from . import transports from .log import logger @@ -413,7 +412,7 @@ class SSLProtocol(protocols.Protocol): def __init__(self, loop, app_protocol, sslcontext, waiter, server_side=False, server_hostname=None, - call_connection_made=True, shutdown_timeout=5.0): + call_connection_made=True): if ssl is None: raise RuntimeError('stdlib ssl module not available') @@ -444,8 +443,6 @@ def __init__(self, loop, app_protocol, sslcontext, waiter, self._session_established = False self._in_handshake = False self._in_shutdown = False - self._shutdown_timeout = shutdown_timeout - self._shutdown_timeout_handle = None # transport, ex: SelectorSocketTransport self._transport = None self._call_connection_made = call_connection_made @@ -560,15 +557,6 @@ def _start_shutdown(self): self._in_shutdown = True self._write_appdata(b'') - if self._shutdown_timeout is not None: - self._shutdown_timeout_handle = self._loop.call_later( - self._shutdown_timeout, self._on_shutdown_timeout) - - def _on_shutdown_timeout(self): - if self._transport is not None: - self._fatal_error( - futures.TimeoutError(), 'Can not complete shitdown operation') - def _write_appdata(self, data): self._write_backlog.append((data, 0)) self._write_buffer_size += len(data) @@ -696,22 +684,12 @@ def _fatal_error(self, exc, message='Fatal error on transport'): }) if self._transport: self._transport._force_close(exc) - self._transport = None - - if self._shutdown_timeout_handle is not None: - self._shutdown_timeout_handle.cancel() - self._shutdown_timeout_handle = None def _finalize(self): self._sslpipe = None if self._transport is not None: self._transport.close() - self._transport = None - - if self._shutdown_timeout_handle is not None: - self._shutdown_timeout_handle.cancel() - self._shutdown_timeout_handle = None def _abort(self): try: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 6bcaa9edb4306d..bcd236ea2632ed 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -96,40 +96,6 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) - def test_close_abort(self): - # From issue #bpo-29406 - # abort connection if server does not complete shutdown procedure - ssl_proto = self.ssl_protocol() - transport = self.connection_made(ssl_proto) - ssl_proto._on_handshake_complete(None) - ssl_proto._start_shutdown() - self.assertIsNotNone(ssl_proto._shutdown_timeout_handle) - - exc_handler = mock.Mock() - self.loop.set_exception_handler(exc_handler) - ssl_proto._shutdown_timeout_handle._run() - - exc_handler.assert_called_with( - self.loop, {'message': 'Can not complete shitdown operation', - 'exception': mock.ANY, - 'transport': transport, - 'protocol': ssl_proto} - ) - self.assertIsNone(ssl_proto._shutdown_timeout_handle) - - def test_close(self): - # From issue #bpo-29406 - # abort connection if server does not complete shutdown procedure - ssl_proto = self.ssl_protocol() - transport = self.connection_made(ssl_proto) - ssl_proto._on_handshake_complete(None) - ssl_proto._start_shutdown() - self.assertIsNotNone(ssl_proto._shutdown_timeout_handle) - - ssl_proto._finalize() - self.assertIsNone(ssl_proto._transport) - self.assertIsNone(ssl_proto._shutdown_timeout_handle) - def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = asyncio.Future(loop=self.loop) diff --git a/Misc/NEWS b/Misc/NEWS index 5f44a790f7f9f2..be2ec4f9547b2c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,10 +56,6 @@ Library support for ContextManager on all versions. Original PRs by Jelle Zijlstra and Ivan Levkivskyi -- bpo-29406: asyncio SSL contexts leak sockets after calling close with - certain servers. - Patch by Nikolay Kim - - bpo-29870: Fix ssl sockets leaks when connection is aborted in asyncio/ssl implementation. Patch by Michaël Sghaïer. From f57e34b6725810576a45f015f96de35b723b915f Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sun, 11 Jun 2017 11:30:57 -0500 Subject: [PATCH 0410/1345] [3.6] Use Travis to make sure all generated files are up to date (GH-2080) (GH-2092) (cherry picked from commit 0afbabe245) Also fixes some line endings missed in GH-840 backport. --- .travis.yml | 9 + Doc/make.bat | 260 ++++++++++++------------- Lib/ctypes/macholib/fetch_macholib.bat | 2 +- Lib/idlelib/idle.bat | 8 +- 4 files changed, 144 insertions(+), 135 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b4556fc6fc462..6e3620b2379e65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,7 @@ matrix: # Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - | + set -e if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' then echo "Only docs were updated, stopping build process." @@ -77,6 +78,14 @@ before_script: fi ./configure --with-pydebug make -j4 + make -j4 regen-all clinic + changes=`git status --porcelain` + if ! test -z "$changes" + then + echo "Generated files not up to date" + echo "$changes" + exit 1 + fi script: # `-r -w` implicitly provided through `make buildbottest`. diff --git a/Doc/make.bat b/Doc/make.bat index b1a3738f309d3b..d9c0ad0adecfe6 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -1,130 +1,130 @@ -@echo off -setlocal - -pushd %~dp0 - -set this=%~n0 - -if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build -if "%PYTHON%" EQU "" set PYTHON=py - -if "%1" NEQ "htmlhelp" goto :skiphhcsearch -if exist "%HTMLHELP%" goto :skiphhcsearch - -rem Search for HHC in likely places -set HTMLHELP= -where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch -where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" ( - echo. - echo.The HTML Help Workshop was not found. Set the HTMLHELP variable - echo.to the path to hhc.exe or download and install it from - echo.http://msdn.microsoft.com/en-us/library/ms669985 - exit /B 1 -) -:skiphhcsearch - -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v - -if "%BUILDDIR%" EQU "" set BUILDDIR=build - -rem Targets that don't require sphinx-build -if "%1" EQU "" goto help -if "%1" EQU "help" goto help -if "%1" EQU "check" goto check -if "%1" EQU "serve" goto serve -if "%1" == "clean" ( - rmdir /q /s %BUILDDIR% - goto end -) - -%SPHINXBUILD% >nul 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - popd - exit /B 1 -) - -rem Targets that do require sphinx-build and have their own label -if "%1" EQU "htmlview" goto htmlview - -rem Everything else -goto build - -:help -echo.usage: %this% BUILDER [filename ...] -echo. -echo.Call %this% with the desired Sphinx builder as the first argument, e.g. -echo.``%this% html`` or ``%this% doctest``. Interesting targets that are -echo.always available include: -echo. -echo. Provided by Sphinx: -echo. html, htmlhelp, latex, text -echo. suspicious, linkcheck, changes, doctest -echo. Provided by this script: -echo. clean, check, serve, htmlview -echo. -echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.rst in this directory or -echo.the documentation for your version of Sphinx for more exhaustive lists -echo.of available targets and descriptions of each. -echo. -echo.This script assumes that the SPHINXBUILD environment variable contains -echo.a legitimate command for calling sphinx-build, or that sphinx-build is -echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can -echo.be passed by setting the SPHINXOPTS environment variable. -goto end - -:build -if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% -) -cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* - -if "%1" EQU "htmlhelp" ( - cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp - rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 - if not errorlevel 2 cmd /C exit /b 0 -) - -echo. -if errorlevel 1 ( - echo.Build failed (exit code %ERRORLEVEL%^), check for error messages - echo.above. Any output will be found in %BUILDDIR%\%1 -) else ( - echo.Build succeeded. All output should be in %BUILDDIR%\%1 -) -goto end - -:htmlview -if NOT "%2" EQU "" ( - echo.Can't specify filenames to build with htmlview target, ignoring. -) -cmd /C %this% html - -if EXIST %BUILDDIR%\html\index.html ( - echo.Opening %BUILDDIR%\html\index.html in the default web browser... - start %BUILDDIR%\html\index.html -) - -goto end - -:check -cmd /C %PYTHON% tools\rstlint.py -i tools -goto end - -:serve -cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html -goto end - -:end -popd +@echo off +setlocal + +pushd %~dp0 + +set this=%~n0 + +if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build +if "%PYTHON%" EQU "" set PYTHON=py + +if "%1" NEQ "htmlhelp" goto :skiphhcsearch +if exist "%HTMLHELP%" goto :skiphhcsearch + +rem Search for HHC in likely places +set HTMLHELP= +where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch +where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" ( + echo. + echo.The HTML Help Workshop was not found. Set the HTMLHELP variable + echo.to the path to hhc.exe or download and install it from + echo.http://msdn.microsoft.com/en-us/library/ms669985 + exit /B 1 +) +:skiphhcsearch + +if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v + +if "%BUILDDIR%" EQU "" set BUILDDIR=build + +rem Targets that don't require sphinx-build +if "%1" EQU "" goto help +if "%1" EQU "help" goto help +if "%1" EQU "check" goto check +if "%1" EQU "serve" goto serve +if "%1" == "clean" ( + rmdir /q /s %BUILDDIR% + goto end +) + +%SPHINXBUILD% >nul 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + popd + exit /B 1 +) + +rem Targets that do require sphinx-build and have their own label +if "%1" EQU "htmlview" goto htmlview + +rem Everything else +goto build + +:help +echo.usage: %this% BUILDER [filename ...] +echo. +echo.Call %this% with the desired Sphinx builder as the first argument, e.g. +echo.``%this% html`` or ``%this% doctest``. Interesting targets that are +echo.always available include: +echo. +echo. Provided by Sphinx: +echo. html, htmlhelp, latex, text +echo. suspicious, linkcheck, changes, doctest +echo. Provided by this script: +echo. clean, check, serve, htmlview +echo. +echo.All arguments past the first one are passed through to sphinx-build as +echo.filenames to build or are ignored. See README.rst in this directory or +echo.the documentation for your version of Sphinx for more exhaustive lists +echo.of available targets and descriptions of each. +echo. +echo.This script assumes that the SPHINXBUILD environment variable contains +echo.a legitimate command for calling sphinx-build, or that sphinx-build is +echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can +echo.be passed by setting the SPHINXOPTS environment variable. +goto end + +:build +if NOT "%PAPER%" == "" ( + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% +) +cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* + +if "%1" EQU "htmlhelp" ( + cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp + rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 + if not errorlevel 2 cmd /C exit /b 0 +) + +echo. +if errorlevel 1 ( + echo.Build failed (exit code %ERRORLEVEL%^), check for error messages + echo.above. Any output will be found in %BUILDDIR%\%1 +) else ( + echo.Build succeeded. All output should be in %BUILDDIR%\%1 +) +goto end + +:htmlview +if NOT "%2" EQU "" ( + echo.Can't specify filenames to build with htmlview target, ignoring. +) +cmd /C %this% html + +if EXIST %BUILDDIR%\html\index.html ( + echo.Opening %BUILDDIR%\html\index.html in the default web browser... + start %BUILDDIR%\html\index.html +) + +goto end + +:check +cmd /C %PYTHON% tools\rstlint.py -i tools +goto end + +:serve +cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html +goto end + +:end +popd diff --git a/Lib/ctypes/macholib/fetch_macholib.bat b/Lib/ctypes/macholib/fetch_macholib.bat index f474d5cd0a26f7..f9e1c0dc96c318 100644 --- a/Lib/ctypes/macholib/fetch_macholib.bat +++ b/Lib/ctypes/macholib/fetch_macholib.bat @@ -1 +1 @@ -svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . +svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . diff --git a/Lib/idlelib/idle.bat b/Lib/idlelib/idle.bat index 3d619a37eed366..e77b96e9b5149b 100755 --- a/Lib/idlelib/idle.bat +++ b/Lib/idlelib/idle.bat @@ -1,4 +1,4 @@ -@echo off -rem Start IDLE using the appropriate Python interpreter -set CURRDIR=%~dp0 -start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +@echo off +rem Start IDLE using the appropriate Python interpreter +set CURRDIR=%~dp0 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 From a895f91a46c65a6076e8c6a28af0df1a07ed60a2 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 13:50:51 -0400 Subject: [PATCH 0411/1345] [3.6]bpo-29995: Adjust IDLE test for 3.7 re.escape change [GH-1007] (#2114) --- Lib/idlelib/idle_test/test_replace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index 9913ed2b7c81e1..2ecbd34168c54e 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -221,8 +221,8 @@ def test_replace_regex(self): self.assertIn('Invalid Replace Expression', showerror.message) # test access method - self.engine.setcookedpat("\'") - equal(pv.get(), "\\'") + self.engine.setcookedpat("?") + equal(pv.get(), "\\?") def test_replace_backwards(self): equal = self.assertEqual From 57d8de80313c536d409d6a104ae577af8ffc57fb Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 11 Jun 2017 14:29:38 -0400 Subject: [PATCH 0412/1345] [3.6]bpo-20185: Adjust IDLE test to 3.7 Clinic change [GH-542] (#2116) Synchronize 3.6/3.7 test_calltips to the extent possible. Part of patch by Serhiy Storchaka. (cherry-pick from fdd42c4) --- Lib/idlelib/idle_test/test_calltips.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 0b11602ca9e414..97e15fe10e0ff4 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -58,7 +58,7 @@ def gtest(obj, out): 'Create and return a new object. See help(type) for accurate signature.') gtest(list.__init__, 'Initialize self. See help(type(self)) for accurate signature.') - append_doc = "L.append(object) -> None -- append object to end" + append_doc = "L.append(object) -> None -- append object to end" #see3.7 gtest(list.append, append_doc) gtest([].append, append_doc) gtest(List.append, append_doc) @@ -81,9 +81,9 @@ def f(): pass def test_multiline_docstring(self): # Test fewer lines than max. - self.assertEqual(signature(list), - "list() -> new empty list\n" - "list(iterable) -> new list initialized from iterable's items") + self.assertEqual(signature(range), + "range(stop) -> range object\n" + "range(start, stop[, step]) -> range object") # Test max lines self.assertEqual(signature(bytes), '''\ From 2737291134d398da9e5b932b0309a6d9d343cb5a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 11 Jun 2017 11:53:14 -0700 Subject: [PATCH 0413/1345] backport ssl doc fixes (#2117) * clarify recv() and send() on SSLObject (#2100) SSLObject has recv() and send(), but they don't do any network io. * remove extra word (#2101) --- Doc/library/ssl.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 16756e1d73e767..6c30a74a0991aa 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2107,8 +2107,8 @@ provided. When compared to :class:`SSLSocket`, this object lacks the following features: - - Any form of network IO incluging methods such as ``recv()`` and - ``send()``. + - Any form of network IO; ``recv()`` and ``send()`` read and write only to + the underlying :class:`MemoryBIO` buffers. - There is no *do_handshake_on_connect* machinery. You must always manually call :meth:`~SSLSocket.do_handshake` to start the handshake. From 47c9decb5feabb51416deffd2098ee31d73f1f5f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 11 Jun 2017 18:59:05 -0700 Subject: [PATCH 0414/1345] bpo-30621: Update Input Output Tutorial Example(GH-2074) (GH-2118) import json before using json module (cherry picked from commit 1dbce04d0e3e93b715eb0d8024da396361759d16) --- Doc/tutorial/inputoutput.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index bad0302b037861..92f439c31c920b 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -412,6 +412,7 @@ sent over a network connection to some distant machine. If you have an object ``x``, you can view its JSON string representation with a simple line of code:: + >>> import json >>> json.dumps([1, 'simple', 'list']) '[1, "simple", "list"]' From d89dc844d288ee2e403272f3e4552eed6911cafd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 12 Jun 2017 09:02:13 +0300 Subject: [PATCH 0415/1345] [3.6] bpo-28994: Fixed errors handling in atexit._run_exitfuncs(). (GH-2034) (#2121) The traceback no longer displayed for SystemExit raised in a callback registered by atexit.. (cherry picked from commit 3fd54d4a7e604067e2bc0f8cfd58bdbdc09fa7f4) --- Lib/test/test_atexit.py | 10 ++++++++++ Misc/NEWS | 3 +++ Modules/atexitmodule.c | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index c761076c4a0225..1d0b018aafaf93 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -23,6 +23,9 @@ def raise1(): def raise2(): raise SystemError +def exit(): + raise SystemExit + class GeneralTest(unittest.TestCase): @@ -76,6 +79,13 @@ def test_raise_unnormalized(self): self.assertRaises(ZeroDivisionError, atexit._run_exitfuncs) self.assertIn("ZeroDivisionError", self.stream.getvalue()) + def test_exit(self): + # be sure a SystemExit is handled properly + atexit.register(exit) + + self.assertRaises(SystemExit, atexit._run_exitfuncs) + self.assertEqual(self.stream.getvalue(), '') + def test_print_tracebacks(self): # Issue #18776: the tracebacks should be printed when errors occur. def f(): diff --git a/Misc/NEWS b/Misc/NEWS index be2ec4f9547b2c..e56c5a300feb3d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Core and Builtins Library ------- +- bpo-28994: The traceback no longer displayed for SystemExit raised in + a callback registered by atexit. + - bpo-30508: Don't log exceptions if Task/Future "cancel()" method was called. diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 3cdf2d7e56348a..35ebf08ecd3c66 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -97,7 +97,7 @@ atexit_callfuncs(void) Py_XDECREF(exc_tb); } PyErr_Fetch(&exc_type, &exc_value, &exc_tb); - if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { + if (!PyErr_GivenExceptionMatches(exc_type, PyExc_SystemExit)) { PySys_WriteStderr("Error in atexit._run_exitfuncs:\n"); PyErr_NormalizeException(&exc_type, &exc_value, &exc_tb); PyErr_Display(exc_type, exc_value, exc_tb); From e1b690370fd8f93bef1e69eeea2695f95a7cfff5 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 12 Jun 2017 22:28:12 +1000 Subject: [PATCH 0416/1345] bpo-23404: `make regen-all` What's New entry (#2128) While the build changes won't affect most users, they may affect redistributors and folks building their own Python binaries from source. --- Doc/whatsnew/3.6.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index a701cafa610788..1e71bc3b10a2f8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2306,3 +2306,39 @@ There have been several major changes to the :term:`bytecode` in Python 3.6. * The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`STORE_ANNOTATION` opcodes have been added to support the new :term:`variable annotation` syntax. (Contributed by Ivan Levkivskyi in :issue:`27985`.) + + +Notable changes in Python 3.6.2 +=============================== + +New ``make regen-all`` build target +----------------------------------- + +To simplify cross-compilation, and to ensure that CPython can reliably be +compiled without requiring an existing version of Python to already be +available, the autotools-based build system no longer attempts to implicitly +recompile generated files based on file modification times. + +Instead, a new ``make regen-all`` command has been added to force regeneration +of these files when desired (e.g. after an initial version of Python has +already been built based on the pregenerated versions). + +More selective regeneration targets are also defined - see +:source:`Makefile.pre.in` for details. + +(Contributed by Victor Stinner in :issue:`23404`.) + +.. versionadded:: 3.6.2 + + +Removal of ``make touch`` build target +-------------------------------------- + +The ``make touch`` build target previously used to request implicit regeneration +of generated files by updating their modification times has been removed. + +It has been replaced by the new ``make regen-all`` target. + +(Contributed by Victor Stinner in :issue:`23404`.) + +.. versionchanged:: 3.6.2 From 8399a177de8bfa860a66e96665488c17199cb9d2 Mon Sep 17 00:00:00 2001 From: Antoine Pietri Date: Mon, 12 Jun 2017 18:23:24 +0200 Subject: [PATCH 0417/1345] [3.6] bpo-30177: add NEWS entry (#2134) --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index e56c5a300feb3d..a9507311110b44 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +- bpo-30177: path.resolve(strict=False) no longer cuts the path after the first + element not present in the filesystem. Patch by Antoine Pietri. + IDLE ---- From f59cac4b6458e5c47e24a39ba46fb178c3766577 Mon Sep 17 00:00:00 2001 From: Marco Buttu Date: Mon, 12 Jun 2017 21:41:42 +0200 Subject: [PATCH 0418/1345] bpo-30217: add the operators ~ and | to the index (GH-1502) (GH-2136) (cherry picked from commit dc980dfbcfce4695ccde056c3983160ba97b5a36) --- Doc/library/stdtypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ca3404aa21e555..083fe5367af9d8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -394,10 +394,12 @@ Bitwise Operations on Integer Types pair: bitwise; operations pair: shifting; operations pair: masking; operations + operator: | operator: ^ operator: & operator: << operator: >> + operator: ~ Bitwise operations only make sense for integers. Negative numbers are treated as their 2's complement value (this assumes that there are enough bits so that From 2997fec01ee7300c6d5940e6c55e4ccf9f56f1b5 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 12 Jun 2017 18:46:35 -0700 Subject: [PATCH 0419/1345] [3.6] bpo-30604: Move co_extra_freefuncs to interpreter state to avoid crashes in threads (#2015) * Move co_extra_freefuncs to interpreter state to avoid crashes in multi-threaded scenarios involving deletion of code objects * Don't require that extra be zero initialized * Build test list instead of defining empty test class * Ensure extra is always assigned on success * Keep the old fields in the thread state object, just don't use them Add new linked list of code extra objects on a per-interpreter basis so that interpreter state size isn't changed * Rename __PyCodeExtraState_Get and add comment about it going away in 3.7 Fix sort order of import's in test_code.py * Remove an extraneous space * Remove docstrings for comments * Touch up formatting * Fix casing of coextra local * Fix casing of another variable * Prefix PyCodeExtraState with __ to match C API for getting it * Update NEWS file for bpo-30604 --- Include/pystate.h | 16 ++++++- Lib/test/test_code.py | 103 ++++++++++++++++++++++++++++++++++++++++-- Misc/NEWS | 4 +- Objects/codeobject.c | 28 +++++++----- Python/ceval.c | 8 ++-- Python/pystate.c | 44 +++++++++++++++++- 6 files changed, 180 insertions(+), 23 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index afc3c0c6d1d5ac..6ca463f7e15a8f 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -51,6 +51,16 @@ typedef struct _is { } PyInterpreterState; #endif +typedef struct _co_extra_state { + struct _co_extra_state *next; + PyInterpreterState* interp; + + Py_ssize_t co_extra_user_count; + freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; +} __PyCodeExtraState; + +/* This is temporary for backwards compat in 3.6 and will be removed in 3.7 */ +__PyCodeExtraState* __PyCodeExtraState_Get(); /* State unique per thread */ @@ -142,8 +152,10 @@ typedef struct _ts { PyObject *coroutine_wrapper; int in_coroutine_wrapper; - Py_ssize_t co_extra_user_count; - freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; + /* Now used from PyInterpreterState, kept here for ABI + compatibility with PyThreadState */ + Py_ssize_t _preserve_36_ABI_1; + freefunc _preserve_36_ABI_2[MAX_CO_EXTRA_USERS]; PyObject *async_gen_firstiter; PyObject *async_gen_finalizer; diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7975ea0ef5e17d..9f9df9dba6b3d6 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -103,9 +103,11 @@ """ import sys +import threading import unittest import weakref -from test.support import run_doctest, run_unittest, cpython_only +from test.support import (run_doctest, run_unittest, cpython_only, + check_impl_detail) def consts(t): @@ -212,11 +214,106 @@ def callback(code): self.assertTrue(self.called) +if check_impl_detail(cpython=True): + import ctypes + py = ctypes.pythonapi + freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) + + RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = py._PyCode_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = py._PyCode_GetExtra + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp)) + GetExtra.restype = ctypes.c_int + + LAST_FREED = None + def myfree(ptr): + global LAST_FREED + LAST_FREED = ptr + + FREE_FUNC = freefunc(myfree) + FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + + class CoExtra(unittest.TestCase): + def get_func(self): + # Defining a function causes the containing function to have a + # reference to the code object. We need the code objects to go + # away, so we eval a lambda. + return eval('lambda:42') + + def test_get_non_code(self): + f = self.get_func() + + self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + + def test_bad_index(self): + f = self.get_func() + self.assertRaises(SystemError, SetExtra, f.__code__, + FREE_INDEX+100, ctypes.c_voidp(100)) + self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, + ctypes.c_voidp(100)), 0) + + def test_free_called(self): + # Verify that the provided free function gets invoked + # when the code object is cleaned up. + f = self.get_func() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) + del f + self.assertEqual(LAST_FREED, 100) + + def test_get_set(self): + # Test basic get/set round tripping. + f = self.get_func() + + extra = ctypes.c_voidp() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) + # reset should free... + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) + self.assertEqual(LAST_FREED, 200) + + extra = ctypes.c_voidp() + GetExtra(f.__code__, FREE_INDEX, extra) + self.assertEqual(extra.value, 300) + del f + + def test_free_different_thread(self): + # Freeing a code object on a different thread then + # where the co_extra was set should be safe. + f = self.get_func() + class ThreadTest(threading.Thread): + def __init__(self, f, test): + super().__init__() + self.f = f + self.test = test + def run(self): + del self.f + self.test.assertEqual(LAST_FREED, 500) + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) + tt = ThreadTest(f, self) + del f + tt.start() + tt.join() + self.assertEqual(LAST_FREED, 500) + def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) - run_unittest(CodeTest, CodeConstsTest, CodeWeakRefTest) - + tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] + if check_impl_detail(cpython=True): + tests.append(CoExtra) + run_unittest(*tests) if __name__ == "__main__": test_main() diff --git a/Misc/NEWS b/Misc/NEWS index a9507311110b44..f7441600837025 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,4 +1,4 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ @@ -10,6 +10,8 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-30604: Move co_extra_freefuncs to not be per-thread to avoid crashes + - bpo-29104: Fixed parsing backslashes in f-strings. - bpo-27945: Fixed various segfaults with dict when input collections are diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 22c4f856cd83e6..d38f185ba3b8e4 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -411,11 +411,11 @@ static void code_dealloc(PyCodeObject *co) { if (co->co_extra != NULL) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); _PyCodeObjectExtra *co_extra = co->co_extra; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { - freefunc free_extra = tstate->co_extra_freefuncs[i]; + freefunc free_extra = state->co_extra_freefuncs[i]; if (free_extra != NULL) { free_extra(co_extra->ce_extras[i]); @@ -825,8 +825,6 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds) int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { - assert(*extra == NULL); - if (!PyCode_Check(code)) { PyErr_BadInternalCall(); return -1; @@ -837,6 +835,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) if (co_extra == NULL || co_extra->ce_size <= index) { + *extra = NULL; return 0; } @@ -848,10 +847,10 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); if (!PyCode_Check(code) || index < 0 || - index >= tstate->co_extra_user_count) { + index >= state->co_extra_user_count) { PyErr_BadInternalCall(); return -1; } @@ -866,13 +865,13 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) } co_extra->ce_extras = PyMem_Malloc( - tstate->co_extra_user_count * sizeof(void*)); + state->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { PyMem_Free(co_extra); return -1; } - co_extra->ce_size = tstate->co_extra_user_count; + co_extra->ce_size = state->co_extra_user_count; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; @@ -882,20 +881,27 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) } else if (co_extra->ce_size <= index) { void** ce_extras = PyMem_Realloc( - co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); + co_extra->ce_extras, state->co_extra_user_count * sizeof(void*)); if (ce_extras == NULL) { return -1; } for (Py_ssize_t i = co_extra->ce_size; - i < tstate->co_extra_user_count; + i < state->co_extra_user_count; i++) { ce_extras[i] = NULL; } co_extra->ce_extras = ce_extras; - co_extra->ce_size = tstate->co_extra_user_count; + co_extra->ce_size = state->co_extra_user_count; + } + + if (co_extra->ce_extras[index] != NULL) { + freefunc free = state->co_extra_freefuncs[index]; + if (free != NULL) { + free(co_extra->ce_extras[index]); + } } co_extra->ce_extras[index] = extra; diff --git a/Python/ceval.c b/Python/ceval.c index eba892c1ce3b3c..ea79f5f2b57f8f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5453,14 +5453,14 @@ _Py_GetDXProfile(PyObject *self, PyObject *args) Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); Py_ssize_t new_index; - if (tstate->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + if (state->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { return -1; } - new_index = tstate->co_extra_user_count++; - tstate->co_extra_freefuncs[new_index] = free; + new_index = state->co_extra_user_count++; + state->co_extra_freefuncs[new_index] = free; return new_index; } diff --git a/Python/pystate.c b/Python/pystate.c index ccb0092c42b3a4..92d08c41091878 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -55,6 +55,7 @@ static int autoTLSkey = -1; #endif static PyInterpreterState *interp_head = NULL; +static __PyCodeExtraState *coextra_head = NULL; /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ @@ -73,6 +74,12 @@ PyInterpreterState_New(void) PyMem_RawMalloc(sizeof(PyInterpreterState)); if (interp != NULL) { + __PyCodeExtraState* coextra = PyMem_RawMalloc(sizeof(__PyCodeExtraState)); + if (coextra == NULL) { + PyMem_RawFree(interp); + return NULL; + } + HEAD_INIT(); #ifdef WITH_THREAD if (head_mutex == NULL) @@ -92,6 +99,8 @@ PyInterpreterState_New(void) interp->importlib = NULL; interp->import_func = NULL; interp->eval_frame = _PyEval_EvalFrameDefault; + coextra->co_extra_user_count = 0; + coextra->interp = interp; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW interp->dlopenflags = RTLD_NOW; @@ -103,6 +112,8 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; + coextra->next = coextra_head; + coextra_head = coextra; HEAD_UNLOCK(); } @@ -147,9 +158,10 @@ void PyInterpreterState_Delete(PyInterpreterState *interp) { PyInterpreterState **p; + __PyCodeExtraState **pextra; zapthreads(interp); HEAD_LOCK(); - for (p = &interp_head; ; p = &(*p)->next) { + for (p = &interp_head; /* N/A */; p = &(*p)->next) { if (*p == NULL) Py_FatalError( "PyInterpreterState_Delete: invalid interp"); @@ -159,6 +171,18 @@ PyInterpreterState_Delete(PyInterpreterState *interp) if (interp->tstate_head != NULL) Py_FatalError("PyInterpreterState_Delete: remaining threads"); *p = interp->next; + + for (pextra = &coextra_head; ; pextra = &(*pextra)->next) { + if (*pextra == NULL) + Py_FatalError( + "PyInterpreterState_Delete: invalid extra"); + __PyCodeExtraState* extra = *pextra; + if (extra->interp == interp) { + *pextra = extra->next; + PyMem_RawFree(extra); + break; + } + } HEAD_UNLOCK(); PyMem_RawFree(interp); #ifdef WITH_THREAD @@ -224,7 +248,6 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->coroutine_wrapper = NULL; tstate->in_coroutine_wrapper = 0; - tstate->co_extra_user_count = 0; tstate->async_gen_firstiter = NULL; tstate->async_gen_finalizer = NULL; @@ -548,6 +571,23 @@ PyThreadState_Swap(PyThreadState *newts) return oldts; } +__PyCodeExtraState* +__PyCodeExtraState_Get() { + PyInterpreterState* interp = PyThreadState_Get()->interp; + + HEAD_LOCK(); + for (__PyCodeExtraState* cur = coextra_head; cur != NULL; cur = cur->next) { + if (cur->interp == interp) { + HEAD_UNLOCK(); + return cur; + } + } + HEAD_UNLOCK(); + + Py_FatalError("__PyCodeExtraState_Get: no code state for interpreter"); + return NULL; +} + /* An extension mechanism to store arbitrary additional per-thread state. PyThreadState_GetDict() returns a dictionary that can be used to hold such state; the caller should pick a unique key and store its state there. If From 81c05ccc10d044584dff3a69ee531094ed76ee2c Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 12 Jun 2017 22:40:18 -0700 Subject: [PATCH 0420/1345] bpo-6519: Improve Python Input Output Tutorial (GH-2143) (GH-2145) Move up the discussion about 'with' keyword, so it appears earlier in the document. (cherry picked from commit bd4e9e0ca96dabf33605d9b1fd1e0562ece8ae18) --- Doc/tutorial/inputoutput.rst | 50 +++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 92f439c31c920b..74d7bad42a12be 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -262,6 +262,35 @@ to file data is fine for text files, but will corrupt binary data like that in :file:`JPEG` or :file:`EXE` files. Be very careful to use binary mode when reading and writing such files. +It is good practice to use the :keyword:`with` keyword when dealing +with file objects. The advantage is that the file is properly closed +after its suite finishes, even if an exception is raised at some +point. Using :keyword:`with` is also much shorter than writing +equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: + + >>> with open('workfile') as f: + ... read_data = f.read() + >>> f.closed + True + +If you're not using the :keyword:`with` keyword, then you should call +``f.close()`` to close the file and immediately free up any system +resources used by it. If you don't explicitly close a file, Python's +garbage collector will eventually destroy the object and close the +open file for you, but the file may stay open for a while. Another +risk is that different Python implementations will do this clean-up at +different times. + +After a file object is closed, either by a :keyword:`with` statement +or by calling ``f.close()``, attempts to use the file object will +automatically fail. :: + + >>> f.close() + >>> f.read() + Traceback (most recent call last): + File "", line 1, in + ValueError: I/O operation on closed file + .. _tut-filemethods: @@ -354,27 +383,6 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. - -When you're done with a file, call ``f.close()`` to close it and free up any -system resources taken up by the open file. After calling ``f.close()``, -attempts to use the file object will automatically fail. :: - - >>> f.close() - >>> f.read() - Traceback (most recent call last): - File "", line 1, in - ValueError: I/O operation on closed file - -It is good practice to use the :keyword:`with` keyword when dealing with file -objects. This has the advantage that the file is properly closed after its -suite finishes, even if an exception is raised on the way. It is also much -shorter than writing equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - - >>> with open('workfile', 'r') as f: - ... read_data = f.read() - >>> f.closed - True - File objects have some additional methods, such as :meth:`~file.isatty` and :meth:`~file.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. From d0d63dc1e8a2e71c4cd6497e5ea503c702d0fe3b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 13 Jun 2017 10:23:13 +0200 Subject: [PATCH 0421/1345] [3.6] Mention how to disable signal fd wakeup (GH-2140) (#2149) (cherry picked from commit d79c1d4a9406384f10a37f26a7515ce79f9fdd78) --- Doc/library/signal.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 039b666475a247..46d71def08abd6 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -306,8 +306,10 @@ The :mod:`signal` module defines the following functions: a library to wakeup a poll or select call, allowing the signal to be fully processed. - The old wakeup fd is returned. *fd* must be non-blocking. It is up to the - library to remove any bytes before calling poll or select again. + The old wakeup fd is returned (or -1 if file descriptor wakeup was not + enabled). If *fd* is -1, file descriptor wakeup is disabled. + If not -1, *fd* must be non-blocking. It is up to the library to remove + any bytes from *fd* before calling poll or select again. Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the signal numbers list. From 932946ca14168e556293d2508c8eebb23a56a2b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 13 Jun 2017 10:39:30 +0200 Subject: [PATCH 0422/1345] bpo-30604: Fix __PyCodeExtraState_Get() prototype (#2152) Replace __PyCodeExtraState_Get() with __PyCodeExtraState_Get(void) to fix the following GCC warning: ./Include/pystate.h:63:1: warning: function declaration isn't a prototype [-Wstrict-prototypes] __PyCodeExtraState* __PyCodeExtraState_Get(); --- Include/pystate.h | 2 +- Python/pystate.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 6ca463f7e15a8f..1838fa40480713 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -60,7 +60,7 @@ typedef struct _co_extra_state { } __PyCodeExtraState; /* This is temporary for backwards compat in 3.6 and will be removed in 3.7 */ -__PyCodeExtraState* __PyCodeExtraState_Get(); +__PyCodeExtraState* __PyCodeExtraState_Get(void); /* State unique per thread */ diff --git a/Python/pystate.c b/Python/pystate.c index 92d08c41091878..65f9b7ea05dd79 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -571,8 +571,8 @@ PyThreadState_Swap(PyThreadState *newts) return oldts; } -__PyCodeExtraState* -__PyCodeExtraState_Get() { +__PyCodeExtraState* +__PyCodeExtraState_Get(void) { PyInterpreterState* interp = PyThreadState_Get()->interp; HEAD_LOCK(); From 92e9e35292d4e2044c25970389f3cdb30cfc903a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 13 Jun 2017 20:32:46 +1000 Subject: [PATCH 0423/1345] [3.6] bpo-29514: Check magic number for bugfix releases (#2157) Add a test to check the current MAGIC_NUMBER against the expected number for the release if the current release is at candidate or final level. On test failure, describe to the developer the procedure for changing the magic number. This ensures that pre-merge CI will automatically pick up on magic number changes in maintenance releases (and explain why those are problematic), rather than relying on all core developers to be aware of the implications of such changes. --- Lib/test/test_importlib/test_util.py | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index d615375b342091..ac18e5c34b87fa 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -4,6 +4,7 @@ machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') +import importlib.util import os import pathlib import string @@ -757,5 +758,48 @@ def test_source_from_cache_path_like_arg(self): ) = util.test_both(PEP3147Tests, util=importlib_util) +class MagicNumberTests(unittest.TestCase): + """ + Test release compatibility issues relating to importlib + """ + @unittest.skipUnless( + sys.version_info.releaselevel in ('final', 'release'), + 'only applies to candidate or final python release levels' + ) + def test_magic_number(self): + """ + Each python minor release should generally have a MAGIC_NUMBER + that does not change once the release reaches candidate status. + + Once a release reaches candidate status, the value of the constant + EXPECTED_MAGIC_NUMBER in this test should be changed. + This test will then check that the actual MAGIC_NUMBER matches + the expected value for the release. + + In exceptional cases, it may be required to change the MAGIC_NUMBER + for a maintenance release. In this case the change should be + discussed in python-dev. If a change is required, community + stakeholders such as OS package maintainers must be notified + in advance. Such exceptional releases will then require an + adjustment to this test case. + """ + EXPECTED_MAGIC_NUMBER = 3379 + actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') + + msg = ( + "To avoid breaking backwards compatibility with cached bytecode " + "files that can't be automatically regenerated by the current " + "user, candidate and final releases require the current " + "importlib.util.MAGIC_NUMBER to match the expected " + "magic number in this test. Set the expected " + "magic number in this test to the current MAGIC_NUMBER to " + "continue with the release.\n\n" + "Changing the MAGIC_NUMBER for a maintenance release " + "requires discussion in python-dev and notification of " + "community stakeholders." + ) + self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) + + if __name__ == '__main__': unittest.main() From 2bfb45d447c445b3c3afc19d16b4cd4773975993 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 13 Jun 2017 10:11:02 -0400 Subject: [PATCH 0424/1345] bpo-30642: IDLE: Fix test_query refleak (#2147) (#2161) Patch by Louie Lu. (cherry picked from commit b070fd275b68df5c5ba9f6f43197b8d7066f0b18) --- Lib/idlelib/idle_test/test_query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index 66af8eb85b2598..1210afe70df956 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -258,6 +258,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + del cls.dialog.destroy del cls.dialog cls.root.destroy() del cls.root From 6fd03459957ee53941183212457bba19f977679f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 13 Jun 2017 17:51:26 +0200 Subject: [PATCH 0425/1345] [3.6] bpo-24484: Avoid race condition in multiprocessing cleanup (GH-2159) (#2166) * bpo-24484: Avoid race condition in multiprocessing cleanup The finalizer registry can be mutated while inspected by multiprocessing at process exit. * Use test.support.start_threads() * Add Misc/NEWS. (cherry picked from commit 1eb6c0074d17f4fd425cacfdda893d65f5f77f0a) --- Lib/multiprocessing/util.py | 34 ++++++++++------- Lib/test/_test_multiprocessing.py | 63 +++++++++++++++++++++++++++++++ Misc/NEWS | 2 + 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 0ce274ceca6057..b490caa7e64333 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -241,20 +241,28 @@ def _run_finalizers(minpriority=None): return if minpriority is None: - f = lambda p : p[0][0] is not None + f = lambda p : p[0] is not None else: - f = lambda p : p[0][0] is not None and p[0][0] >= minpriority - - items = [x for x in list(_finalizer_registry.items()) if f(x)] - items.sort(reverse=True) - - for key, finalizer in items: - sub_debug('calling %s', finalizer) - try: - finalizer() - except Exception: - import traceback - traceback.print_exc() + f = lambda p : p[0] is not None and p[0] >= minpriority + + # Careful: _finalizer_registry may be mutated while this function + # is running (either by a GC run or by another thread). + + # list(_finalizer_registry) should be atomic, while + # list(_finalizer_registry.items()) is not. + keys = [key for key in list(_finalizer_registry) if f(key)] + keys.sort(reverse=True) + + for key in keys: + finalizer = _finalizer_registry.get(key) + # key may have been removed from the registry + if finalizer is not None: + sub_debug('calling %s', finalizer) + try: + finalizer() + except Exception: + import traceback + traceback.print_exc() if minpriority is None: _finalizer_registry.clear() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f1f93674935e7a..cd2c8dbc487fad 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3075,6 +3075,14 @@ class _TestFinalize(BaseTestCase): ALLOWED_TYPES = ('processes',) + def setUp(self): + self.registry_backup = util._finalizer_registry.copy() + util._finalizer_registry.clear() + + def tearDown(self): + self.assertFalse(util._finalizer_registry) + util._finalizer_registry.update(self.registry_backup) + @classmethod def _test_finalize(cls, conn): class Foo(object): @@ -3124,6 +3132,61 @@ def test_finalize(self): result = [obj for obj in iter(conn.recv, 'STOP')] self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) + def test_thread_safety(self): + # bpo-24484: _run_finalizers() should be thread-safe + def cb(): + pass + + class Foo(object): + def __init__(self): + self.ref = self # create reference cycle + # insert finalizer at random key + util.Finalize(self, cb, exitpriority=random.randint(1, 100)) + + finish = False + exc = None + + def run_finalizers(): + nonlocal exc + while not finish: + time.sleep(random.random() * 1e-1) + try: + # A GC run will eventually happen during this, + # collecting stale Foo's and mutating the registry + util._run_finalizers() + except Exception as e: + exc = e + + def make_finalizers(): + nonlocal exc + d = {} + while not finish: + try: + # Old Foo's get gradually replaced and later + # collected by the GC (because of the cyclic ref) + d[random.getrandbits(5)] = {Foo() for i in range(10)} + except Exception as e: + exc = e + d.clear() + + old_interval = sys.getswitchinterval() + old_threshold = gc.get_threshold() + try: + sys.setswitchinterval(1e-6) + gc.set_threshold(5, 5, 5) + threads = [threading.Thread(target=run_finalizers), + threading.Thread(target=make_finalizers)] + with test.support.start_threads(threads): + time.sleep(4.0) # Wait a bit to trigger race condition + finish = True + if exc is not None: + raise exc + finally: + sys.setswitchinterval(old_interval) + gc.set_threshold(*old_threshold) + gc.collect() # Collect remaining Foo's + + # # Test that from ... import * works for each module # diff --git a/Misc/NEWS b/Misc/NEWS index f7441600837025..f3874140a5249c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,8 @@ Core and Builtins Library ------- +- bpo-24484: Avoid race condition in multiprocessing cleanup (#2159) + - bpo-28994: The traceback no longer displayed for SystemExit raised in a callback registered by atexit. From b0efd493b6af24a6ae744e7e02f4b69c70e88f3d Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 13 Jun 2017 11:52:08 -0400 Subject: [PATCH 0426/1345] [3.6]bpo-30642: Fix ref leak in idle_test.test_macosx (#2163) (#2165) (cherry picked from commit 8323189ff1a19566f923c04b95e4d17fa57d1f56) --- Lib/idlelib/idle_test/test_macosx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/idlelib/idle_test/test_macosx.py b/Lib/idlelib/idle_test/test_macosx.py index 775697b278d30d..3d85f3ca72254c 100644 --- a/Lib/idlelib/idle_test/test_macosx.py +++ b/Lib/idlelib/idle_test/test_macosx.py @@ -77,6 +77,10 @@ def setUpClass(cls): requires('gui') cls.root = tk.Tk() cls.root.withdraw() + def cmd(tkpath, func): + assert isinstance(tkpath, str) + assert isinstance(func, type(cmd)) + cls.root.createcommand = cmd @classmethod def tearDownClass(cls): From d92ee3ea622b6eee5846681bad5595cfedcf20b6 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 13 Jun 2017 15:40:59 -0400 Subject: [PATCH 0427/1345] [3.6]bpo-27922: Stop gui flash from idle_test.test_parenmatch (#2171) (#2172) For unknown reasons, this does not work when running leak tests. (cherry picked from commit 049cf2bb44038351e1b2eed4fc7b1b522329e550) --- Lib/idlelib/idle_test/test_parenmatch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index 051f7eac2dce06..b5fd0d1c69e4f1 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -24,6 +24,7 @@ class ParenMatchTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.root = Tk() + cls.root.withdraw() cls.text = Text(cls.root) cls.editwin = DummyEditwin(cls.text) cls.editwin.text_frame = Mock() From 22d909f8c24bd7768df1a5bf0e52c597ea947cfb Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 13 Jun 2017 22:13:15 -0400 Subject: [PATCH 0428/1345] [3.6]bpo-25514: Improve IDLE's connection refused message (#2177) (#2178) When IDLE fail to start because the socket connection fails, direct people to a new subsection of the IDLE doc listing various causes and remedies. (cherry picked from commit 188aedf8bb623d41302e10503268b0852ea91134) --- Doc/library/idle.rst | 44 +++++++++++++++++++++ Lib/idlelib/help.html | 91 +++++++++++++++++++++++++++++++------------ Lib/idlelib/help.py | 2 +- Lib/idlelib/run.py | 17 ++++---- 4 files changed, 119 insertions(+), 35 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 1a2dac0233c4d2..a945b6d771225b 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -552,6 +552,50 @@ If there are arguments: ``sys.argv`` reflects the arguments passed to IDLE itself. +Startup failure +^^^^^^^^^^^^^^^ + +IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +'RESTART'). If the user process fails to connect to the GUI process, it +displays a ``Tk`` error box with a 'cannot connect' message that directs the +user here. It then exits. + +A common cause of failure is a user-written file with the same name as a +standard library module, such as *random.py* and *tkinter.py*. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file. + +Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections. + +Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over. + +A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem. + +When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one's home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files. + +If IDLE quits with no message, and it was not started from a console, try +starting from a console (``python -m idlelib)`` and see if a message appears. + + IDLE-console differences ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index f10cd345e886c8..0a3062e156421c 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -6,7 +6,7 @@ - 25.5. IDLE — Python 3.5.2 documentation + 25.5. IDLE — Python 3.7.0a0 documentation @@ -14,7 +14,7 @@ - + + + + +
+

25.5.3.2. Startup failure

+

IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +‘RESTART’). If the user process fails to connect to the GUI process, it +displays a Tk error box with a ‘cannot connect’ message that directs the +user here. It then exits.

+

A common cause of failure is a user-written file with the same name as a +standard library module, such as random.py and tkinter.py. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file.

+

Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections.

+

Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over.

+

A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem.

+

When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one’s home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files.

+

If IDLE quits with no message, and it was not started from a console, try +starting from a console (python -m idlelib) and see if a message appears.

+
-

25.5.3.2. IDLE-console differences

+

25.5.3.3. IDLE-console differences

As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, +interface and operation occasionally affect visible results. For instance, sys.modules starts with more entries.

IDLE also replaces sys.stdin, sys.stdout, and sys.stderr with objects that get input from and send output to the Shell window. @@ -551,14 +590,14 @@

25.5.3.2. IDLE-console differences -

25.5.3.3. Running without a subprocess

+

25.5.3.4. Running without a subprocess

By default, IDLE executes user code in a separate subprocess via a socket, which uses the internal loopback interface. This connection is not externally visible and no data is sent to or received from the Internet. If firewall software complains anyway, you can ignore it.

If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch.

If IDLE is started with the -n command line switch it will run in a @@ -590,12 +629,12 @@

25.5.4.1. Additional help sources

The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab.

25.5.4.3. Extensions

-

IDLE contains an extension facility. Peferences for extensions can be +

IDLE contains an extension facility. Preferences for extensions can be changed with Configure Extensions. See the beginning of config-extensions.def in the idlelib directory for further information. The default extensions are currently:

@@ -646,8 +685,9 @@

Table Of Contents

  • 25.5.3. Startup and code execution
  • 25.5.4. Help and preferences
    • @@ -697,7 +740,7 @@

    Navigation

    style="vertical-align: middle; margin-top: -1px"/>
  • Python »
  • - 3.5.2 Documentation » + 3.7.0a0 Documentation »
  • The Python Standard Library »
    • @@ -720,12 +763,12 @@

    Navigation

    - © Copyright 2001-2016, Python Software Foundation. + © Copyright 2001-2017, Python Software Foundation.
    The Python Software Foundation is a non-profit corporation. Please donate.
    - Last updated on Sep 12, 2016. + Last updated on Jun 13, 2017. Found a bug?
    Created using Sphinx 1.3.6. diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index 77e01a31c08f60..fa6112a339444f 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -260,7 +260,7 @@ def copy_strip(): open(dst, 'wb') as out: for line in inn: out.write(line.rstrip() + b'\n') - print('idle.html copied to help.html') + print(f'{src} copied to {dst}') def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 12c339f1058b72..9f6604bb0acc58 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -180,19 +180,16 @@ def manage_socket(address): server.handle_request() # A single request only def show_socket_error(err, address): + "Display socket error from manage_socket." import tkinter - import tkinter.messagebox as tkMessageBox + from tkinter.messagebox import showerror root = tkinter.Tk() root.withdraw() - if err.args[0] == 61: # connection refused - msg = "IDLE's subprocess can't connect to %s:%d. This may be due "\ - "to your personal firewall configuration. It is safe to "\ - "allow this internal connection because no data is visible on "\ - "external ports." % address - tkMessageBox.showerror("IDLE Subprocess Error", msg, parent=root) - else: - tkMessageBox.showerror("IDLE Subprocess Error", - "Socket Error: %s" % err.args[1], parent=root) + msg = f"IDLE's subprocess can't connect to {address[0]}:{address[1]}.\n"\ + f"Fatal OSError #{err.errno}: {err.strerror}.\n"\ + f"See the 'Startup failure' section of the IDLE doc, online at\n"\ + f"https://docs.python.org/3/library/idle.html#startup-failure" + showerror("IDLE Subprocess Error", msg, parent=root) root.destroy() def print_exception(): From c2a506e40e9135bfd17b59fffc62261dc95bc535 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Jun 2017 14:26:52 +0200 Subject: [PATCH 0429/1345] [3.6] bpo-30649: test_os tolerates 50 ms delta for utime (#2156) (#2175) * bpo-30649: test_os tolerates 50 ms delta for utime (#2156) On Windows, tolerate a delta of 50 ms instead of 20 ms in test_utime_current() and test_utime_current_old() of test_os. On other platforms, reduce the delta from 20 ms to 10 ms. (cherry picked from commit c94caca65cd38802243b5279cf85ee44ffb2abb8) * bpo-30649: Revert utime delta in test_os (#2176) PPC64 Fedora 3.x buildbot requires at least a delta of 14 ms: revert the utime delta to 20 ms. I tried 10 ms, but test_os failed on the PPC64 Fedora 3.x buildbot. (cherry picked from commit 3402f7268897db15053866e1e68404cfa0e02706) --- Lib/test/test_os.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 46ad2099a9f524..8612ec9edb5356 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -622,8 +622,13 @@ def _test_utime_current(self, set_time): if not self.support_subsecond(self.fname): delta = 1.0 + elif os.name == 'nt': + # On Windows, the usual resolution of time.time() is 15.6 ms. + # bpo-30649: Tolerate 50 ms for slow Windows buildbots. + delta = 0.050 else: - # On Windows, the usual resolution of time.time() is 15.6 ms + # bpo-30649: PPC64 Fedora 3.x buildbot requires + # at least a delta of 14 ms delta = 0.020 st = os.stat(self.fname) msg = ("st_time=%r, current=%r, dt=%r" From 3280579450e3ee94e4916e2543b4dd8898d51b02 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 14 Jun 2017 11:43:49 -0400 Subject: [PATCH 0430/1345] bpo-15786: IDLE: Fix mouse clicks on autocompletetion window (#1811) (#2187) Patch by Louie Lu. (cherry picked from commit 778b484145edfd0d9b65129322d3295bed8eb71a) --- Lib/idlelib/autocomplete_w.py | 58 ++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index 3374c6e94510aa..e7354d0b95ba5a 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -1,6 +1,8 @@ """ An auto-completion window for IDLE, used by the autocomplete extension """ +import platform + from tkinter import * from tkinter.ttk import Scrollbar @@ -8,7 +10,8 @@ from idlelib.multicall import MC_SHIFT HIDE_VIRTUAL_EVENT_NAME = "<>" -HIDE_SEQUENCES = ("", "") +HIDE_FOCUS_OUT_SEQUENCE = "" +HIDE_SEQUENCES = (HIDE_FOCUS_OUT_SEQUENCE, "") KEYPRESS_VIRTUAL_EVENT_NAME = "<>" # We need to bind event beyond so that the function will be called # before the default specific IDLE function @@ -201,10 +204,12 @@ def show_window(self, comp_lists, index, complete, mode, userWantsWin): self._selection_changed() # bind events - self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, - self.hide_event) + self.hideaid = acw.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event) + self.hidewid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event) + acw.event_add(HIDE_VIRTUAL_EVENT_NAME, HIDE_FOCUS_OUT_SEQUENCE) for seq in HIDE_SEQUENCES: self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) + self.keypressid = self.widget.bind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypress_event) for seq in KEYPRESS_SEQUENCES: @@ -240,9 +245,37 @@ def winconfig_event(self, event): new_y -= acw_height acw.wm_geometry("+%d+%d" % (new_x, new_y)) + if platform.system().startswith('Windows'): + # See issue 15786. When on windows platform, Tk will misbehaive + # to call winconfig_event multiple times, we need to prevent this, + # otherwise mouse button double click will not be able to used. + acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + self.winconfigid = None + + def _hide_event_check(self): + if not self.autocompletewindow: + return + + try: + if not self.autocompletewindow.focus_get(): + self.hide_window() + except KeyError: + # See issue 734176, when user click on menu, acw.focus_get() + # will get KeyError. + self.hide_window() + def hide_event(self, event): + # Hide autocomplete list if it exists and does not have focus or + # mouse click on widget / text area. if self.is_active(): - self.hide_window() + if event.type == EventType.FocusOut: + # On windows platform, it will need to delay the check for + # acw.focus_get() when click on acw, otherwise it will return + # None and close the window + self.widget.after(1, self._hide_event_check) + elif event.type == EventType.ButtonPress: + # ButtonPress event only bind to self.widget + self.hide_window() def listselect_event(self, event): if self.is_active(): @@ -391,10 +424,15 @@ def hide_window(self): return # unbind events + self.autocompletewindow.event_delete(HIDE_VIRTUAL_EVENT_NAME, + HIDE_FOCUS_OUT_SEQUENCE) for seq in HIDE_SEQUENCES: self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) - self.hideid = None + + self.autocompletewindow.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideaid) + self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hidewid) + self.hideaid = None + self.hidewid = None for seq in KEYPRESS_SEQUENCES: self.widget.event_delete(KEYPRESS_VIRTUAL_EVENT_NAME, seq) self.widget.unbind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypressid) @@ -405,8 +443,12 @@ def hide_window(self): self.keyreleaseid = None self.listbox.unbind(LISTUPDATE_SEQUENCE, self.listupdateid) self.listupdateid = None - self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) - self.winconfigid = None + if self.winconfigid: + self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + self.winconfigid = None + + # Re-focusOn frame.text (See issue #15786) + self.widget.focus_set() # destroy widgets self.scrollbar.destroy() From dd47b1944df116cbdbadce3af463c8ccf9699418 Mon Sep 17 00:00:00 2001 From: csabella Date: Wed, 14 Jun 2017 12:26:41 -0400 Subject: [PATCH 0431/1345] Fix trivial typo in Python Setup documentation (GH-2185) (GH-2188) Replace platform with platforms. (cherry picked from commit 4ebf03d109f827c91a23256a447c1d74a203dfee) --- Doc/using/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/index.rst b/Doc/using/index.rst index a5df713944cc5d..4f0aa7d9577df6 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -6,7 +6,7 @@ This part of the documentation is devoted to general information on the setup -of the Python environment on different platform, the invocation of the +of the Python environment on different platforms, the invocation of the interpreter and things that make working with Python easier. From 789f47ebb56283bd0156957121ac2854817b2bf9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 14 Jun 2017 11:59:37 -0700 Subject: [PATCH 0432/1345] bpo-30659 : Use ** for kwargs in namedtuple._replace() signature (GH-2173) (GH-2195) (cherry picked from commit 184bd82ba8106785ba22f0d2477dbd08bef821fb) --- Doc/library/collections.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 2d51f0cfdc094d..d6d2056dfc496c 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -866,7 +866,7 @@ field names, the method and attribute names start with an underscore. .. versionchanged:: 3.1 Returns an :class:`OrderedDict` instead of a regular :class:`dict`. -.. method:: somenamedtuple._replace(kwargs) +.. method:: somenamedtuple._replace(**kwargs) Return a new instance of the named tuple replacing specified fields with new values:: From 6628006941f4382813a9af4628894c99bb333b8f Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 14 Jun 2017 16:10:10 -0400 Subject: [PATCH 0433/1345] [3.6]bpo-15786: Fix IDLE autocomplete return problem. (#2198) (#2199) Before, Enter would not, for instance, complete 're.c' to 're.compile' even with 'compile' highlighted. Now it does. Before, '\n' was inserted into text, which in Shell meant compile() and possibly execute. Now cursor is left after completion. (cherry picked from commit 32fd874afe55e396e3c9a5af35e7bb3d8e0b8f02) --- Lib/idlelib/autocomplete_w.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index e7354d0b95ba5a..7c3138f4040a30 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -325,8 +325,9 @@ def keypress_event(self, event): return "break" elif keysym == "Return": + self.complete() self.hide_window() - return None + return 'break' elif (self.mode == COMPLETE_ATTRIBUTES and keysym in ("period", "space", "parenleft", "parenright", "bracketleft", From 8a39af945796164a16a607fecfef5297b7a87deb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Jun 2017 22:43:01 +0200 Subject: [PATCH 0434/1345] bpo-30231: Remove skipped test_imaplib tests (#1419) (#2192) The public cyrus.andrew.cmu.edu IMAP server (port 993) doesn't accept TLS connection using our self-signed x509 certificate. Remove the two tests which are already skipped. Write a new test_certfile_arg_warn() unit test for the certfile deprecation warning. (cherry picked from commit b18563da8803433509e9a0e29718e0271014659f) --- Lib/test/test_imaplib.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 9df37b1de58361..93ba61a1a99e5d 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -13,7 +13,7 @@ import inspect from test.support import (reap_threads, verbose, transient_internet, - run_with_tz, run_with_locale) + run_with_tz, run_with_locale, cpython_only) import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -504,6 +504,15 @@ def test_ssl_verified(self): ssl_context=ssl_context) client.shutdown() + # Mock the private method _connect(), so mark the test as specific + # to CPython stdlib + @cpython_only + def test_certfile_arg_warn(self): + with support.check_warnings(('', DeprecationWarning)): + with mock.patch.object(self.imap_class, 'open'): + with mock.patch.object(self.imap_class, '_connect'): + self.imap_class('localhost', 143, certfile=CERTFILE) + class ThreadedNetworkedTests(unittest.TestCase): server_class = socketserver.TCPServer imap_class = imaplib.IMAP4 @@ -966,25 +975,6 @@ def test_logincapa(self): _server = self.imap_class(self.host, self.port) self.check_logincapa(_server) - @unittest.skipIf(True, - "bpo-30175: FIXME: cyrus.andrew.cmu.edu doesn't accept " - "our randomly generated client x509 certificate anymore") - def test_logincapa_with_client_certfile(self): - with transient_internet(self.host): - with support.check_warnings(('', DeprecationWarning)): - _server = self.imap_class(self.host, self.port, - certfile=CERTFILE) - self.check_logincapa(_server) - - @unittest.skipIf(True, - "bpo-30175: FIXME: cyrus.andrew.cmu.edu doesn't accept " - "our randomly generated client x509 certificate anymore") - def test_logincapa_with_client_ssl_context(self): - with transient_internet(self.host): - _server = self.imap_class( - self.host, self.port, ssl_context=self.create_ssl_context()) - self.check_logincapa(_server) - def test_logout(self): with transient_internet(self.host): _server = self.imap_class(self.host, self.port) From 86b95370c45dedb8a56c9894372a43681de47a73 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Jun 2017 01:26:16 +0200 Subject: [PATCH 0435/1345] bpo-29591: Upgrade Modules/expat to libexpat 2.2 (#2164) (#2200) * bpo-29591: Upgrade Modules/expat to libexpat 2.2 * bpo-29591: Restore Python changes on expat * bpo-29591: Remove expat config of unsupported platforms Remove the configuration (Modules/expat/*config.h) of unsupported platforms: * Amiga * MacOS Classic on PPC32 * Open Watcom * bpo-29591: Remove useless XML_HAS_SET_HASH_SALT The XML_HAS_SET_HASH_SALT define of Modules/expat/expat.h became useless since our local expat copy was upgrade to expat 2.1 (it's now expat 2.2.0). (cherry picked from commit 23ec4b57e1359f9c539b8defc317542173ae087e) --- Misc/NEWS | 4 + Modules/expat/COPYING | 5 +- Modules/expat/amigaconfig.h | 32 ----- Modules/expat/expat.h | 17 ++- Modules/expat/expat_external.h | 14 ++ Modules/expat/internal.h | 22 ++++ Modules/expat/macconfig.h | 53 -------- Modules/expat/watcomconfig.h | 47 ------- Modules/expat/xmlparse.c | 113 +++++++++++----- Modules/expat/xmlrole.c | 224 ++++++++++++++++---------------- Modules/expat/xmltok.c | 230 +++++++++++++++++++++++---------- Modules/expat/xmltok.h | 10 +- Modules/expat/xmltok_impl.c | 226 ++++++++++++++++---------------- Modules/pyexpat.c | 6 +- 14 files changed, 527 insertions(+), 476 deletions(-) delete mode 100644 Modules/expat/amigaconfig.h delete mode 100644 Modules/expat/macconfig.h delete mode 100644 Modules/expat/watcomconfig.h diff --git a/Misc/NEWS b/Misc/NEWS index f3874140a5249c..22f980ce83099c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,10 @@ Core and Builtins Library ------- +- [Security] bpo-29591: Update expat copy from 2.1.1 to 2.2.0 to get fixes + of CVE-2016-0718 and CVE-2016-4472. See + https://sourceforge.net/p/expat/bugs/537/ for more information. + - bpo-24484: Avoid race condition in multiprocessing cleanup (#2159) - bpo-28994: The traceback no longer displayed for SystemExit raised in diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index dcb4506429684b..092c83baee13ae 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,6 +1,5 @@ -Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers. +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2016 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/amigaconfig.h b/Modules/expat/amigaconfig.h deleted file mode 100644 index 86c611504025fb..00000000000000 --- a/Modules/expat/amigaconfig.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef AMIGACONFIG_H -#define AMIGACONFIG_H - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#define HAVE_BCOPY 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_CHECK_H - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -#endif /* AMIGACONFIG_H */ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index e8eefddc6d8474..086e24b39c51ae 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -342,7 +342,7 @@ XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler); /* OBSOLETE -- OBSOLETE -- OBSOLETE - This handler has been superceded by the EntityDeclHandler above. + This handler has been superseded by the EntityDeclHandler above. It is provided here for backward compatibility. This is called for a declaration of an unparsed (NDATA) entity. @@ -915,8 +915,6 @@ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); -#define XML_HAS_SET_HASH_SALT /* Python Only: Defined for pyexpat.c. */ - /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -975,9 +973,12 @@ XML_FreeContentModel(XML_Parser parser, XML_Content *model); /* Exposing the memory handling functions used in Expat */ XMLPARSEAPI(void *) +XML_ATTR_MALLOC +XML_ATTR_ALLOC_SIZE(2) XML_MemMalloc(XML_Parser parser, size_t size); XMLPARSEAPI(void *) +XML_ATTR_ALLOC_SIZE(3) XML_MemRealloc(XML_Parser parser, void *ptr, size_t size); XMLPARSEAPI(void) @@ -1033,14 +1034,12 @@ XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -/* Expat follows the GNU/Linux convention of odd number minor version for - beta/development releases and even number minor version for stable - releases. Micro is bumped with each release, and set to 0 with each - change to major or minor version. +/* Expat follows the semantic versioning convention. + See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 1 -#define XML_MICRO_VERSION 1 +#define XML_MINOR_VERSION 2 +#define XML_MICRO_VERSION 0 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f337e1c5622a95..4860c5777aa018 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -69,12 +69,26 @@ #endif #endif /* not defined XML_STATIC */ +#if !defined(XMLIMPORT) && defined(__GNUC__) && (__GNUC__ >= 4) +#define XMLIMPORT __attribute__ ((visibility ("default"))) +#endif /* If we didn't define it above, define it away: */ #ifndef XMLIMPORT #define XMLIMPORT #endif +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +#define XML_ATTR_MALLOC __attribute__((__malloc__)) +#else +#define XML_ATTR_MALLOC +#endif + +#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +#else +#define XML_ATTR_ALLOC_SIZE(x) +#endif #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index dd5454831da2b4..94cb98e15cae40 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -71,3 +71,25 @@ #define inline #endif #endif + +#ifndef UNUSED_P +# ifdef __GNUC__ +# define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__)) +# else +# define UNUSED_P(p) UNUSED_ ## p +# endif +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + + +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef); + + +#ifdef __cplusplus +} +#endif diff --git a/Modules/expat/macconfig.h b/Modules/expat/macconfig.h deleted file mode 100644 index 2725caaf54a2e2..00000000000000 --- a/Modules/expat/macconfig.h +++ /dev/null @@ -1,53 +0,0 @@ -/*================================================================ -** Copyright 2000, Clark Cooper -** All rights reserved. -** -** This is free software. You are permitted to copy, distribute, or modify -** it under the terms of the MIT/X license (contained in the COPYING file -** with this distribution.) -** -*/ - -#ifndef MACCONFIG_H -#define MACCONFIG_H - - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#undef HAVE_BCOPY - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE - -/* Define to 1 if you have a working `mmap' system call. */ -#undef HAVE_MMAP - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#undef XML_CONTEXT_BYTES - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -/* Define to empty if `const' does not conform to ANSI C. */ -#undef const - -/* Define to `long' if does not define. */ -#define off_t long - -/* Define to `unsigned' if does not define. */ -#undef size_t - - -#endif /* ifndef MACCONFIG_H */ diff --git a/Modules/expat/watcomconfig.h b/Modules/expat/watcomconfig.h deleted file mode 100644 index 2f05e3f227c162..00000000000000 --- a/Modules/expat/watcomconfig.h +++ /dev/null @@ -1,47 +0,0 @@ -/* expat_config.h for use with Open Watcom 1.5 and above. */ - -#ifndef WATCOMCONFIG_H -#define WATCOMCONFIG_H - -#ifdef __NT__ -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN -#endif - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 1234 - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "expat-bugs@mail.libexpat.org" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "expat" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "expat 2.0.0" - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.0.0" - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD 1 - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS 1 - -#endif - diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 412838794d703b..b308e67e2d390d 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -2,9 +2,22 @@ See the file COPYING for copying permission. */ +#include +#include /* memset(), memcpy() */ +#include +#include /* UINT_MAX */ + +#ifdef WIN32 +#define getpid GetCurrentProcessId +#else +#include /* gettimeofday() */ +#include /* getpid() */ +#include /* getpid() */ +#endif + #define XML_BUILDING_EXPAT 1 -#ifdef COMPILED_FROM_DSP +#ifdef WIN32 #include "winconfig.h" #elif defined(MACOS_CLASSIC) #include "macconfig.h" @@ -14,13 +27,7 @@ #include "watcomconfig.h" #elif defined(HAVE_EXPAT_CONFIG_H) #include -#endif /* ndef COMPILED_FROM_DSP */ - -#include -#include /* memset(), memcpy() */ -#include -#include /* UINT_MAX */ -#include /* time() */ +#endif /* ndef WIN32 */ #include "ascii.h" #include "expat.h" @@ -432,7 +439,7 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static unsigned long generate_hash_secret_salt(void); +static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); static XML_Parser @@ -691,11 +698,38 @@ static const XML_Char implicitContext[] = { }; static unsigned long -generate_hash_secret_salt(void) +gather_time_entropy(void) { - unsigned int seed = time(NULL) % UINT_MAX; - srand(seed); - return rand(); +#ifdef WIN32 + FILETIME ft; + GetSystemTimeAsFileTime(&ft); /* never fails */ + return ft.dwHighDateTime ^ ft.dwLowDateTime; +#else + struct timeval tv; + int gettimeofday_res; + + gettimeofday_res = gettimeofday(&tv, NULL); + assert (gettimeofday_res == 0); + + /* Microseconds time is <20 bits entropy */ + return tv.tv_usec; +#endif +} + +static unsigned long +generate_hash_secret_salt(XML_Parser parser) +{ + /* Process ID is 0 bits entropy if attacker has local access + * XML_Parser address is few bits of entropy if attacker has local access */ + const unsigned long entropy = + gather_time_entropy() ^ getpid() ^ (unsigned long)parser; + + /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ + if (sizeof(unsigned long) == 4) { + return entropy * 2147483647; + } else { + return entropy * (unsigned long)2305843009213693951; + } } static XML_Bool /* only valid for root parser */ @@ -703,7 +737,7 @@ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ if (hash_secret_salt == 0) - hash_secret_salt = generate_hash_secret_salt(); + hash_secret_salt = generate_hash_secret_salt(parser); if (ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1695,15 +1729,15 @@ XML_GetBuffer(XML_Parser parser, int len) if (len > bufferLim - bufferEnd) { #ifdef XML_CONTEXT_BYTES int keep; -#endif - int neededSize = len + (int)(bufferEnd - bufferPtr); +#endif /* defined XML_CONTEXT_BYTES */ + /* Do not invoke signed arithmetic overflow: */ + int neededSize = (int) ((unsigned)len + (unsigned)(bufferEnd - bufferPtr)); if (neededSize < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; } #ifdef XML_CONTEXT_BYTES keep = (int)(bufferPtr - buffer); - if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; neededSize += keep; @@ -1728,7 +1762,8 @@ XML_GetBuffer(XML_Parser parser, int len) if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { - bufferSize *= 2; + /* Do not invoke signed arithmetic overflow: */ + bufferSize = (int) (2U * (unsigned) bufferSize); } while (bufferSize < neededSize && bufferSize > 0); if (bufferSize <= 0) { errorCode = XML_ERROR_NO_MEMORY; @@ -1855,7 +1890,7 @@ XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { if (eventPtr) - return parseEndByteIndex - (parseEndPtr - eventPtr); + return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; } @@ -2429,11 +2464,11 @@ doContent(XML_Parser parser, for (;;) { int bufSize; int convLen; - XmlConvert(enc, + const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if (fromPtr == rawNameEnd) { + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2654,11 +2689,11 @@ doContent(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -3264,11 +3299,11 @@ doCdataSection(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = next; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -4927,9 +4962,9 @@ internalEntityProcessor(XML_Parser parser, static enum XML_Error PTRCALL errorProcessor(XML_Parser parser, - const char *s, - const char *end, - const char **nextPtr) + const char *UNUSED_P(s), + const char *UNUSED_P(end), + const char **UNUSED_P(nextPtr)) { return errorCode; } @@ -5345,6 +5380,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, const char *end) { if (MUST_CONVERT(enc, s)) { + enum XML_Convert_Result convert_res; const char **eventPP; const char **eventEndPP; if (enc == encoding) { @@ -5357,11 +5393,11 @@ reportDefault(XML_Parser parser, const ENCODING *enc, } do { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); + convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; defaultHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); *eventPP = s; - } while (s != end); + } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else defaultHandler(handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s)); @@ -6166,8 +6202,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, if (!pool->ptr && !poolGrow(pool)) return NULL; for (;;) { - XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); - if (ptr == end) + const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; if (!poolGrow(pool)) return NULL; @@ -6251,8 +6287,13 @@ poolGrow(STRING_POOL *pool) } } if (pool->blocks && pool->start == pool->blocks->s) { - int blockSize = (int)(pool->end - pool->start)*2; - BLOCK *temp = (BLOCK *) + BLOCK *temp; + int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + + if (blockSize < 0) + return XML_FALSE; + + temp = (BLOCK *) pool->mem->realloc_fcn(pool->blocks, (offsetof(BLOCK, s) + blockSize * sizeof(XML_Char))); @@ -6267,6 +6308,10 @@ poolGrow(STRING_POOL *pool) else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + + if (blockSize < 0) + return XML_FALSE; + if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; else diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 44772e21dd337b..fcd0dc6948f442 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -4,7 +4,7 @@ #include -#ifdef COMPILED_FROM_DSP +#ifdef WIN32 #include "winconfig.h" #elif defined(MACOS_CLASSIC) #include "macconfig.h" @@ -16,7 +16,7 @@ #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef WIN32 */ #include "expat_external.h" #include "internal.h" @@ -195,9 +195,9 @@ prolog1(PROLOG_STATE *state, static int PTRCALL prolog2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -216,9 +216,9 @@ prolog2(PROLOG_STATE *state, static int PTRCALL doctype0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -264,9 +264,9 @@ doctype1(PROLOG_STATE *state, static int PTRCALL doctype2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -281,9 +281,9 @@ doctype2(PROLOG_STATE *state, static int PTRCALL doctype3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -298,9 +298,9 @@ doctype3(PROLOG_STATE *state, static int PTRCALL doctype4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -318,9 +318,9 @@ doctype4(PROLOG_STATE *state, static int PTRCALL doctype5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -437,9 +437,9 @@ externalSubset1(PROLOG_STATE *state, static int PTRCALL entity0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -457,9 +457,9 @@ entity0(PROLOG_STATE *state, static int PTRCALL entity1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -502,9 +502,9 @@ entity2(PROLOG_STATE *state, static int PTRCALL entity3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -519,9 +519,9 @@ entity3(PROLOG_STATE *state, static int PTRCALL entity4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -559,9 +559,9 @@ entity5(PROLOG_STATE *state, static int PTRCALL entity6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -605,9 +605,9 @@ entity7(PROLOG_STATE *state, static int PTRCALL entity8(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -622,9 +622,9 @@ entity8(PROLOG_STATE *state, static int PTRCALL entity9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -639,9 +639,9 @@ entity9(PROLOG_STATE *state, static int PTRCALL entity10(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -656,9 +656,9 @@ entity10(PROLOG_STATE *state, static int PTRCALL notation0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -697,9 +697,9 @@ notation1(PROLOG_STATE *state, static int PTRCALL notation2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -714,9 +714,9 @@ notation2(PROLOG_STATE *state, static int PTRCALL notation3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -732,9 +732,9 @@ notation3(PROLOG_STATE *state, static int PTRCALL notation4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -753,9 +753,9 @@ notation4(PROLOG_STATE *state, static int PTRCALL attlist0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -771,9 +771,9 @@ attlist0(PROLOG_STATE *state, static int PTRCALL attlist1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -833,9 +833,9 @@ attlist2(PROLOG_STATE *state, static int PTRCALL attlist3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -852,9 +852,9 @@ attlist3(PROLOG_STATE *state, static int PTRCALL attlist4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -872,9 +872,9 @@ attlist4(PROLOG_STATE *state, static int PTRCALL attlist5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -889,9 +889,9 @@ attlist5(PROLOG_STATE *state, static int PTRCALL attlist6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -906,9 +906,9 @@ attlist6(PROLOG_STATE *state, static int PTRCALL attlist7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -967,9 +967,9 @@ attlist8(PROLOG_STATE *state, static int PTRCALL attlist9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -984,9 +984,9 @@ attlist9(PROLOG_STATE *state, static int PTRCALL element0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1072,9 +1072,9 @@ element2(PROLOG_STATE *state, static int PTRCALL element3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1097,9 +1097,9 @@ element3(PROLOG_STATE *state, static int PTRCALL element4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1115,9 +1115,9 @@ element4(PROLOG_STATE *state, static int PTRCALL element5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1136,9 +1136,9 @@ element5(PROLOG_STATE *state, static int PTRCALL element6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1166,9 +1166,9 @@ element6(PROLOG_STATE *state, static int PTRCALL element7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1240,9 +1240,9 @@ condSect0(PROLOG_STATE *state, static int PTRCALL condSect1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1258,9 +1258,9 @@ condSect1(PROLOG_STATE *state, static int PTRCALL condSect2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1277,9 +1277,9 @@ condSect2(PROLOG_STATE *state, static int PTRCALL declClose(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1292,11 +1292,11 @@ declClose(PROLOG_STATE *state, } static int PTRCALL -error(PROLOG_STATE *state, - int tok, - const char *ptr, - const char *end, - const ENCODING *enc) +error(PROLOG_STATE *UNUSED_P(state), + int UNUSED_P(tok), + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { return XML_ROLE_NONE; } diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index bf09dfc72b9fc8..a29d9e2f8ac386 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -4,7 +4,7 @@ #include -#ifdef COMPILED_FROM_DSP +#ifdef WIN32 #include "winconfig.h" #elif defined(MACOS_CLASSIC) #include "macconfig.h" @@ -16,7 +16,7 @@ #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef WIN32 */ #include "expat_external.h" #include "internal.h" @@ -46,7 +46,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1 << ((lo) & 0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -56,7 +56,7 @@ (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \ + ((((byte)[0]) & 3) << 1) \ + ((((byte)[1]) >> 5) & 1)] \ - & (1 << (((byte)[1]) & 0x1F))) + & (1u << (((byte)[1]) & 0x1F))) /* A 3 byte UTF-8 representation splits the characters 16 bits between the bottom 4, 6 and 6 bits of the bytes. We need 8 bits to index @@ -69,7 +69,7 @@ << 3) \ + ((((byte)[1]) & 3) << 1) \ + ((((byte)[2]) >> 5) & 1)] \ - & (1 << (((byte)[2]) & 0x1F))) + & (1u << (((byte)[2]) & 0x1F))) #define UTF8_GET_NAMING(pages, p, n) \ ((n) == 2 \ @@ -122,19 +122,19 @@ ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) static int PTRFASTCALL -isNever(const ENCODING *enc, const char *p) +isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p)) { return 0; } static int PTRFASTCALL -utf8_isName2(const ENCODING *enc, const char *p) +utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isName3(const ENCODING *enc, const char *p) +utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); } @@ -142,13 +142,13 @@ utf8_isName3(const ENCODING *enc, const char *p) #define utf8_isName4 isNever static int PTRFASTCALL -utf8_isNmstrt2(const ENCODING *enc, const char *p) +utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isNmstrt3(const ENCODING *enc, const char *p) +utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); } @@ -156,19 +156,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) #define utf8_isNmstrt4 isNever static int PTRFASTCALL -utf8_isInvalid2(const ENCODING *enc, const char *p) +utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID2((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid3(const ENCODING *enc, const char *p) +utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID3((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid4(const ENCODING *enc, const char *p) +utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID4((const unsigned char *)p); } @@ -222,6 +222,17 @@ struct normal_encoding { E ## isInvalid3, \ E ## isInvalid4 +#define NULL_VTABLE \ + /* isName2 */ NULL, \ + /* isName3 */ NULL, \ + /* isName4 */ NULL, \ + /* isNmstrt2 */ NULL, \ + /* isNmstrt3 */ NULL, \ + /* isNmstrt4 */ NULL, \ + /* isInvalid2 */ NULL, \ + /* isInvalid3 */ NULL, \ + /* isInvalid4 */ NULL + static int FASTCALL checkCharRefNumber(int); #include "xmltok_impl.h" @@ -318,39 +329,89 @@ enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */ UTF8_cval4 = 0xf0 }; -static void PTRCALL -utf8_toUtf8(const ENCODING *enc, +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef) +{ + const char * fromLim = *fromLimRef; + size_t walked = 0; + for (; fromLim > from; fromLim--, walked++) { + const unsigned char prev = (unsigned char)fromLim[-1]; + if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */ + if (walked + 1 >= 4) { + fromLim += 4 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */ + if (walked + 1 >= 3) { + fromLim += 3 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */ + if (walked + 1 >= 2) { + fromLim += 2 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */ + break; + } + } + *fromLimRef = fromLim; +} + +static enum XML_Convert_Result PTRCALL +utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; char *to; const char *from; if (fromLim - *fromP > toLim - *toP) { /* Avoid copying partial characters. */ - for (fromLim = *fromP + (toLim - *toP); fromLim > *fromP; fromLim--) - if (((unsigned char)fromLim[-1] & 0xc0) != 0x80) - break; + res = XML_CONVERT_OUTPUT_EXHAUSTED; + fromLim = *fromP + (toLim - *toP); + align_limit_to_full_utf8_characters(*fromP, &fromLim); } - for (to = *toP, from = *fromP; from != fromLim; from++, to++) + for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++) *to = *from; *fromP = from; *toP = to; + + if ((to == toLim) && (from < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return res; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; unsigned short *to = *toP; const char *from = *fromP; - while (from != fromLim && to != toLim) { + while (from < fromLim && to < toLim) { switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { case BT_LEAD2: + if (fromLim - from < 2) { + res = XML_CONVERT_INPUT_INCOMPLETE; + break; + } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; break; case BT_LEAD3: + if (fromLim - from < 3) { + res = XML_CONVERT_INPUT_INCOMPLETE; + break; + } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); from += 3; @@ -358,8 +419,14 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD4: { unsigned long n; - if (to + 1 == toLim) + if (toLim - to < 2) { + res = XML_CONVERT_OUTPUT_EXHAUSTED; goto after; + } + if (fromLim - from < 4) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12) | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f); n -= 0x10000; @@ -377,6 +444,7 @@ utf8_toUtf16(const ENCODING *enc, after: *fromP = from; *toP = to; + return res; } #ifdef XML_NS @@ -425,38 +493,43 @@ static const struct normal_encoding internal_utf8_encoding = { STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_) }; -static void PTRCALL -latin1_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { for (;;) { unsigned char c; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; c = (unsigned char)**fromP; if (c & 0x80) { if (toLim - *toP < 2) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = (char)((c >> 6) | UTF8_cval2); *(*toP)++ = (char)((c & 0x3f) | 0x80); (*fromP)++; } else { if (*toP == toLim) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = *(*fromP)++; } } } -static void PTRCALL -latin1_toUtf16(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf16(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = (unsigned char)*(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -467,7 +540,7 @@ static const struct normal_encoding latin1_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -480,16 +553,21 @@ static const struct normal_encoding latin1_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; -static void PTRCALL -ascii_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +ascii_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = *(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -500,7 +578,7 @@ static const struct normal_encoding ascii_encoding_ns = { #include "asciitab.h" /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -513,7 +591,7 @@ static const struct normal_encoding ascii_encoding = { #undef BT_COLON /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; static int PTRFASTCALL @@ -536,13 +614,14 @@ unicode_byte_type(char hi, char lo) } #define DEFINE_UTF16_TO_UTF8(E) \ -static void PTRCALL \ -E ## toUtf8(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf8(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ char **toP, const char *toLim) \ { \ - const char *from; \ - for (from = *fromP; from != fromLim; from += 2) { \ + const char *from = *fromP; \ + fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */ \ + for (; from < fromLim; from += 2) { \ int plane; \ unsigned char lo2; \ unsigned char lo = GET_LO(from); \ @@ -552,7 +631,7 @@ E ## toUtf8(const ENCODING *enc, \ if (lo < 0x80) { \ if (*toP == toLim) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = lo; \ break; \ @@ -562,7 +641,7 @@ E ## toUtf8(const ENCODING *enc, \ case 0x4: case 0x5: case 0x6: case 0x7: \ if (toLim - *toP < 2) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2); \ *(*toP)++ = ((lo & 0x3f) | 0x80); \ @@ -570,7 +649,7 @@ E ## toUtf8(const ENCODING *enc, \ default: \ if (toLim - *toP < 3) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \ *(*toP)++ = ((hi >> 4) | UTF8_cval3); \ @@ -580,7 +659,11 @@ E ## toUtf8(const ENCODING *enc, \ case 0xD8: case 0xD9: case 0xDA: case 0xDB: \ if (toLim - *toP < 4) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + } \ + if (fromLim - from < 4) { \ + *fromP = from; \ + return XML_CONVERT_INPUT_INCOMPLETE; \ } \ plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \ *(*toP)++ = ((plane >> 2) | UTF8_cval4); \ @@ -596,20 +679,32 @@ E ## toUtf8(const ENCODING *enc, \ } \ } \ *fromP = from; \ + if (from < fromLim) \ + return XML_CONVERT_INPUT_INCOMPLETE; \ + else \ + return XML_CONVERT_COMPLETED; \ } #define DEFINE_UTF16_TO_UTF16(E) \ -static void PTRCALL \ -E ## toUtf16(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf16(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) \ { \ + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ + fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ /* Avoid copying first half only of surrogate */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) \ + && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ fromLim -= 2; \ - for (; *fromP != fromLim && *toP != toLim; *fromP += 2) \ + res = XML_CONVERT_INPUT_INCOMPLETE; \ + } \ + for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \ *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \ + if ((*toP == toLim) && (*fromP < fromLim)) \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + else \ + return res; \ } #define SET2(ptr, ch) \ @@ -726,7 +821,7 @@ static const struct normal_encoding little2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -745,7 +840,7 @@ static const struct normal_encoding little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #if BYTEORDER != 4321 @@ -758,7 +853,7 @@ static const struct normal_encoding internal_little2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -771,7 +866,7 @@ static const struct normal_encoding internal_little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -867,7 +962,7 @@ static const struct normal_encoding big2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -886,7 +981,7 @@ static const struct normal_encoding big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #if BYTEORDER != 1234 @@ -899,7 +994,7 @@ static const struct normal_encoding internal_big2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -912,7 +1007,7 @@ static const struct normal_encoding internal_big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -938,7 +1033,7 @@ streqci(const char *s1, const char *s2) } static void PTRCALL -initUpdatePosition(const ENCODING *enc, const char *ptr, +initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, POSITION *pos) { normal_updatePosition(&utf8_encoding.enc, ptr, end, pos); @@ -1288,7 +1383,7 @@ unknown_isInvalid(const ENCODING *enc, const char *p) return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) @@ -1299,21 +1394,21 @@ unknown_toUtf8(const ENCODING *enc, const char *utf8; int n; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; utf8 = uenc->utf8[(unsigned char)**fromP]; n = *utf8++; if (n == 0) { int c = uenc->convert(uenc->userData, *fromP); n = XmlUtf8Encode(c, buf); if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; utf8 = buf; *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP] - (BT_LEAD2 - 2)); } else { if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; (*fromP)++; } do { @@ -1322,13 +1417,13 @@ unknown_toUtf8(const ENCODING *enc, } } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); - while (*fromP != fromLim && *toP != toLim) { + while (*fromP < fromLim && *toP < toLim) { unsigned short c = uenc->utf16[(unsigned char)**fromP]; if (c == 0) { c = (unsigned short) @@ -1340,6 +1435,11 @@ unknown_toUtf16(const ENCODING *enc, (*fromP)++; *(*toP)++ = c; } + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } ENCODING * @@ -1503,7 +1603,7 @@ initScan(const ENCODING * const *encodingTable, { const ENCODING **encPtr; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; encPtr = enc->encPtr; if (ptr + 1 == end) { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index ca867aa6b429cf..752007e8b9e219 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -130,6 +130,12 @@ typedef int (PTRCALL *SCANNER)(const ENCODING *, const char *, const char **); +enum XML_Convert_Result { + XML_CONVERT_COMPLETED = 0, + XML_CONVERT_INPUT_INCOMPLETE = 1, + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining as well */ +}; + struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; @@ -158,12 +164,12 @@ struct encoding { const char *ptr, const char *end, const char **badPtr); - void (PTRCALL *utf8Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim); - void (PTRCALL *utf16Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 9c2895b87739ef..5f779c0571b5b1 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -87,27 +87,45 @@ #define PREFIX(ident) ident #endif + +#define HAS_CHARS(enc, ptr, end, count) \ + (end - ptr >= count * MINBPC(enc)) + +#define HAS_CHAR(enc, ptr, end) \ + HAS_CHARS(enc, ptr, end, 1) + +#define REQUIRE_CHARS(enc, ptr, end, count) \ + { \ + if (! HAS_CHARS(enc, ptr, end, count)) { \ + return XML_TOK_PARTIAL; \ + } \ + } + +#define REQUIRE_CHAR(enc, ptr, end) \ + REQUIRE_CHARS(enc, ptr, end, 1) + + /* ptr points to character following " */ switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) { case BT_S: case BT_CR: case BT_LF: case BT_PERCNT: @@ -175,7 +191,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, +PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, int *tokPtr) { int upper = 0; @@ -225,15 +241,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, { int tok; const char *target = ptr; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -242,13 +257,12 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_QUEST: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -266,8 +280,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -282,15 +295,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, +PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, const char **nextTokPtr) { static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB }; int i; /* CDATA[ */ - if (end - ptr < 6 * MINBPC(enc)) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 6); for (i = 0; i < 6; i++, ptr += MINBPC(enc)) { if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) { *nextTokPtr = ptr; @@ -305,7 +317,7 @@ static int PTRCALL PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -319,13 +331,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, switch (BYTE_TYPE(enc, ptr)) { case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); break; @@ -334,8 +344,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, return XML_TOK_CDATA_SECT_CLOSE; case BT_CR: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); *nextTokPtr = ptr; @@ -348,7 +357,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -383,19 +392,18 @@ static int PTRCALL PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: case BT_LF: break; @@ -432,7 +440,7 @@ static int PTRCALL PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -441,7 +449,7 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -464,7 +472,7 @@ static int PTRCALL PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { if (CHAR_MATCHES(enc, ptr, ASCII_x)) return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); switch (BYTE_TYPE(enc, ptr)) { @@ -474,7 +482,7 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: break; @@ -496,8 +504,7 @@ static int PTRCALL PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_NUM: @@ -506,7 +513,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -529,7 +536,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon = 0; #endif - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -540,8 +547,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -555,8 +561,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, int t; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == BT_EQUALS) break; @@ -579,8 +584,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #endif for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); open = BYTE_TYPE(enc, ptr); if (open == BT_QUOT || open == BT_APOS) break; @@ -598,8 +602,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* in attribute value */ for (;;) { int t; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == open) break; @@ -624,8 +627,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: @@ -642,8 +644,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to closing quote */ for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -655,8 +656,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -688,13 +688,12 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon; #endif - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_EXCL: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_MINUS: return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -716,7 +715,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, hadColon = 0; #endif /* we have a start-tag */ - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -727,8 +726,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -740,7 +738,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_CR: case BT_LF: { ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_GT: @@ -765,8 +763,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -785,7 +782,7 @@ static int PTRCALL PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -803,7 +800,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); case BT_CR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -814,12 +811,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_NEWLINE; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); @@ -832,7 +829,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -845,12 +842,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) #undef LEAD_CASE case BT_RSQB: - if (ptr + MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 2)) { if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) { ptr += MINBPC(enc); break; } - if (ptr + 2*MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 3)) { if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) { ptr += MINBPC(enc); break; @@ -884,8 +881,7 @@ static int PTRCALL PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_LF: case BT_CR: case BT_PERCNT: @@ -895,7 +891,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -913,15 +909,14 @@ static int PTRCALL PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_CR: case BT_LF: case BT_S: @@ -941,7 +936,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { int t = BYTE_TYPE(enc, ptr); switch (t) { INVALID_CASES(ptr, nextTokPtr) @@ -950,7 +945,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, ptr += MINBPC(enc); if (t != open) break; - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_LITERAL; *nextTokPtr = ptr; switch (BYTE_TYPE(enc, ptr)) { @@ -973,7 +968,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -992,8 +987,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_LT: { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_EXCL: return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -1021,7 +1015,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_LF: for (;;) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) break; switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_LF: @@ -1048,11 +1042,10 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_BRACKET; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_BRACKET; if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if (ptr + MINBPC(enc) == end) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 2); if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) { *nextTokPtr = ptr + 2*MINBPC(enc); return XML_TOK_COND_SECT_CLOSE; @@ -1065,7 +1058,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_PAREN; case BT_RPAR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_PAREN; switch (BYTE_TYPE(enc, ptr)) { case BT_AST: @@ -1141,7 +1134,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_GT: case BT_RPAR: case BT_COMMA: @@ -1154,8 +1147,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); switch (tok) { case XML_TOK_NAME: - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); tok = XML_TOK_PREFIXED_NAME; switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) @@ -1204,10 +1196,12 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) + return XML_TOK_PARTIAL; start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1232,7 +1226,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1262,10 +1256,12 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) + return XML_TOK_PARTIAL; start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1294,7 +1290,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1326,15 +1322,15 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, end = ptr + n; } } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_LT: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_EXCL)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_LSQB)) { ++level; ptr += MINBPC(enc); @@ -1342,11 +1338,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, } break; case BT_RSQB: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr += MINBPC(enc); if (level == 0) { @@ -1373,7 +1369,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, { ptr += MINBPC(enc); end -= MINBPC(enc); - for (; ptr != end; ptr += MINBPC(enc)) { + for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -1521,7 +1517,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, } static int PTRFASTCALL -PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) +PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr) { int result = 0; /* skip &# */ @@ -1565,7 +1561,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) } static int PTRCALL -PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, +PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end) { switch ((end - ptr)/MINBPC(enc)) { @@ -1683,11 +1679,11 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } static int PTRCALL -PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, +PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (ptr1 == end1) + if (end1 - ptr1 < MINBPC(enc)) return 0; if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; @@ -1744,7 +1740,7 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *end, POSITION *pos) { - while (ptr < end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -1760,7 +1756,7 @@ PREFIX(updatePosition)(const ENCODING *enc, case BT_CR: pos->lineNumber++; ptr += MINBPC(enc); - if (ptr != end && BYTE_TYPE(enc, ptr) == BT_LF) + if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); pos->columnNumber = (XML_Size)-1; break; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 47f70a27f15846..47c3e86c209d9c 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,10 +1191,8 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 || defined(XML_HAS_SET_HASH_SALT) - /* This feature was added upstream in libexpat 2.1.0. Our expat copy - * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT - * to indicate that we can still use it. */ +#if XML_COMBINED_VERSION >= 20100 + /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); #endif From 523a243840feb4d5f444a1b128e540876afac3d2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 15 Jun 2017 16:55:22 +0300 Subject: [PATCH 0436/1345] [3.6] bpo-30605: Fix compiling binary regexs with BytesWarnings enabled. (GH-2016) (#2214) Running our unit tests with `-bb` enabled triggered this failure.. (cherry picked from commit 171b9a354e816eebc6d4c3a8553303942e9c5025) --- Lib/sre_parse.py | 2 +- Lib/test/test_re.py | 16 ++++++++++++++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 608f9a26642f05..b505e11a7eea91 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -736,7 +736,7 @@ def _parse(source, state, verbose, nested, first=False): if not first or subpattern: import warnings warnings.warn( - 'Flags not at the start of the expression %s%s' % ( + 'Flags not at the start of the expression %r%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index e88d0b3dcf2a78..a6cbbd0b67abf8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1346,7 +1346,7 @@ def test_inline_flags(self): self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s' % p + 'Flags not at the start of the expression %r' % p ) self.assertEqual(warns.warnings[0].filename, __file__) @@ -1355,10 +1355,22 @@ def test_inline_flags(self): self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s (truncated)' % p[:20] + 'Flags not at the start of the expression %r (truncated)' % p[:20] ) self.assertEqual(warns.warnings[0].filename, __file__) + # bpo-30605: Compiling a bytes instance regex was throwing a BytesWarning + with warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + p = b'A(?i)' + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, b'a')) + self.assertEqual( + str(warns.warnings[0].message), + 'Flags not at the start of the expression %r' % p + ) + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning): self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) with self.assertWarns(DeprecationWarning): diff --git a/Misc/ACKS b/Misc/ACKS index c2ebf7faa40a9e..1f3a0cceb274b2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1663,6 +1663,7 @@ Jakub Wilk Gerald S. Williams Jason Williams John Williams +Roy Williams Sue Williams Carol Willing Steven Willis diff --git a/Misc/NEWS b/Misc/NEWS index 22f980ce83099c..a258f52e9003ca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- bpo-30605: re.compile() no longer raises a BytesWarning when compiling a + bytes instance with misplaced inline modifier. Patch by Roy Williams. + - [Security] bpo-29591: Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more information. From f75f6edb1fdbaeb12f9de368ebdeb1b7d1836433 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 15 Jun 2017 16:57:53 +0300 Subject: [PATCH 0437/1345] [3.6] bpo-30650: Fixed a syntax error: missed right parentheses (GH-2154) (#2215) (cherry picked from commit 0d32218) --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f4bbc8931b39fa..0a9123b61bdc06 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12756,7 +12756,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif #ifdef SCHED_SPORADIC - if (PyModule_AddIntMacro(m, SCHED_SPORADIC) return -1; + if (PyModule_AddIntMacro(m, SCHED_SPORADIC)) return -1; #endif #ifdef SCHED_BATCH if (PyModule_AddIntMacro(m, SCHED_BATCH)) return -1; From cf58dfb44cc11d41ea1473cd7436618b210b8258 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 15 Jun 2017 17:16:38 +0300 Subject: [PATCH 0438/1345] [3.6] bpo-29931 fix __lt__ check in ipaddress.ip_interface for both v4 and v6. (GH-879) (#2217) the original logic was just comparing the network address but this is wrong because if the network address is equal then we need to compare the ip address for breaking the tie add more ip_interface comparison tests. (cherry picked from commit 7bd8d3e794782582a4ad1c9749424fff86802c3e) --- Lib/ipaddress.py | 6 ++++-- Lib/test/test_ipaddress.py | 37 +++++++++++++++++++++++++++++-------- Misc/NEWS | 3 +++ 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 20f33cbdeb67d0..70746f8de85c09 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1410,7 +1410,8 @@ def __lt__(self, other): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. @@ -2100,7 +2101,8 @@ def __lt__(self, other): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 0e39516dc033e3..5d9633024f7064 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1405,14 +1405,35 @@ def testAddressComparison(self): ipaddress.ip_address('::2')) def testInterfaceComparison(self): - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.1')) - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.2')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::1')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::2')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') == + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') < + ipaddress.ip_interface('1.1.1.2/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.1/16')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/24') > + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.2/16')) + + self.assertTrue(ipaddress.ip_interface('::1/64') == + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::1/80')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::2/64')) + self.assertTrue(ipaddress.ip_interface('::2/48') < + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/80') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::2/64') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') > + ipaddress.ip_interface('::2/48')) def testNetworkComparison(self): # ip1 and ip2 have the same network address diff --git a/Misc/NEWS b/Misc/NEWS index a258f52e9003ca..1dc0019239d2c9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- bpo-29931: Fixed comparison check for ipaddress.ip_interface objects. + Patch by Sanjay Sundaresan. + - bpo-30605: re.compile() no longer raises a BytesWarning when compiling a bytes instance with misplaced inline modifier. Patch by Roy Williams. From e45ea377b8d9ae23893d4587003c6d3e7f54b99a Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 15 Jun 2017 23:41:57 +0900 Subject: [PATCH 0439/1345] bpo-30149: Fix partialmethod without explicit self parameter (#1308) (#1662) --- Lib/inspect.py | 13 +++++++++---- Lib/test/test_inspect.py | 35 +++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 2894672f501e8b..3317f58f475dc8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2241,11 +2241,16 @@ def _signature_from_callable(obj, *, sigcls=sigcls) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) - - return sig.replace(parameters=new_params) + if first_wrapped_param.kind is Parameter.VAR_POSITIONAL: + # First argument of the wrapped callable is `*args`, as in + # `partialmethod(lambda *args)`. + return sig + else: + sig_params = tuple(sig.parameters.values()) + assert first_wrapped_param is not sig_params[0] + new_params = (first_wrapped_param,) + sig_params + return sig.replace(parameters=new_params) if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 37e451188a97be..facf040d3cd302 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1989,6 +1989,41 @@ def test(a, b:'foo'=10, *args:'bar', spam:'baz', ham=123, **kwargs:int): ('kwargs', ..., int, "var_keyword")), ...)) + def test_signature_without_self(self): + def test_args_only(*args): # NOQA + pass + + def test_args_kwargs_only(*args, **kwargs): # NOQA + pass + + class A: + @classmethod + def test_classmethod(*args): # NOQA + pass + + @staticmethod + def test_staticmethod(*args): # NOQA + pass + + f1 = functools.partialmethod((test_classmethod), 1) + f2 = functools.partialmethod((test_args_only), 1) + f3 = functools.partialmethod((test_staticmethod), 1) + f4 = functools.partialmethod((test_args_kwargs_only),1) + + self.assertEqual(self.signature(test_args_only), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(test_args_kwargs_only), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + self.assertEqual(self.signature(A.f1), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f2), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f3), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f4), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") diff --git a/Misc/NEWS b/Misc/NEWS index 1dc0019239d2c9..00749f0852bc2c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,10 @@ Core and Builtins Library ------- +- bpo-30149: inspect.signature() now supports callables with + variable-argument parameters wrapped with partialmethod. + Patch by Dong-hee Na. + - bpo-29931: Fixed comparison check for ipaddress.ip_interface objects. Patch by Sanjay Sundaresan. From fab05de21480fc54a5101cf813195cb32724b5ad Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 15 Jun 2017 21:15:11 +0300 Subject: [PATCH 0440/1345] [3.6] bpo-30626: Fix error handling in PyImport_Import(). (GH-2103) (#2221) In rare circumstances PyImport_Import() could return NULL without raising an error. (cherry picked from commit 145541c) --- Python/import.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/import.c b/Python/import.c index cd865a5423666e..a23a1026e11169 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1785,9 +1785,13 @@ PyImport_Import(PyObject *module_name) Py_DECREF(r); modules = PyImport_GetModuleDict(); - r = PyDict_GetItem(modules, module_name); - if (r != NULL) + r = PyDict_GetItemWithError(modules, module_name); + if (r != NULL) { Py_INCREF(r); + } + else if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, module_name); + } err: Py_XDECREF(globals); From e064d4dfeda09fd206653697b70b434e98cc1b57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Jun 2017 00:14:35 +0200 Subject: [PATCH 0441/1345] bpo-30673: test.bat: add -t option (timeout) (#2211) (#2225) (cherry picked from commit 258bfc462b1e58689b43f662a10e44ece3a10bef) --- Tools/buildbot/test.bat | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 0b3a3091c1e46e..dac038fa6995f9 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -1,11 +1,19 @@ @echo off -rem Used by the buildbot "test" step. +rem Script to run the Python test suite used by the "test" step +rem of Windows buildbot slaves. +rem +rem See PCbuild/rt.bat for options, extra options: +rem -t TIMEOUT: set a timeout in seconds + setlocal set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 +rem Use a timeout of 20 minutes per test file by default +set timeout=1200 + :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -13,7 +21,8 @@ if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if "%1"=="-t" (set timeout=%2) & shift & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=%timeout% %regrtest_args% From 50dbf577e10f806056d60ac956db0748d2cc8257 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Jun 2017 00:18:15 +0200 Subject: [PATCH 0442/1345] bpo-23890: Fix ref cycle in TestCase.assertRaises (#858) unittest.TestCase.assertRaises() now manually breaks a reference cycle to not keep objects alive longer than expected. (cherry picked from commit bbd3cf8f1ef1e91a8d6dac6411e18b4b9084abf5) --- Lib/unittest/case.py | 52 ++++++++++++++++++++-------------- Lib/unittest/test/test_case.py | 13 +++++++++ Misc/NEWS | 3 ++ 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index b523f73ff22784..f4dbc52852dc13 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -153,28 +153,32 @@ def handle(self, name, args, kwargs): If args is not empty, call a callable passing positional and keyword arguments. """ - if not _is_subtype(self.expected, self._base_type): - raise TypeError('%s() arg 1 must be %s' % - (name, self._base_type_str)) - if args and args[0] is None: - warnings.warn("callable is None", - DeprecationWarning, 3) - args = () - if not args: - self.msg = kwargs.pop('msg', None) - if kwargs: - warnings.warn('%r is an invalid keyword argument for ' - 'this function' % next(iter(kwargs)), - DeprecationWarning, 3) - return self - - callable_obj, *args = args try: - self.obj_name = callable_obj.__name__ - except AttributeError: - self.obj_name = str(callable_obj) - with self: - callable_obj(*args, **kwargs) + if not _is_subtype(self.expected, self._base_type): + raise TypeError('%s() arg 1 must be %s' % + (name, self._base_type_str)) + if args and args[0] is None: + warnings.warn("callable is None", + DeprecationWarning, 3) + args = () + if not args: + self.msg = kwargs.pop('msg', None) + if kwargs: + warnings.warn('%r is an invalid keyword argument for ' + 'this function' % next(iter(kwargs)), + DeprecationWarning, 3) + return self + + callable_obj, *args = args + try: + self.obj_name = callable_obj.__name__ + except AttributeError: + self.obj_name = str(callable_obj) + with self: + callable_obj(*args, **kwargs) + finally: + # bpo-23890: manually break a reference cycle + self = None class _AssertRaisesContext(_AssertRaisesBaseContext): @@ -725,7 +729,11 @@ def assertRaises(self, expected_exception, *args, **kwargs): self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(expected_exception, self) - return context.handle('assertRaises', args, kwargs) + try: + return context.handle('assertRaises', args, kwargs) + finally: + # bpo-23890: manually break a reference cycle + context = None def assertWarns(self, expected_warning, *args, **kwargs): """Fail unless a warning of class warnClass is triggered diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index 8f752b8ae0d0e2..b84959150542bd 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1273,6 +1273,19 @@ def testAssertRaisesNoExceptionType(self): with self.assertRaises(TypeError): self.assertRaises((ValueError, object)) + def testAssertRaisesRefcount(self): + # bpo-23890: assertRaises() must not keep objects alive longer + # than expected + def func() : + try: + raise ValueError + except ValueError: + raise ValueError + + refcount = sys.getrefcount(func) + self.assertRaises(ValueError, func) + self.assertEqual(refcount, sys.getrefcount(func)) + def testAssertRaisesRegex(self): class ExceptionMock(Exception): pass diff --git a/Misc/NEWS b/Misc/NEWS index 00749f0852bc2c..476a50c9d9b22a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- bpo-23890: unittest.TestCase.assertRaises() now manually breaks a reference + cycle to not keep objects alive longer than expected. + - bpo-30149: inspect.signature() now supports callables with variable-argument parameters wrapped with partialmethod. Patch by Dong-hee Na. From 6a90a124706b73824aebf5c2bd7b27daf82b2342 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 15 Jun 2017 18:49:43 -0700 Subject: [PATCH 0443/1345] bpo-30656: Fix Python C API Module Objects documentation (GH-2170) (GH-2230) `PyModule_New()` now refers to `PyModule_NewObject()` (cherry picked from commit 2d0afef82a07afdb666f2ca0c533aac5d39155cd) --- Doc/c-api/module.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7724350d3c5f95..d3125b86f0729a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -50,7 +50,7 @@ Module Objects .. c:function:: PyObject* PyModule_New(const char *name) - Similar to :c:func:`PyImport_NewObject`, but the name is a UTF-8 encoded + Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded string instead of a Unicode object. From 9cead06076f890d4951b3ea2571408efe76f8b2c Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 15 Jun 2017 19:12:46 -0700 Subject: [PATCH 0444/1345] Check that Python is 64-bit before enabling BLAKE2_USE_SSE. (GH-1332) (GH-2042) (cherry picked from commit 6c991bdee7ec4bedd8c1b8d3812dc884b654b57c) --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f04bf229f1fa97..fe477974bd0b7e 100644 --- a/setup.py +++ b/setup.py @@ -897,8 +897,11 @@ def detect_modules(self): blake2_deps.append('hashlib.h') blake2_macros = [] - if not cross_compiling and os.uname().machine == "x86_64": - # Every x86_64 machine has at least SSE2. + if (not cross_compiling and + os.uname().machine == "x86_64" and + sys.maxsize > 2**32): + # Every x86_64 machine has at least SSE2. Check for sys.maxsize + # in case that kernel is 64-bit but userspace is 32-bit. blake2_macros.append(('BLAKE2_USE_SSE', '1')) exts.append( Extension('_blake2', From 292b421d48ab91b0c68ea4040fc7371e7d6d610e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 15 Jun 2017 19:56:52 -0700 Subject: [PATCH 0445/1345] bpo-28837: Fix lib2to3 handling of map/zip/filter calls when followed with a 'trailer', e.g. zip()[x] (GH-24) (GH-2235) (cherry picked from commit 93b4b47e3a720171d67f3b608de406aef462835c) --- Lib/lib2to3/fixes/fix_filter.py | 21 ++++++++++-- Lib/lib2to3/fixes/fix_map.py | 37 ++++++++++++++++----- Lib/lib2to3/fixes/fix_zip.py | 21 +++++++++--- Lib/lib2to3/tests/test_fixers.py | 56 +++++++++++++++++++++++++++----- Misc/ACKS | 1 + 5 files changed, 110 insertions(+), 26 deletions(-) diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py index bb6718cbf770f5..a7a5a154f6fb11 100644 --- a/Lib/lib2to3/fixes/fix_filter.py +++ b/Lib/lib2to3/fixes/fix_filter.py @@ -15,7 +15,10 @@ # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, ListComp, in_special_context + class FixFilter(fixer_base.ConditionalFix): BM_compatible = True @@ -34,16 +37,19 @@ class FixFilter(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< 'filter' trailer< '(' arglist< none='None' ',' seq=any > ')' > + [extra_trailers=trailer*] > | power< 'filter' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -53,23 +59,32 @@ def transform(self, node, results): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if "filter_lambda" in results: new = ListComp(results.get("fp").clone(), results.get("fp").clone(), results.get("it").clone(), results.get("xp").clone()) + new = Node(syms.power, [new] + trailers, prefix="") elif "none" in results: new = ListComp(Name("_f"), Name("_f"), results["seq"].clone(), Name("_f")) + new = Node(syms.power, [new] + trailers, prefix="") else: if in_special_context(node): return None - new = node.clone() + + args = results['args'].clone() + new = Node(syms.power, [Name("filter"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py index 9f966feedea5aa..78cf81c6f94098 100644 --- a/Lib/lib2to3/fixes/fix_map.py +++ b/Lib/lib2to3/fixes/fix_map.py @@ -22,8 +22,10 @@ # Local imports from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context from ..pygram import python_symbols as syms +from ..pytree import Node + class FixMap(fixer_base.ConditionalFix): BM_compatible = True @@ -32,6 +34,7 @@ class FixMap(fixer_base.ConditionalFix): map_none=power< 'map' trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > + [extra_trailers=trailer*] > | map_lambda=power< @@ -47,10 +50,12 @@ class FixMap(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< - 'map' trailer< '(' [arglist=any] ')' > + 'map' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -60,6 +65,11 @@ def transform(self, node, results): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if node.parent.type == syms.simple_stmt: self.warning(node, "You should use a for loop here") new = node.clone() @@ -69,23 +79,32 @@ def transform(self, node, results): new = ListComp(results["xp"].clone(), results["fp"].clone(), results["it"].clone()) + new = Node(syms.power, [new] + trailers, prefix="") + else: if "map_none" in results: new = results["arg"].clone() + new.prefix = "" else: - if "arglist" in results: - args = results["arglist"] - if args.type == syms.arglist and \ - args.children[0].type == token.NAME and \ - args.children[0].value == "None": + if "args" in results: + args = results["args"] + if args.type == syms.trailer and \ + args.children[1].type == syms.arglist and \ + args.children[1].children[0].type == token.NAME and \ + args.children[1].children[0].value == "None": self.warning(node, "cannot convert map(None, ...) " "with multiple arguments because map() " "now truncates to the shortest sequence") return + + new = Node(syms.power, [Name("map"), args.clone()]) + new.prefix = "" + if in_special_context(node): return None - new = node.clone() + + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) + new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py index 8f36a94fb4f550..52c28df6aab411 100644 --- a/Lib/lib2to3/fixes/fix_zip.py +++ b/Lib/lib2to3/fixes/fix_zip.py @@ -9,13 +9,16 @@ # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, in_special_context + class FixZip(fixer_base.ConditionalFix): BM_compatible = True PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > + power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] > """ @@ -28,8 +31,16 @@ def transform(self, node, results): if in_special_context(node): return None - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) + args = results['args'].clone() + args.prefix = "" + + trailers = [] + if 'trailers' in results: + trailers = [n.clone() for n in results['trailers']] + for n in trailers: + n.prefix = "" + + new = Node(syms.power, [Name("zip"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index b3f2680725961e..3e1a255737ec4b 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2954,10 +2954,23 @@ def test_filter_basic(self): a = """x = [x for x in range(10) if x%2 == 0]""" self.check(b, a) - # XXX This (rare) case is not supported -## b = """x = filter(f, 'abc')[0]""" -## a = """x = list(filter(f, 'abc'))[0]""" -## self.check(b, a) + def test_filter_trailers(self): + b = """x = filter(None, 'abc')[0]""" + a = """x = [_f for _f in 'abc' if _f][0]""" + self.check(b, a) + + b = """x = len(filter(f, 'abc')[0])""" + a = """x = len(list(filter(f, 'abc'))[0])""" + self.check(b, a) + + b = """x = filter(lambda x: x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) + + # Note the parens around x + b = """x = filter(lambda (x): x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) def test_filter_nochange(self): a = """b.join(filter(f, 'abc'))""" @@ -3022,6 +3035,23 @@ def test_prefix_preservation(self): a = """x = list(map( f, 'abc' ))""" self.check(b, a) + def test_map_trailers(self): + b = """x = map(f, 'abc')[0]""" + a = """x = list(map(f, 'abc'))[0]""" + self.check(b, a) + + b = """x = map(None, l)[0]""" + a = """x = list(l)[0]""" + self.check(b, a) + + b = """x = map(lambda x:x, l)[0]""" + a = """x = [x for x in l][0]""" + self.check(b, a) + + b = """x = map(f, 'abc')[0][1]""" + a = """x = list(map(f, 'abc'))[0][1]""" + self.check(b, a) + def test_trailing_comment(self): b = """x = map(f, 'abc') # foo""" a = """x = list(map(f, 'abc')) # foo""" @@ -3066,11 +3096,6 @@ def test_map_basic(self): """ self.warns(b, a, "You should use a for loop here") - # XXX This (rare) case is not supported -## b = """x = map(f, 'abc')[0]""" -## a = """x = list(map(f, 'abc'))[0]""" -## self.check(b, a) - def test_map_nochange(self): a = """b.join(map(f, 'abc'))""" self.unchanged(a) @@ -3130,6 +3155,10 @@ def check(self, b, a): super(Test_zip, self).check(b, a) def test_zip_basic(self): + b = """x = zip()""" + a = """x = list(zip())""" + self.check(b, a) + b = """x = zip(a, b, c)""" a = """x = list(zip(a, b, c))""" self.check(b, a) @@ -3138,6 +3167,15 @@ def test_zip_basic(self): a = """x = len(list(zip(a, b)))""" self.check(b, a) + def test_zip_trailers(self): + b = """x = zip(a, b, c)[0]""" + a = """x = list(zip(a, b, c))[0]""" + self.check(b, a) + + b = """x = zip(a, b, c)[0][1]""" + a = """x = list(zip(a, b, c))[0][1]""" + self.check(b, a) + def test_zip_nochange(self): a = """b.join(zip(a, b))""" self.unchanged(a) diff --git a/Misc/ACKS b/Misc/ACKS index 1f3a0cceb274b2..f1686b3118ff93 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -128,6 +128,7 @@ Andrew Bennetts Andy Bensky Bennett Benson Ezra Berch +Stuart Berg Michel Van den Bergh Julian Berman Brice Berna From 04431c926a88be433203f89e3577addd0d41b602 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 16 Jun 2017 01:06:36 -0500 Subject: [PATCH 0446/1345] bpo-30450: Pull Windows dependencies from GitHub rather than svn (GH-1783) (GH-2237) The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`. This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio. Also fixes an outdated note about _lzma in PCbuild/readme.txt (cherry-picked from commit 51599e2bdd10ab77212a7cbb41a13ea70ee13da8) --- Misc/NEWS | 10 ++- PCbuild/get_external.py | 60 +++++++++++++++++ PCbuild/get_externals.bat | 126 +++++++++++++++++++----------------- PCbuild/readme.txt | 24 ++++--- Tools/msi/get_externals.bat | 118 +++++++++++++++++++++++++++------ 5 files changed, 250 insertions(+), 88 deletions(-) create mode 100644 PCbuild/get_external.py diff --git a/Misc/NEWS b/Misc/NEWS index 476a50c9d9b22a..21457303f2923b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -286,7 +286,7 @@ Tools/Demos Tests ----- -* bpo-30357: test_thread: setUp() now uses support.threading_setup() and +- bpo-30357: test_thread: setUp() now uses support.threading_setup() and support.threading_cleanup() to wait until threads complete to avoid random side effects on following tests. Initial patch written by Grzegorz Grzywacz. @@ -297,6 +297,14 @@ Tests if it doesn't exist) now will be assigned to the target of the "as" clause, if there is one. +Windows +------- + +- bpo-30450: The build process on Windows no longer depends on Subversion, + instead pulling external code from GitHub via a Python script. If Python 3.6 + is not found on the system (via ``py -3.6``), NuGet is used to download a + copy of 32-bit Python. + What's New in Python 3.6.1? =========================== diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py new file mode 100644 index 00000000000000..a682d3849f14c6 --- /dev/null +++ b/PCbuild/get_external.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import zipfile +from urllib.request import urlretrieve + + +def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): + repo = f'cpython-{"bin" if binary else "source"}-deps' + url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' + reporthook = None + if verbose: + reporthook = print + zip_dir.mkdir(parents=True, exist_ok=True) + filename, headers = urlretrieve( + url, + zip_dir / f'{commit_hash}.zip', + reporthook=reporthook, + ) + return filename + + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-b', '--binary', action='store_true', + help='Is the dependency in the binary repo?') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='python') + p.add_argument('-e', '--externals-dir', type=pathlib.Path, + help='Directory in which to store dependencies', + default=pathlib.Path(__file__).parent.parent / 'externals') + p.add_argument('tag', + help='tag of the dependency') + return p.parse_args() + + +def main(): + args = parse_args() + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + binary=args.binary, + verbose=args.verbose, + ) + final_name = args.externals_dir / args.tag + extract_zip(args.externals_dir, zip_path).replace(final_name) + + +if __name__ == '__main__': + main() diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 5767ab2eacf25b..6e466a34484d77 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -2,58 +2,59 @@ setlocal rem Simple script to fetch source for external libraries -if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" -pushd "%~dp0..\externals" +if "%PCBUILD%"=="" (set PCBUILD=%~dp0) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) +if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe) +if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ +set DO_FETCH=true +set DO_CLEAN=false -rem Optionally clean up first. Be warned that this can be very destructive! -if not "%1"=="" ( - for %%c in (-c --clean --clean-only) do ( - if "%1"=="%%c" goto clean - ) - goto usage -) -goto fetch +:CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts +if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts +if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean +if "x%~1" NEQ "x" goto usage +if "%DO_CLEAN%"=="false" goto fetch :clean echo.Cleaning up external libraries. -for /D %%d in ( - bzip2-* - db-* - nasm-* - openssl-* - tcl-* - tcltk* - tk-* - tix-* - sqlite-* - xz-* - ) do ( - echo.Removing %%d - rmdir /s /q %%d -) -if "%1"=="--clean-only" ( - goto end +if exist "%EXTERNALS_DIR%" ( + rem Sometimes this fails the first time; try it twice + rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" ) +if "%DO_FETCH%"=="false" goto end :fetch -rem Fetch current versions - -svn --version > nul 2>&1 -if ERRORLEVEL 9009 ( - echo.svn.exe must be on your PATH. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 + +if "%ORG%"=="" (set ORG=python) + +if "%PYTHON_FOR_BUILD%"=="" ( + echo Checking for installed python... + py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) +) +if "%PYTHON_FOR_BUILD%"=="" ( + if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" + if NOT exist "%NUGET%" ( + echo Downloading nuget... + rem NB: Must use single quotes around NUGET here, NOT double! + rem Otherwise, a space in the path would break things + powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' + ) + echo Installing Python via nuget... + "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" + rem Quote it here; it's not quoted later because "py -3.6" wouldn't work + set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" ) echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 @@ -62,43 +63,48 @@ if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 set libraries=%libraries% xz-5.2.2 for %%e in (%libraries%) do ( - if exist %%e ( + if exist "%EXTERNALS_DIR%\%%e" ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export -q %SVNROOT%%%e + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e + ) +) + +echo.Fetching external binaries... + +set binaries= +set binaries=%binaries% +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 + +for %%b in (%binaries%) do ( + if exist "%EXTERNALS_DIR%\%%b" ( + echo.%%b already exists, skipping. + ) else ( + echo.Fetching %%b... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b ) ) +echo Finished. goto end :usage -echo.invalid argument: %1 -echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo.Valid options: -c, --clean, --clean-only, --organization, --python, +echo.--no-tkinter, --no-openssl echo. -echo.Pull all sources necessary for compiling optional extension modules -echo.that rely on external libraries. Requires svn.exe to be on your PATH -echo.and pulls sources from %SVNROOT%. +echo.Pull all sources and binaries necessary for compiling optional extension +echo.modules that rely on external libraries. echo. -echo.Use the -c or --clean option to clean up all external library sources -echo.before pulling in the current versions. +echo.The --organization option determines which github organization to download +echo.from, the --python option determines which Python 3.6+ interpreter to use +echo.with PCbuild\get_external.py. +echo. +echo.Use the -c or --clean option to remove the entire externals directory. echo. echo.Use the --clean-only option to do the same cleaning, without pulling in echo.anything new. echo. -echo.Only the first argument is checked, all others are ignored. -echo. -echo.**WARNING**: the cleaning options unconditionally remove any directory -echo.that is a child of -echo. %CD% -echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, -echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential -echo.to be very destructive if you are not aware of what it is doing. Use with -echo.caution! -popd exit /b -1 - :end -echo Finished. -popd diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 7c87cbb1ae3a96..f2b59e96bf7228 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -2,9 +2,11 @@ Quick Start Guide ----------------- 1. Install Microsoft Visual Studio 2015, any edition. -2. Install Subversion, and make sure 'svn.exe' is on your PATH. -3. Run "build.bat -e" to build Python in 32-bit Release configuration. -4. (Optional, but recommended) Run the test suite with "rt.bat -q". +1a. Optionally install Python 3.6 or later. If not installed, + get_externals.bat (build.bat -e) will download and use Python via + NuGet. +2. Run "build.bat -e" to build Python in 32-bit Release configuration. +3. (Optional, but recommended) Run the test suite with "rt.bat -q". Building Python using Microsoft Visual C++ @@ -164,8 +166,7 @@ _bz2 Homepage: http://www.bzip.org/ _lzma - Python wrapper for the liblzma compression library, using pre-built - binaries of XZ Utils version 5.0.5 + Python wrapper for version 5.2.2 of the liblzma compression library Homepage: http://tukaani.org/xz/ _ssl @@ -236,9 +237,16 @@ order to download the relevant source files for each project before they can be built. However, a simple script is provided to make this as painless as possible, called "get_externals.bat" and located in this directory. This script extracts all the external sub-projects from - http://svn.python.org/projects/external -via Subversion (so you'll need svn.exe on your PATH) and places them -in ..\externals (relative to this directory). + https://github.com/python/cpython-source-deps +and + https://github.com/python/cpython-bin-deps +via a Python script called "get_external.py", located in this directory. +If Python 3.6 or later is not available via the "py.exe" launcher, the +path or command to use for Python can be provided in the PYTHON_FOR_BUILD +environment variable, or get_externals.bat will download the latest +version of NuGet and use it to download the latest "pythonx86" package +for use with get_external.py. Everything downloaded by these scripts is +stored in ..\externals (relative to this directory). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index e1d74de6ac78f5..aece81fbb1cf3e 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -1,27 +1,107 @@ @echo off setlocal -rem Simple script to fetch source for external tools - -where /Q svn -if ERRORLEVEL 1 ( - echo.svn.exe must be on your PATH to get external tools. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 +rem Simple script to fetch source for external libraries + +set HERE=%~dp0 +if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer) +if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe) +if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) + +set DO_FETCH=true +set DO_CLEAN=false + +:CheckOpts +if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts +if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean +if "x%~1" NEQ "x" goto usage + +if "%DO_CLEAN%"=="false" goto fetch +:clean +echo.Cleaning up external libraries. +if exist "%EXTERNALS_DIR%" ( + rem Sometimes this fails the first time; try it twice + rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" ) -if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals" -pushd "%~dp0..\..\externals" +if "%DO_FETCH%"=="false" goto end +:fetch -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ +if "%ORG%"=="" (set ORG=python) -if not exist "windows-installer\.svn" ( - echo.Checking out installer dependencies to %CD%\windows-installer - svn co %SVNROOT%windows-installer -) else ( - echo.Updating installer dependencies in %CD%\windows-installer - svn up windows-installer +if "%PYTHON_FOR_BUILD%"=="" ( + echo Checking for installed python... + py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) ) +if "%PYTHON_FOR_BUILD%"=="" ( + if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" + if NOT exist "%NUGET%" ( + echo Downloading nuget... + rem NB: Must use single quotes around NUGET here, NOT double! + rem Otherwise, a space in the path would break things + powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' + ) + echo Installing Python via nuget... + "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" + rem Quote it here; it's not quoted later because "py -3.6" wouldn't work + set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" +) + +echo.Fetching external libraries... + +set libraries= + +for %%e in (%libraries%) do ( + if exist "%EXTERNALS_DIR%\%%e" ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e + ) +) + +echo.Fetching external tools... + +set binaries= +rem We always use whatever's latest in the repo for these +set binaries=%binaries% binutils +set binaries=%binaries% gpg +set binaries=%binaries% htmlhelp +set binaries=%binaries% nuget +set binaries=%binaries% redist +set binaries=%binaries% wix + +for %%b in (%binaries%) do ( + if exist "%EXTERNALS_DIR%\%%b" ( + echo.%%b already exists, skipping. + ) else ( + echo.Fetching %%b... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b + ) +) + +echo Finished. +goto end + +:usage +echo.Valid options: -c, --clean, --clean-only, --organization, --python, +echo.--no-tkinter, --no-openssl +echo. +echo.Pull all sources and binaries necessary for compiling optional extension +echo.modules that rely on external libraries. +echo. +echo.The --organization option determines which github organization to download +echo.from, the --python option determines which Python 3.6+ interpreter to use +echo.with PCbuild\get_external.py. +echo. +echo.Use the -c or --clean option to remove the entire externals directory. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +exit /b -1 -popd +:end From 0a4bcf18a785ed79a6faba45c05887762c1f56b4 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 16 Jun 2017 02:11:31 -0400 Subject: [PATCH 0447/1345] [3.6]Add IDLE items to NEWS and idlelib/NEWS.txt (#2239) (#2240) --- Lib/idlelib/NEWS.txt | 16 +++++++++++++++- Misc/NEWS | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index ee2cbacbea646f..f12c185bbbac81 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,7 +1,21 @@ What's New in IDLE 3.6.2 -Released on 2017-06-23? +Released on 2017-06-30? ======================== +bpo-15786: Fix several problems with IDLE's autocompletion box. +The following should now work: clicking on selection box items; +using the scrollbar; selecting an item by hitting Return. +Hangs on MacOSX should no longer happen. Patch by Louie Lu. + +bpo-25514: Add doc subsubsection about IDLE failure to start. +Popup no-connection message directs users to this section. + +bpo-30642: Fix reference leaks in IDLE tests. +Patches by Louie Lu and Terry Jan Reedy. + +bpo-30495: Add docstrings for textview.py and use PEP8 names. +Patches by Cheryl Sabella and Terry Jan Reedy. + bpo-30290: Help-about: use pep8 names and add tests. Increase coverage to 100%. Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. diff --git a/Misc/NEWS b/Misc/NEWS index 21457303f2923b..1c6eb720f9044e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -228,6 +228,20 @@ Library IDLE ---- +- bpo-15786: Fix several problems with IDLE's autocompletion box. + The following should now work: clicking on selection box items; + using the scrollbar; selecting an item by hitting Return. + Hangs on MacOSX should no longer happen. Patch by Louie Lu. + +- bpo-25514: Add doc subsubsection about IDLE failure to start. + Popup no-connection message directs users to this section. + +- bpo-30642: Fix reference leaks in IDLE tests. + Patches by Louie Lu and Terry Jan Reedy. + +- bpo-30495: Add docstrings for textview.py and use PEP8 names. + Patches by Cheryl Sabella and Terry Jan Reedy. + - bpo-30290: Help-about: use pep8 names and add tests. Increase coverage to 100%. Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. From 085a57a8d63b3c8057b5f793518e3d04179710a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Jun 2017 14:38:32 +0200 Subject: [PATCH 0448/1345] Revert "bpo-30673: test.bat: add -t option (timeout) (#2211) (#2225)" (#2246) This reverts commit e064d4dfeda09fd206653697b70b434e98cc1b57. --- Tools/buildbot/test.bat | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index dac038fa6995f9..0b3a3091c1e46e 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -1,19 +1,11 @@ @echo off -rem Script to run the Python test suite used by the "test" step -rem of Windows buildbot slaves. -rem -rem See PCbuild/rt.bat for options, extra options: -rem -t TIMEOUT: set a timeout in seconds - +rem Used by the buildbot "test" step. setlocal set here=%~dp0 set rt_opts=-q -d set regrtest_args=-j1 -rem Use a timeout of 20 minutes per test file by default -set timeout=1200 - :CheckOpts if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts @@ -21,8 +13,7 @@ if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if "%1"=="-t" (set timeout=%2) & shift & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=%timeout% %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args% From a0ccc54e6dffacf9e7c06f2a3e9056d2d35d21eb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Jun 2017 14:39:09 +0200 Subject: [PATCH 0449/1345] Synchronize libregrtest from master to 3.6 (#2244) * bpo-30523: regrtest: Add --list-cases option (#2238) * bpo-30284: Fix regrtest for out of tree build (#1481) * bpo-30540: regrtest: add --matchfile option (#1909) * bpo-30258: regrtest: Fix run_tests_multiprocess() (#1479) * bpo-30263: regrtest: log system load (#1452) --- Lib/test/libregrtest/cmdline.py | 25 +++++- Lib/test/libregrtest/main.py | 119 +++++++++++++++++++---------- Lib/test/libregrtest/refleak.py | 2 +- Lib/test/libregrtest/runtest.py | 14 ++-- Lib/test/libregrtest/runtest_mp.py | 11 ++- Lib/test/support/__init__.py | 10 ++- Lib/test/test_regrtest.py | 105 ++++++++++++++++++++++++- 7 files changed, 228 insertions(+), 58 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index cdbd1b8b92553c..bf64062ef964b8 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -117,6 +117,13 @@ To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. + +--matchfile filters tests using a text file, one pattern per line. +Pattern examples: + +- test method: test_stat_attributes +- test class: FileTests +- test identifier: test_os.FileTests.test_stat_attributes """ @@ -189,8 +196,12 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', + dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('--matchfile', metavar='FILENAME', + dest='match_filename', + help='similar to --match but get patterns from a ' + 'text file, one pattern per line') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -239,6 +250,9 @@ def _create_parser(): group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") + group.add_argument('--list-cases', action='store_true', + help='only write the name of test cases that will be run' + ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization training') @@ -343,10 +357,19 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.verbose: + ns.header = True if ns.huntrleaks and ns.verbose3: ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks: see http://bugs.python.org/issue27103", file=sys.stderr) + if ns.match_filename: + if ns.match_tests is None: + ns.match_tests = [] + filename = os.path.join(support.SAVEDCWD, ns.match_filename) + with open(filename) as fp: + for line in fp: + ns.match_tests.append(line.strip()) return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f0effc973c1b3a..527de177792ce5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,9 +10,10 @@ import tempfile import textwrap import time +import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest, get_abs_module, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, format_test_result) @@ -28,7 +29,13 @@ # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') + TEMPDIR = sysconfig.get_config_var('abs_builddir') + if TEMPDIR is None: + # bpo-30284: On Windows, only srcdir is available. Using abs_builddir + # mostly matters on UNIX when building Python out of the source tree, + # especially when the source tree is read only. + TEMPDIR = sysconfig.get_config_var('srcdir') + TEMPDIR = os.path.join(TEMPDIR, 'build') else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) @@ -107,7 +114,7 @@ def accumulate_result(self, test, result): self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) - elif ok == FAILED: + elif ok in (FAILED, CHILD_ERROR): self.bad.append(test) elif ok == ENV_CHANGED: self.environment_changed.append(test) @@ -116,22 +123,28 @@ def accumulate_result(self, test, result): elif ok == RESOURCE_DENIED: self.skipped.append(test) self.resource_denieds.append(test) + elif ok != INTERRUPTED: + raise ValueError("invalid test result: %r" % ok) def display_progress(self, test_index, test): if self.ns.quiet: return + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count}" if self.bad and not self.ns.pgo: - fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" - else: - fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + line = f"{line}/{len(self.bad)}" + line = f"[{line}] {test}" + + # add the system load prefix: "load avg: 1.80 " + if hasattr(os, 'getloadavg'): + load_avg_1min = os.getloadavg()[0] + line = f"load avg: {load_avg_1min:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " test_time = time.monotonic() - self.start_time test_time = datetime.timedelta(seconds=int(test_time)) - line = fmt.format(count_width=self.test_count_width, - test_index=test_index, - test_count=self.test_count, - nbad=len(self.bad), - test_name=test, - time=test_time) + line = f"{test_time} {line}" print(line, flush=True) def parse_args(self, kwargs): @@ -179,19 +192,14 @@ def find_tests(self, tests): self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' - regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' - r'(?:\[[0-9/ ]+\] *)?' - r'(test_[a-zA-Z0-9_]+)') - regex = re.compile(regex) + regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: for line in fp: + line = line.split('#', 1)[0] line = line.strip() - if line.startswith('#'): - continue - match = regex.match(line) - if match is None: - continue - self.tests.append(match.group(1)) + match = regex.search(line) + if match is not None: + self.tests.append(match.group()) removepy(self.tests) @@ -241,6 +249,29 @@ def list_tests(self): for name in self.selected: print(name) + def _list_cases(self, suite): + for test in suite: + if isinstance(test, unittest.loader._FailedTest): + continue + if isinstance(test, unittest.TestSuite): + self._list_cases(test) + elif isinstance(test, unittest.TestCase): + print(test.id()) + + def list_cases(self): + for test in self.selected: + abstest = get_abs_module(self.ns, test) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(abstest) + self._list_cases(suite) + except unittest.SkipTest: + self.skipped.append(test) + + if self.skipped: + print(file=sys.stderr) + print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) + printlist(self.skipped, file=sys.stderr) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -381,23 +412,28 @@ def _test_forever(self, tests): if self.bad: return + def display_header(self): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== cwd:", os.getcwd()) + cpu_count = os.cpu_count() + if cpu_count: + print("== CPU count:", cpu_count) + print("== encodings: locale=%s, FS=%s" + % (locale.getpreferredencoding(False), + sys.getfilesystemencoding())) + print("Testing with flags:", sys.flags) + def run_tests(self): # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== cwd:", os.getcwd()) - print("== encodings: locale=%s, FS=%s" - % (locale.getpreferredencoding(False), - sys.getfilesystemencoding())) - print("Testing with flags:", sys.flags) + if (self.ns.header + or not(self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + self.display_header() if self.ns.randomize: print("Using random seed", self.ns.random_seed) @@ -487,6 +523,10 @@ def _main(self, tests, kwargs): self.list_tests() sys.exit(0) + if self.ns.list_cases: + self.list_cases() + sys.exit(0) + self.run_tests() self.display_result() @@ -513,7 +553,7 @@ def count(n, word): return "%d %ss" % (n, word) -def printlist(x, width=70, indent=4): +def printlist(x, width=70, indent=4, file=None): """Print the elements of iterable x to stdout. Optional arg width (default 70) is the maximum line length. @@ -524,7 +564,8 @@ def printlist(x, width=70, indent=4): blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + initial_indent=blanks, subsequent_indent=blanks), + file=file) def main(tests=None, **kwargs): diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index e2e58bd872f094..c69fc0f147706b 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -84,7 +84,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): indirect_test() alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - print('.', end='', flush=True) + print('.', end='', file=sys.stderr, flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ba0df0a317a500..fda4ca1ebca094 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -71,6 +71,14 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def get_abs_module(ns, test): + if test.startswith('test.') or ns.testdir: + return test + else: + # Always import it from the test package + return 'test.' + test + + def runtest(ns, test): """Run a single test. @@ -141,11 +149,7 @@ def runtest_inner(ns, test, display_failure=True): test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.') or ns.testdir: - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test + abstest = get_abs_module(ns, test) clear_caches() with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: start_time = time.time() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 9604c166009fe9..779ff01a649c6c 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -41,7 +41,7 @@ def run_test_in_subprocess(testname, ns): slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), - '-X', 'faulthandler', + '-u', # Unbuffered stdout and stderr '-m', 'test.regrtest', '--slaveargs', slaveargs] if ns.pgo: @@ -124,13 +124,13 @@ def _runtest(self): finally: self.current_test = None - stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return True + return False + stdout, _, result = stdout.strip().rpartition("\n") if not result: self.output.put((None, None, None, None)) return True @@ -203,6 +203,8 @@ def get_running(workers): and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): text += ' (%.0f sec)' % test_time + elif ok == CHILD_ERROR: + text = '%s (%s)' % (text, test_time) running = get_running(workers) if running and not regrtest.ns.pgo: text += ' -- running: %s' % ', '.join(running) @@ -216,9 +218,6 @@ def get_running(workers): if result[0] == INTERRUPTED: raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - msg = "Child error on {}: {}".format(test, result[1]) - raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f8d177990bfb03..be926ea6c71d2a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1915,9 +1915,15 @@ def run_unittest(*classes): def case_pred(test): if match_tests is None: return True - for name in test.id().split("."): - if fnmatch.fnmatchcase(name, match_tests): + test_id = test.id() + + for match_test in match_tests: + if fnmatch.fnmatchcase(test_id, match_test): return True + + for name in test_id.split("."): + if fnmatch.fnmatchcase(name, match_test): + return True return False _filter_suite(suite, case_pred) _run_suite(suite) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 751df1537d2c9b..5c6154a347d332 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -118,6 +118,9 @@ def test_header(self): ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) + ns = libregrtest._parse_args(['--verbose']) + self.assertTrue(ns.header) + def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): @@ -156,9 +159,24 @@ def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, 'pattern') + self.assertEqual(ns.match_tests, ['pattern']) self.checkError([opt], 'expected one argument') + ns = libregrtest._parse_args(['-m', 'pattern1', + '-m', 'pattern2']) + self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--matchfile', filename]) + self.assertEqual(ns.match_tests, + ['match', 'matchfile1', 'matchfile2']) + def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): @@ -272,7 +290,6 @@ def test_forever(self): ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) - def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') @@ -354,7 +371,7 @@ def check_line(self, output, regex): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + regex = (r'^[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' % self.TESTNAME_REGEX) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) @@ -454,7 +471,6 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): args = [sys.executable, '-X', 'faulthandler', '-I', *args] proc = self.run_command(args, **kw) @@ -676,6 +692,14 @@ def test_fromfile(self): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + # test format 'Lib/test/test_opcodes.py' + with open(filename, "w") as fp: + for name in tests: + print('Lib/test/%s.py' % name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) @@ -801,6 +825,79 @@ def test_list_tests(self): self.assertEqual(output.rstrip().splitlines(), tests) + def test_list_cases(self): + # test --list-cases + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + """) + testname = self.create_test(code=code) + all_methods = ['%s.Tests.test_method1' % testname, + '%s.Tests.test_method2' % testname] + output = self.run_tests('--list-cases', testname) + self.assertEqual(output.splitlines(), all_methods) + + def test_crashed(self): + # Any code which causes a crash + code = 'import faulthandler; faulthandler._sigsegv()' + crash_test = self.create_test(name="crash", code=code) + ok_test = self.create_test(name="ok") + + tests = [crash_test, ok_test] + output = self.run_tests("-j2", *tests, exitcode=1) + self.check_executed_tests(output, tests, failed=crash_test, + randomize=True) + + def parse_methods(self, output): + regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) + return [match.group(1) for match in regex.finditer(output)] + + def test_matchfile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # by default, all methods should be run + output = self.run_tests("-v", testname) + methods = self.parse_methods(output) + self.assertEqual(methods, all_methods) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only match the method name + 'test_method1', + # match the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--matchfile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method1', 'test_method3'] + self.assertEqual(methods, subset) + if __name__ == '__main__': unittest.main() From 2eca5b465f7404cc8484457b7966f828f434ec20 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 16 Jun 2017 16:29:42 +0300 Subject: [PATCH 0450/1345] [3.6] bpo-30682: Removed a too-strict assertion that failed for certain f-strings. (GH-2232) (#2242) This caused a segfault on eval("f'\\\n'") and eval("f'\\\r'") in debug build.. (cherry picked from commit 11e97f2f80bf65cc828c127eafc95229df35d403) --- Lib/test/test_fstring.py | 6 ++++++ Misc/NEWS | 3 +++ Python/ast.c | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b39870457af492..3d762b5aa9bf63 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -780,5 +780,11 @@ def test_dict(self): self.assertEqual(f'{d["foo"]}', 'bar') self.assertEqual(f"{d['foo']}", 'bar') + def test_backslash_char(self): + # Check eval of a backslash followed by a control char. + # See bpo-30682: this used to raise an assert in pydebug mode. + self.assertEqual(eval('f"\\\n"'), '') + self.assertEqual(eval('f"\\\r"'), '') + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 1c6eb720f9044e..fd7ebf92dc5627 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-30682: Removed a too-strict assertion that failed for certain f-strings, + such as eval("f'\\\n'") and eval("f'\\\r'"). + - bpo-30604: Move co_extra_freefuncs to not be per-thread to avoid crashes - bpo-29104: Fixed parsing backslashes in f-strings. diff --git a/Python/ast.c b/Python/ast.c index c61ca4bbcd5528..8d53736f081b80 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4887,6 +4887,8 @@ FstringParser_ConcatFstring(FstringParser *state, const char **str, /* Do nothing. Just leave last_str alone (and possibly NULL). */ } else if (!state->last_str) { + /* Note that the literal can be zero length, if the + input string is "\\\n" or "\\\r", among others. */ state->last_str = literal; literal = NULL; } else { @@ -4896,8 +4898,6 @@ FstringParser_ConcatFstring(FstringParser *state, const char **str, return -1; literal = NULL; } - assert(!state->last_str || - PyUnicode_GET_LENGTH(state->last_str) != 0); /* We've dealt with the literal now. It can't be leaked on further errors. */ From e9f4d8db5f30a9603fa0c005382bff78aa1ce232 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 16 Jun 2017 07:18:58 -0700 Subject: [PATCH 0451/1345] [email] bpo-29478: Fix passing max_line_length=None from Compat32 policy (GH-595) (GH-2233) If max_line_length=None is specified while using the Compat32 policy, it is no longer ignored.. (cherry picked from commit b459f7482612d340b88b62edc024628595ec6337) --- Lib/email/_policybase.py | 8 ++++++-- Lib/test/test_email/test_generator.py | 7 +++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index df4649676aed72..c9cbadd2a80c48 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -361,8 +361,12 @@ def _fold(self, name, value, sanitize): # Assume it is a Header-like object. h = value if h is not None: - parts.append(h.encode(linesep=self.linesep, - maxlinelen=self.max_line_length)) + # The Header class interprets a value of None for maxlinelen as the + # default value of 78, as recommended by RFC 2822. + maxlinelen = 0 + if self.max_line_length is not None: + maxlinelen = self.max_line_length + parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen)) parts.append(self.linesep) return ''.join(parts) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 7c8877fdcb090e..c4f182903afefe 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -162,6 +162,13 @@ def test_set_mangle_from_via_policy(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_compat32_max_line_length_does_not_fold_when_none(self): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=policy.compat32.clone(max_line_length=None)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Misc/ACKS b/Misc/ACKS index f1686b3118ff93..64484f3f8f26aa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -309,6 +309,7 @@ Garrett Cooper Greg Copeland Ian Cordasco Aldo Cortesi +Mircea Cosbuc David Costanzo Scott Cotton Greg Couch diff --git a/Misc/NEWS b/Misc/NEWS index fd7ebf92dc5627..2cc71bded5f044 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. +- bpo-29478: If max_line_length=None is specified while using the Compat32 policy, + it is no longer ignored. Patch by Mircea Cosbuc. + Library ------- From 304a96e5868c1b38a87692668f297aaf041e6e67 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 16 Jun 2017 07:19:56 -0700 Subject: [PATCH 0452/1345] bpo-30176: Add missing curses cell attributes constants (GH-1302). (GH-2241) (cherry picked from commit 116dd5eba60a940b35db6aaf4e8c998ac30ad440) --- Doc/library/curses.rst | 54 +++++++++++++++++++++++++++++++++++------- Misc/NEWS | 2 ++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 3442e4e75a3e45..668d2826b0a1b3 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1271,27 +1271,63 @@ The :mod:`curses` module defines the following data members: A string representing the current version of the module. Also available as :const:`__version__`. -Several constants are available to specify character cell attributes: +Some constants are available to specify character cell attributes. +The exact constants available are system dependent. +------------------+-------------------------------+ | Attribute | Meaning | +==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode. | +| ``A_ALTCHARSET`` | Alternate character set mode | +------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode. | +| ``A_BLINK`` | Blink mode | +------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode. | +| ``A_BOLD`` | Bold mode | +------------------+-------------------------------+ -| ``A_DIM`` | Dim mode. | +| ``A_DIM`` | Dim mode | +------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute. | +| ``A_INVIS`` | Invisible or blank mode | ++------------------+-------------------------------+ +| ``A_NORMAL`` | Normal attribute | ++------------------+-------------------------------+ +| ``A_PROTECT`` | Protected mode | +------------------+-------------------------------+ | ``A_REVERSE`` | Reverse background and | -| | foreground colors. | +| | foreground colors | ++------------------+-------------------------------+ +| ``A_STANDOUT`` | Standout mode | ++------------------+-------------------------------+ +| ``A_UNDERLINE`` | Underline mode | ++------------------+-------------------------------+ +| ``A_HORIZONTAL`` | Horizontal highlight | ++------------------+-------------------------------+ +| ``A_LEFT`` | Left highlight | ++------------------+-------------------------------+ +| ``A_LOW`` | Low highlight | ++------------------+-------------------------------+ +| ``A_RIGHT`` | Right highlight | ++------------------+-------------------------------+ +| ``A_TOP`` | Top highlight | ++------------------+-------------------------------+ +| ``A_VERTICAL`` | Vertical highlight | ++------------------+-------------------------------+ +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | ++------------------+-------------------------------+ + +Several constants are available to extract corresponding attributes returned +by some methods. + ++------------------+-------------------------------+ +| Bit-mask | Meaning | ++==================+===============================+ +| ``A_ATTRIBUTES`` | Bit-mask to extract | +| | attributes | +------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode. | +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | +------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode. | +| ``A_COLOR`` | Bit-mask to extract | +| | color-pair field information | +------------------+-------------------------------+ Keys are referred to by integer constants with names starting with ``KEY_``. diff --git a/Misc/NEWS b/Misc/NEWS index 2cc71bded5f044..0106fa70865889 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -286,6 +286,8 @@ Build Documentation ------------- +- bpo-30176: Add missing attribute related constants in curses documentation. + - Issue #30052: the link targets for :func:`bytes` and :func:`bytearray` are now their respective type definitions, rather than the corresponding builtin function entries. Use :ref:`bytes ` From b7c59f0d05bde0d4dd2d7bffc1b8af72c25d3f71 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 16 Jun 2017 22:30:50 -0400 Subject: [PATCH 0453/1345] bpo-23894: add Misc/NEWS entry. (#2255) --- Misc/NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 0106fa70865889..7b5da07512a399 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -57,6 +57,8 @@ Core and Builtins Library ------- +- bpo-23894: lib2to3 now recognizes ``rb'...'`` and ``f'...'`` strings. + - bpo-23890: unittest.TestCase.assertRaises() now manually breaks a reference cycle to not keep objects alive longer than expected. From 0a794a3256b24ccf57b18ec9964f2367ac1f3d30 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 16 Jun 2017 23:01:09 -0400 Subject: [PATCH 0454/1345] bpo-30038: add Misc/NEWS entry. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 7b5da07512a399..c483e5f0f13487 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -57,6 +57,9 @@ Core and Builtins Library ------- +- bpo-30038: Fix race condition between signal delivery and wakeup file + descriptor. Patch by Nathaniel Smith. + - bpo-23894: lib2to3 now recognizes ``rb'...'`` and ``f'...'`` strings. - bpo-23890: unittest.TestCase.assertRaises() now manually breaks a reference From 4da52247d6ae53b8384f9753430e8cd2672ff97d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 16 Jun 2017 18:47:48 -0400 Subject: [PATCH 0455/1345] Add option to pass vcs info into Mac installer build --- Mac/BuildScript/build-installer.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 7b4376f42b88f3..e17299affba11c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1142,8 +1142,25 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make") - runCommand("make") + # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS + # and, if defined, append its value to the make command. This allows + # us to pass in version control tags, like GITTAG, to a build from a + # tarball rather than from a vcs checkout, thus eliminating the need + # to have a working copy of the vcs program on the build machine. + # + # A typical use might be: + # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ + # GITVERSION='echo 123456789a' \ + # GITTAG='echo v3.6.0' \ + # GITBRANCH='echo 3.6'" + + make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") + if make_extras: + make_cmd = "make " + make_extras + else: + make_cmd = "make" + print("Running " + make_cmd) + runCommand(make_cmd) print("Running make install") runCommand("make install DESTDIR=%s"%( From 4f05ef4e483b40a4bf6b06431197fac78cd26ffe Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 16 Jun 2017 21:00:49 -0400 Subject: [PATCH 0456/1345] Remove check for git binary in Mac installer build --- Mac/BuildScript/build-installer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index e17299affba11c..1fff052e2489b2 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of git and a third-party version of +requires an installed third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), git, and for the documentation +(2.5 is needed for Python parser updates) and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -635,9 +635,8 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to git and to sphinx-build. - # You may have to create links in /usr/bin for them. - runCommand('git --version') + # Ensure we have access to sphinx-build. + # You may have to create a link in /usr/bin for it. runCommand('sphinx-build --version') def parseOptions(args=None): From 8c5483eb2292f6ca01ebbf965fd0e5f6b2c69dc6 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 17 Jun 2017 04:37:19 -0400 Subject: [PATCH 0457/1345] Update pydoc topics for v3.6.2rc1 --- Lib/pydoc_data/topics.py | 232 ++++++++++++++++++++------------------- 1 file changed, 122 insertions(+), 110 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 2caab630978fa0..015afb30b45e55 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +# Autogenerated by Sphinx on Sat Jun 17 04:32:54 2017 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1300,7 +1300,7 @@ ' 2 1\n' ' >>> f(a=1, *(2,))\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' " TypeError: f() got multiple values for keyword argument 'a'\n" ' >>> f(1, *(2,))\n' ' 1 2\n' @@ -1669,7 +1669,7 @@ 'they\n' ' have equal *(key, value)* pairs. Equality comparison of the ' 'keys and\n' - ' elements enforces reflexivity.\n' + ' values enforces reflexivity.\n' '\n' ' Order comparisons ("<", ">", "<=", and ">=") raise ' '"TypeError".\n' @@ -1761,9 +1761,9 @@ '\n' 'The operators "in" and "not in" test for membership. "x in ' 's"\n' - 'evaluates to true if *x* is a member of *s*, and false ' - 'otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in ' + 'evaluates to "True" if *x* is a member of *s*, and "False" ' + 'otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' 'sequences\n' 'and set types support this as well as dictionary, for which ' '"in" tests\n' @@ -1774,30 +1774,32 @@ 'for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only ' - 'if *x* is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != ' - '-1". Empty\n' - 'strings are always considered to be a substring of any other ' - 'string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and ' + 'only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != ' + '-1".\n' + 'Empty strings are always considered to be a substring of any ' + 'other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" ' 'method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true ' + 'value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" ' 'but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with ' - '"x == z"\n' - 'is produced while iterating over "y". If an exception is ' + 'define "__iter__()", "x in y" is "True" if some value "z" ' + 'with "x ==\n' + 'z" is produced while iterating over "y". If an exception is ' 'raised\n' 'during the iteration, it is as if "in" raised that ' 'exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class ' 'defines\n' - '"__getitem__()", "x in y" is true if and only if there is a ' + '"__getitem__()", "x in y" is "True" if and only if there is a ' 'non-\n' 'negative integer index *i* such that "x == y[i]", and all ' 'lower\n' @@ -2839,11 +2841,11 @@ ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -2878,7 +2880,7 @@ ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -3027,8 +3029,8 @@ '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -3346,7 +3348,7 @@ ' >>> import mymodule\n' ' >>> mymodule.test()\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' ' File "./mymodule.py", line 4, in test\n' ' test2()\n' ' File "./mymodule.py", line 3, in test2\n' @@ -4010,16 +4012,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are ' - 'illegal when used\n' - 'in conjunction with nested scopes that contain free ' - 'variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile ' - 'time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access ' 'to the full\n' 'environment for resolving names. Names may be resolved ' @@ -4323,6 +4315,13 @@ 'Builtins and restricted execution\n' '---------------------------------\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. ' + 'Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -4335,16 +4334,7 @@ 'in any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -4360,14 +4350,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal ' - 'when used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -5521,8 +5503,8 @@ 'change\n' 'the meaning of the program.\n' '\n' - '**Programmer\'s note:** the "global" is a directive to the ' - 'parser. It\n' + '**Programmer\'s note:** "global" is a directive to the parser. ' + 'It\n' 'applies only to code parsed at the same time as the "global"\n' 'statement. In particular, a "global" statement contained in a ' 'string\n' @@ -6031,8 +6013,9 @@ '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to true if *x* is a member of *s*, and false otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in sequences\n' + 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' + 'sequences\n' 'and set types support this as well as dictionary, for which "in" ' 'tests\n' 'whether the dictionary has a given key. For container types such as\n' @@ -6040,22 +6023,22 @@ 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only if *x* ' - 'is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != -1". Empty\n' - 'strings are always considered to be a substring of any other string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' + 'Empty strings are always considered to be a substring of any other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with "x == z"\n' - 'is produced while iterating over "y". If an exception is raised\n' + 'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n' + 'z" is produced while iterating over "y". If an exception is raised\n' 'during the iteration, it is as if "in" raised that exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is true if and only if there is a non-\n' + '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' 'negative integer index *i* such that "x == y[i]", and all lower\n' 'integer indices do not raise "IndexError" exception. (If any other\n' 'exception is raised, it is as if "in" raised that exception).\n' @@ -6305,6 +6288,12 @@ 'Builtins and restricted execution\n' '=================================\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -6317,15 +6306,7 @@ 'any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -6341,14 +6322,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal when ' - 'used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is illegal ' - 'to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -6980,7 +6953,7 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' The above exception was the direct cause of the following ' 'exception:\n' @@ -7002,7 +6975,7 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' During handling of the above exception, another exception ' 'occurred:\n' @@ -7011,10 +6984,30 @@ ' File "", line 4, in \n' ' RuntimeError: Something bad happened\n' '\n' + 'Exception chaining can be explicitly suppressed by specifying ' + '"None"\n' + 'in the "from" clause:\n' + '\n' + ' >>> try:\n' + ' ... print(1 / 0)\n' + ' ... except:\n' + ' ... raise RuntimeError("Something bad happened") from None\n' + ' ...\n' + ' Traceback (most recent call last):\n' + ' File "", line 4, in \n' + ' RuntimeError: Something bad happened\n' + '\n' 'Additional information on exceptions can be found in section\n' 'Exceptions, and information about handling exceptions is in ' 'section\n' - 'The try statement.\n', + 'The try statement.\n' + '\n' + 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' + 'from Y".\n' + '\n' + 'New in version 3.3: The "__suppress_context__" attribute to ' + 'suppress\n' + 'automatic display of the exception context.\n', 'return': 'The "return" statement\n' '**********************\n' '\n' @@ -7122,6 +7115,16 @@ ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the ' + 'length is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may ' + 'raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -7497,11 +7500,11 @@ ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -7536,7 +7539,7 @@ ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -7683,8 +7686,8 @@ '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -8740,6 +8743,15 @@ ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the length ' + 'is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -9564,13 +9576,13 @@ 'str.join(iterable)\n' '\n' ' Return a string which is the concatenation of the ' - 'strings in the\n' - ' *iterable* *iterable*. A "TypeError" will be raised if ' - 'there are\n' - ' any non-string values in *iterable*, including "bytes" ' - 'objects.\n' - ' The separator between elements is the string providing ' - 'this method.\n' + 'strings in\n' + ' *iterable*. A "TypeError" will be raised if there are ' + 'any non-\n' + ' string values in *iterable*, including "bytes" ' + 'objects. The\n' + ' separator between elements is the string providing this ' + 'method.\n' '\n' 'str.ljust(width[, fillchar])\n' '\n' @@ -10695,11 +10707,11 @@ '8-bit\n' ' bytes, represented by integers in the range 0 <= x < 256.\n' ' Bytes literals (like "b\'abc\'") and the built-in ' - 'function\n' - ' "bytes()" can be used to construct bytes objects. Also,\n' - ' bytes objects can be decoded to strings via the ' - '"decode()"\n' - ' method.\n' + '"bytes()"\n' + ' constructor can be used to create bytes objects. Also, ' + 'bytes\n' + ' objects can be decoded to strings via the "decode()" ' + 'method.\n' '\n' ' Mutable sequences\n' ' Mutable sequences can be changed after they are created. ' @@ -10725,7 +10737,7 @@ ' the built-in "bytearray()" constructor. Aside from being\n' ' mutable (and hence unhashable), byte arrays otherwise ' 'provide\n' - ' the same interface and functionality as immutable bytes\n' + ' the same interface and functionality as immutable "bytes"\n' ' objects.\n' '\n' ' The extension module "array" provides an additional example ' @@ -11998,7 +12010,7 @@ 'comparison operations. The "+" (concatenation) and "*" ' '(repetition)\n' 'operations have the same priority as the corresponding numeric\n' - 'operations.\n' + 'operations. [3]\n' '\n' '+----------------------------+----------------------------------+------------+\n' '| Operation | Result ' From 268e1fbf215fb17e1ebed793190457dab6d182cf Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 17 Jun 2017 04:48:35 -0400 Subject: [PATCH 0458/1345] Version bump to 3.6.2rc1 --- Include/patchlevel.h | 8 ++++---- Misc/NEWS | 2 +- README.rst | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index a658e9f5840e1c..9c22385f3b36c4 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 1 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL -#define PY_RELEASE_SERIAL 0 +#define PY_MICRO_VERSION 2 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.1+" +#define PY_VERSION "3.6.2rc1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index c483e5f0f13487..bccdf94e24983b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -5,7 +5,7 @@ Python News What's New in Python 3.6.2 release candidate 1? =============================================== -*Release date: XXXX-XX-XX* +*Release date: 2017-06-17* Core and Builtins ----------------- diff --git a/README.rst b/README.rst index b0ce13599ad653..71eb257b529d4e 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.6.1+ -============================= +This is Python version 3.6.2 release candidate 1 +================================================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI From ccc06d32708157b6712808309faf52ceaf7c2870 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 17 Jun 2017 21:58:49 -0400 Subject: [PATCH 0459/1345] Bump version for post 3.6.2rc1 --- Include/patchlevel.h | 2 +- Misc/NEWS | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9c22385f3b36c4..b4996d55ec841f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.2rc1" +#define PY_VERSION "3.6.2rc1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index bccdf94e24983b..868bfc4fec0ef5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,7 +1,19 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ +What's New in Python 3.6.3 release candidate 1? +=============================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 3.6.2 release candidate 1? =============================================== From 51f40a81a4acbc85cb1034750fb16cb1854b2315 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 19 Jun 2017 04:22:54 -0700 Subject: [PATCH 0460/1345] [3.6] bpo-29887: Test normalization now fails if download fails (GH-905) (#2271) * test_normalization fails if download fails bpo-29887. The test is still skipped if "-u urlfetch" option is not passed to regrtest (python3 -m test -u urlfetch test_normalization). * Fix ResourceWarning in test_normalization bpo-29887: Fix ResourceWarning in test_normalization if tests are interrupted by CTRL+c. (cherry picked from commit 722a3af092b94983aa26f5e591fb1b45e2c2a0ff) --- Lib/test/test_normalization.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 5b590e1af34495..c6f8c93119f436 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -36,15 +36,20 @@ def unistr(data): class NormalizationTest(unittest.TestCase): def test_main(self): - part = None - part1_data = {} # Hit the exception early try: testdata = open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version) except (OSError, HTTPException): - self.skipTest("Could not retrieve " + TESTDATAURL) - self.addCleanup(testdata.close) + self.fail(f"Could not retrieve {TESTDATAURL}") + + with testdata: + self.run_normalization_tests(testdata) + + def run_normalization_tests(self, testdata): + part = None + part1_data = {} + for line in testdata: if '#' in line: line = line.split('#')[0] From acdf159b245aaac28f54dd10606911386ab39058 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 19 Jun 2017 15:42:43 +0200 Subject: [PATCH 0461/1345] bpo-29887: test_normalization handles PermissionError (#1196) (#2275) Skip test_normalization.test_main() if download raises a permission error. (cherry picked from commit d13d54748d3a7db023d9db37223ea7d40bb8f8e3) --- Lib/test/test_normalization.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index c6f8c93119f436..30424564450227 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -40,6 +40,9 @@ def test_main(self): try: testdata = open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version) + except PermissionError: + self.skipTest(f"Permission error when downloading {TESTDATAURL} " + f"into the test data directory") except (OSError, HTTPException): self.fail(f"Could not retrieve {TESTDATAURL}") From 06d6e3d0bb5b8a3d3105289034953a8014356a0b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 19 Jun 2017 11:12:56 -0700 Subject: [PATCH 0462/1345] bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat (#2252) (#2280) * Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat Also fixes bdist_wininst.vcxproj to use correct version in generated name. --- Misc/NEWS | 3 ++ PC/bdist_wininst/bdist_wininst.vcxproj | 4 ++ PC/bdist_wininst/build.bat | 11 ++---- PCbuild/build.bat | 32 ++++++---------- PCbuild/find_msbuild.bat | 51 ++++++++++++++++++++++++++ PCbuild/python.vcxproj | 5 ++- Tools/msi/build.bat | 10 ++--- Tools/msi/buildrelease.bat | 15 ++++---- Tools/msi/uploadrelease.bat | 15 ++++---- Tools/nuget/build.bat | 7 ++-- 10 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 PCbuild/find_msbuild.bat diff --git a/Misc/NEWS b/Misc/NEWS index 868bfc4fec0ef5..e24fd296de8a25 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -339,6 +339,9 @@ Tests Windows ------- +- bpo-30687: Locate msbuild.exe on Windows when building rather than + vcvarsall.bat + - bpo-30450: The build process on Windows no longer depends on Subversion, instead pulling external code from GitHub via a Python script. If Python 3.6 is not found on the system (via ``py -3.6``), NuGet is used to download a diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj b/PC/bdist_wininst/bdist_wininst.vcxproj index 4dc5009b58e6ac..9bbfda28ccbe2b 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj +++ b/PC/bdist_wininst/bdist_wininst.vcxproj @@ -58,6 +58,10 @@ $(PySourcePath)lib\distutils\command\ false wininst-$(VisualStudioVersion) + wininst-14.0 + wininst-12.0 + wininst-11.0 + wininst-10.0 $(TargetName)-amd64 .exe diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index ee6856754faa8c..d5cf75206dd810 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -7,16 +7,13 @@ set PCBUILD=%~dp0..\..\PCBuild\ echo Building Lib\distutils\command\wininst-xx.0.exe -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 if errorlevel 1 goto :eof echo Building Lib\distutils\command\wininst-xx.0-amd64.exe -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8e1c168a8591f8..b63843fa95d971 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -48,7 +48,6 @@ exit /b 127 :Run setlocal set platf=Win32 -set vs_platf=x86 set conf=Release set target=Build set dir=%~dp0 @@ -57,10 +56,6 @@ set verbose=/nologo /v:m set kill= set do_pgo= set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) :CheckOpts if "%~1"=="-h" goto Usage @@ -90,18 +85,12 @@ if "%IncludeTkinter%"=="" set IncludeTkinter=true if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 ) ) @@ -110,7 +99,8 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul +call "%dir%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if "%kill%"=="true" call :Kill @@ -129,7 +119,7 @@ if "%do_pgo%"=="true" ( goto Build :Kill echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ +%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:KillPython=true @@ -141,7 +131,7 @@ rem Call on MSBuild to do the work, echo the command. rem Passing %1-9 is not the preferred option, but argument parsing in rem batch is, shall we say, "lackluster" echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ +%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ @@ -153,4 +143,4 @@ goto :eof :Version rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat new file mode 100644 index 00000000000000..1877906e00a55d --- /dev/null +++ b/PCbuild/find_msbuild.bat @@ -0,0 +1,51 @@ +@rem +@rem Searches for MSBuild.exe. This is the only tool we need to initiate +@rem a build, so we no longer search for the full VC toolset. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found + +:begin_search +@set MSBUILD= + +@rem If msbuild.exe is on the PATH, assume that the user wants that one. +@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" +@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found + +@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( + @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found + +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + + +@exit /b 1 + +:found +@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%) +@set _Py_MSBuild_Source= diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 807213f12a76f4..2786ac2ebf7b6b 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -83,7 +83,7 @@ - + ucrtbase ucrtbased @@ -94,6 +94,8 @@ set PYTHONPATH=$(PySourcePath)Lib + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64 <_Content>@rem This script invokes the most recently built Python with all arguments @rem passed through to the interpreter. This file is generated by the @rem build process and any changes *will* be thrown away by the next @@ -103,6 +105,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) +$(_PGOPath) @"$(OutDir)python$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index 59e0261c6d5236..f720ea9a86a8cb 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,8 +22,8 @@ if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%get_externals.bat" - -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined BUILDX86 ( call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% @@ -44,7 +44,7 @@ if defined BUILDDOC ( ) rem Build the launcher MSI separately -msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 +%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 set BUILD_CMD="%D%bundle\snapshot.wixproj" if defined BUILDTEST ( @@ -58,11 +58,11 @@ if defined REBUILD ( ) if defined BUILDX86 ( - msbuild %BUILD_CMD% + %MSBUILD% %BUILD_CMD% if errorlevel 1 goto :eof ) if defined BUILDX64 ( - msbuild /p:Platform=x64 %BUILD_CMD% + %MSBUILD% /p:Platform=x64 %BUILD_CMD% if errorlevel 1 goto :eof ) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index ed930c653c11ea..f60fe6af393165 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -69,6 +69,8 @@ if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) :builddoc if "%SKIPBUILD%" EQU "1" goto skipdoc @@ -165,28 +167,27 @@ if not "%SKIPBUILD%" EQU "1" ( @echo off ) -call "%PCBUILD%env.bat" if "%OUTDIR_PLAT%" EQU "win32" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true +%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true if errorlevel 1 exit /B -msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false +%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false if errorlevel 1 exit /B if defined BUILDZIP ( - msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) if defined BUILDNUGET ( - msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 0873d55589f0db..610b092b863374 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -49,16 +49,17 @@ if defined NOGPG ( echo Found gpg2.exe at %GPG% ) -call "%PCBUILD%env.bat" > nul 2> nul +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% if not defined NOTEST ( - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 ) -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd exit /B 0 diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 5068c17244456e..54498ca28a2471 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -21,7 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%..\msi\get_externals.bat" -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" @@ -30,7 +31,7 @@ if defined BUILDX86 ( ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) @@ -39,7 +40,7 @@ if defined BUILDX64 ( ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) From 0a4fe1d8578fa59004518f8deef137282be4d71a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 19 Jun 2017 20:28:16 -0700 Subject: [PATCH 0463/1345] [3.6] bpo-30420: List cwd parameter in subprocess convenience APIs (GH-1685) (GH-2253) Partially clarify the subprocess convenience API documentation by explicitly listing the `cwd` parameter in their abbreviated signatures. While this has been merged as an improvement, it doesn't fully resolve the issue, as the `cwd` should also be covered in the "Frequently Used Arguments" section, and the fact these APIs pass unlisted keyword arguments down to the lower level APIs is currently still unclear. (cherry picked from commit 368cf1d20630498ca7939069a05d744fabb570aa) --- Doc/library/subprocess.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 548e4a68cbf1d1..663e232851b067 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -38,7 +38,7 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - shell=False, timeout=None, check=False, \ + shell=False, cwd=None, timeout=None, check=False, \ encoding=None, errors=None) Run the command described by *args*. Wait for command to complete, then @@ -856,7 +856,7 @@ Prior to Python 3.5, these three functions comprised the high level API to subprocess. You can now use :func:`run` in many cases, but lots of existing code calls these functions. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) Run the command described by *args*. Wait for command to complete, then return the :attr:`~Popen.returncode` attribute. @@ -882,7 +882,7 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) Run command with arguments. Wait for command to complete. If the return code was zero then return, otherwise raise :exc:`CalledProcessError`. The @@ -912,7 +912,7 @@ calls these functions. .. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ - encoding=None, errors=None, \ + cwd=None, encoding=None, errors=None, \ universal_newlines=False, timeout=None) Run command with arguments and return its output. From 536c1f1246f4faa302f9f5613fc3444e7ae09b4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 20 Jun 2017 15:37:24 +0200 Subject: [PATCH 0464/1345] bpo-30500: urllib: Simplify splithost by calling into urlparse. (#1849) (#2289) The current regex based splitting produces a wrong result. For example:: http://abc#@def Web browsers parse that URL as ``http://abc/#@def``, that is, the host is ``abc``, the path is ``/``, and the fragment is ``#@def``. (cherry picked from commit 90e01e50ef8a9e6c91f30d965563c378a4ad26de) --- Lib/test/test_urlparse.py | 51 ++++++++++++++++++++++++++++++--------- Lib/urllib/parse.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 5 ++++ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 3c89ed928ff51f..e5f6130e4a075e 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -755,28 +755,35 @@ def test_default_scheme(self): def test_parse_fragments(self): # Exercise the allow_fragments parameter of urlparse() and urlsplit() tests = ( - ("http:#frag", "path"), - ("//example.net#frag", "path"), - ("index.html#frag", "path"), - (";a=b#frag", "params"), - ("?a=b#frag", "query"), - ("#frag", "path"), + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), ) - for url, attr in tests: + for url, attr, expected_frag in tests: for func in (urllib.parse.urlparse, urllib.parse.urlsplit): if attr == "params" and func is urllib.parse.urlsplit: attr = "path" with self.subTest(url=url, function=func): result = func(url, allow_fragments=False) self.assertEqual(result.fragment, "") - self.assertTrue(getattr(result, attr).endswith("#frag")) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) self.assertEqual(func(url, "", False).fragment, "") result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, "frag") - self.assertFalse(getattr(result, attr).endswith("frag")) - self.assertEqual(func(url, "", True).fragment, "frag") - self.assertEqual(func(url).fragment, "frag") + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -983,6 +990,26 @@ def test_splithost(self): self.assertEqual(splithost('/foo/bar/baz.html'), (None, '/foo/bar/baz.html')) + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + def test_splituser(self): splituser = urllib.parse.splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 888247b8e90f56..3cab2d13d5fb56 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -944,7 +944,7 @@ def splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: diff --git a/Misc/ACKS b/Misc/ACKS index 64484f3f8f26aa..03f9ea6cc97ae3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1081,6 +1081,7 @@ Max Neunhöffer Anthon van der Neut George Neville-Neil Hieu Nguyen +Nam Nguyen Johannes Nicolai Samuel Nicolary Jonathan Niehof diff --git a/Misc/NEWS b/Misc/NEWS index e24fd296de8a25..beb63bd7399fb3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,11 @@ Core and Builtins Library ------- +- [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse + fragments. For example, ``splithost('http://127.0.0.1#@evil.com/')`` now + correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` + as the host in an authentification (``login@host``). + What's New in Python 3.6.2 release candidate 1? =============================================== From 82acabd3c52508d9e3f83a41fe7c684619cbbe7b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 20 Jun 2017 16:09:06 +0200 Subject: [PATCH 0465/1345] bpo-30500: Fix the NEWS entry (#2296) splithost() expects an URL starting with "//" not with "http://". --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index beb63bd7399fb3..51edb1debb4a42 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -14,7 +14,7 @@ Library ------- - [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse - fragments. For example, ``splithost('http://127.0.0.1#@evil.com/')`` now + fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` as the host in an authentification (``login@host``). From a1115e1a0454f0548f96cace6ee97b286dfa1c0d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 20 Jun 2017 18:06:35 +0300 Subject: [PATCH 0466/1345] [3.6] bpo-29755: Fixed the lgettext() family of functions in the gettext module. (GH-2266) (#2297) They now always return bytes. Updated the gettext documentation.. (cherry picked from commit 26cb4657bcc9a7adffa95798ececb588dddfeadb) --- Doc/library/gettext.rst | 161 +++++++++++++++++++-------------------- Lib/gettext.py | 40 +++++----- Lib/test/test_gettext.py | 132 +++++++++++++++++++++++++++++--- Misc/NEWS | 3 + 4 files changed, 229 insertions(+), 107 deletions(-) diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 3a87bf591776c4..053d9d3840cfc0 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -48,9 +48,10 @@ class-based API instead. .. function:: bind_textdomain_codeset(domain, codeset=None) - Bind the *domain* to *codeset*, changing the encoding of strings returned by the - :func:`gettext` family of functions. If *codeset* is omitted, then the current - binding is returned. + Bind the *domain* to *codeset*, changing the encoding of byte strings + returned by the :func:`lgettext`, :func:`ldgettext`, :func:`lngettext` + and :func:`ldngettext` functions. + If *codeset* is omitted, then the current binding is returned. .. function:: textdomain(domain=None) @@ -67,28 +68,14 @@ class-based API instead. :func:`_` in the local namespace (see examples below). -.. function:: lgettext(message) - - Equivalent to :func:`gettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. - - .. function:: dgettext(domain, message) - Like :func:`gettext`, but look the message up in the specified *domain*. - - -.. function:: ldgettext(domain, message) - - Equivalent to :func:`dgettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. + Like :func:`.gettext`, but look the message up in the specified *domain*. .. function:: ngettext(singular, plural, n) - Like :func:`gettext`, but consider plural forms. If a translation is found, + Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some languages have more than two plural forms). If no translation is found, return *singular* if *n* is 1; return *plural* otherwise. @@ -101,24 +88,33 @@ class-based API instead. formulas for a variety of languages. -.. function:: lngettext(singular, plural, n) - - Equivalent to :func:`ngettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. - - .. function:: dngettext(domain, singular, plural, n) Like :func:`ngettext`, but look the message up in the specified *domain*. +.. function:: lgettext(message) +.. function:: ldgettext(domain, message) +.. function:: lngettext(singular, plural, n) .. function:: ldngettext(domain, singular, plural, n) - Equivalent to :func:`dngettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with + Equivalent to the corresponding functions without the ``l`` prefix + (:func:`.gettext`, :func:`dgettext`, :func:`ngettext` and :func:`dngettext`), + but the translation is returned as a byte string encoded in the preferred + system encoding if no other encoding was explicitly set with :func:`bind_textdomain_codeset`. + .. warning:: + + These functions should be avoided in Python 3, because they return + encoded bytes. It's much better to use alternatives which return + Unicode strings instead, since most Python applications will want to + manipulate human readable text as strings instead of bytes. Further, + it's possible that you may get unexpected Unicode-related exceptions + if there are encoding problems with the translated strings. It is + possible that the ``l*()`` functions will be deprecated in future Python + versions due to their inherent problems and limitations. + Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but this was deemed not useful and so it is currently unimplemented. @@ -179,8 +175,9 @@ class can also install themselves in the built-in namespace as the function names are cached. The actual class instantiated is either *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the :meth:`lgettext` and - :meth:`lngettext` methods. + the charset used to encode translated strings in the + :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` + methods. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -250,26 +247,29 @@ are the methods of :class:`NullTranslations`: .. method:: gettext(message) - If a fallback has been set, forward :meth:`gettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. - - - .. method:: lgettext(message) - - If a fallback has been set, forward :meth:`lgettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + If a fallback has been set, forward :meth:`.gettext` to the fallback. + Otherwise, return *message*. Overridden in derived classes. .. method:: ngettext(singular, plural, n) If a fallback has been set, forward :meth:`ngettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + Otherwise, return *singular* if *n* is 1; return *plural* otherwise. + Overridden in derived classes. + .. method:: lgettext(message) .. method:: lngettext(singular, plural, n) - If a fallback has been set, forward :meth:`lngettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + Equivalent to :meth:`.gettext` and :meth:`ngettext`, but the translation + is returned as a byte string encoded in the preferred system encoding + if no encoding was explicitly set with :meth:`set_output_charset`. + Overridden in derived classes. + + .. warning:: + + These methods should be avoided in Python 3. See the warning for the + :func:`lgettext` function. .. method:: info() @@ -279,32 +279,28 @@ are the methods of :class:`NullTranslations`: .. method:: charset() - Return the "protected" :attr:`_charset` variable, which is the encoding of - the message catalog file. + Return the encoding of the message catalog file. .. method:: output_charset() - Return the "protected" :attr:`_output_charset` variable, which defines the - encoding used to return translated messages in :meth:`lgettext` and - :meth:`lngettext`. + Return the encoding used to return translated messages in :meth:`.lgettext` + and :meth:`.lngettext`. .. method:: set_output_charset(charset) - Change the "protected" :attr:`_output_charset` variable, which defines the - encoding used to return translated messages. + Change the encoding used to return translated messages. .. method:: install(names=None) - This method installs :meth:`self.gettext` into the built-in namespace, + This method installs :meth:`.gettext` into the built-in namespace, binding it to ``_``. If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'`` (bound to - :meth:`self.gettext`), ``'ngettext'`` (bound to :meth:`self.ngettext`), + addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, ``'lgettext'`` and ``'lngettext'``. Note that this is only one way, albeit the most convenient way, to make @@ -349,49 +345,52 @@ If the :file:`.mo` file's magic number is invalid, the major version number is unexpected, or if other problems occur while reading the file, instantiating a :class:`GNUTranslations` class can raise :exc:`OSError`. -The following methods are overridden from the base class implementation: - +.. class:: GNUTranslations -.. method:: GNUTranslations.gettext(message) + The following methods are overridden from the base class implementation: - Look up the *message* id in the catalog and return the corresponding message - string, as a Unicode string. If there is no entry in the catalog for the - *message* id, and a fallback has been set, the look up is forwarded to the - fallback's :meth:`gettext` method. Otherwise, the *message* id is returned. + .. method:: gettext(message) + Look up the *message* id in the catalog and return the corresponding message + string, as a Unicode string. If there is no entry in the catalog for the + *message* id, and a fallback has been set, the look up is forwarded to the + fallback's :meth:`~NullTranslations.gettext` method. Otherwise, the + *message* id is returned. -.. method:: GNUTranslations.lgettext(message) - Equivalent to :meth:`gettext`, but the translation is returned as a - bytestring encoded in the selected output charset, or in the preferred system - encoding if no encoding was explicitly set with :meth:`set_output_charset`. + .. method:: ngettext(singular, plural, n) + Do a plural-forms lookup of a message id. *singular* is used as the message id + for purposes of lookup in the catalog, while *n* is used to determine which + plural form to use. The returned message string is a Unicode string. -.. method:: GNUTranslations.ngettext(singular, plural, n) + If the message id is not found in the catalog, and a fallback is specified, + the request is forwarded to the fallback's :meth:`~NullTranslations.ngettext` + method. Otherwise, when *n* is 1 *singular* is returned, and *plural* is + returned in all other cases. - Do a plural-forms lookup of a message id. *singular* is used as the message id - for purposes of lookup in the catalog, while *n* is used to determine which - plural form to use. The returned message string is a Unicode string. + Here is an example:: - If the message id is not found in the catalog, and a fallback is specified, the - request is forwarded to the fallback's :meth:`ngettext` method. Otherwise, when - *n* is 1 *singular* is returned, and *plural* is returned in all other cases. + n = len(os.listdir('.')) + cat = GNUTranslations(somefile) + message = cat.ngettext( + 'There is %(num)d file in this directory', + 'There are %(num)d files in this directory', + n) % {'num': n} - Here is an example:: - n = len(os.listdir('.')) - cat = GNUTranslations(somefile) - message = cat.ngettext( - 'There is %(num)d file in this directory', - 'There are %(num)d files in this directory', - n) % {'num': n} + .. method:: lgettext(message) + .. method:: lngettext(singular, plural, n) + Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation + is returned as a byte string encoded in the preferred system encoding + if no encoding was explicitly set with + :meth:`~NullTranslations.set_output_charset`. -.. method:: GNUTranslations.lngettext(singular, plural, n) + .. warning:: - Equivalent to :meth:`gettext`, but the translation is returned as a - bytestring encoded in the selected output charset, or in the preferred system - encoding if no encoding was explicitly set with :meth:`set_output_charset`. + These methods should be avoided in Python 3. See the warning for the + :func:`lgettext` function. Solaris message catalog support @@ -509,7 +508,7 @@ module:: import gettext t = gettext.translation('spam', '/usr/share/locale') - _ = t.lgettext + _ = t.gettext Localizing your application diff --git a/Lib/gettext.py b/Lib/gettext.py index 57d2c74982e1d4..6b215af45ef317 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -270,7 +270,9 @@ def gettext(self, message): def lgettext(self, message): if self._fallback: return self._fallback.lgettext(message) - return message + if self._output_charset: + return message.encode(self._output_charset) + return message.encode(locale.getpreferredencoding()) def ngettext(self, msgid1, msgid2, n): if self._fallback: @@ -284,9 +286,12 @@ def lngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.lngettext(msgid1, msgid2, n) if n == 1: - return msgid1 + tmsg = msgid1 else: - return msgid2 + tmsg = msgid2 + if self._output_charset: + return tmsg.encode(self._output_charset) + return tmsg.encode(locale.getpreferredencoding()) def info(self): return self._info @@ -368,7 +373,7 @@ def _parse(self, fp): if mlen == 0: # Catalog description lastk = None - for b_item in tmsg.split('\n'.encode("ascii")): + for b_item in tmsg.split(b'\n'): item = b_item.decode().strip() if not item: continue @@ -416,7 +421,7 @@ def lgettext(self, message): if tmsg is missing: if self._fallback: return self._fallback.lgettext(message) - return message + tmsg = message if self._output_charset: return tmsg.encode(self._output_charset) return tmsg.encode(locale.getpreferredencoding()) @@ -424,16 +429,16 @@ def lgettext(self, message): def lngettext(self, msgid1, msgid2, n): try: tmsg = self._catalog[(msgid1, self.plural(n))] - if self._output_charset: - return tmsg.encode(self._output_charset) - return tmsg.encode(locale.getpreferredencoding()) except KeyError: if self._fallback: return self._fallback.lngettext(msgid1, msgid2, n) if n == 1: - return msgid1 + tmsg = msgid1 else: - return msgid2 + tmsg = msgid2 + if self._output_charset: + return tmsg.encode(self._output_charset) + return tmsg.encode(locale.getpreferredencoding()) def gettext(self, message): missing = object() @@ -573,11 +578,11 @@ def dgettext(domain, message): return t.gettext(message) def ldgettext(domain, message): + codeset = _localecodesets.get(domain) try: - t = translation(domain, _localedirs.get(domain, None), - codeset=_localecodesets.get(domain)) + t = translation(domain, _localedirs.get(domain, None), codeset=codeset) except OSError: - return message + return message.encode(codeset or locale.getpreferredencoding()) return t.lgettext(message) def dngettext(domain, msgid1, msgid2, n): @@ -592,14 +597,15 @@ def dngettext(domain, msgid1, msgid2, n): return t.ngettext(msgid1, msgid2, n) def ldngettext(domain, msgid1, msgid2, n): + codeset = _localecodesets.get(domain) try: - t = translation(domain, _localedirs.get(domain, None), - codeset=_localecodesets.get(domain)) + t = translation(domain, _localedirs.get(domain, None), codeset=codeset) except OSError: if n == 1: - return msgid1 + tmsg = msgid1 else: - return msgid2 + tmsg = msgid2 + return tmsg.encode(codeset or locale.getpreferredencoding()) return t.lngettext(msgid1, msgid2, n) def gettext(message): diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index a852443e687837..18738457208187 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -1,6 +1,7 @@ import os import base64 import gettext +import locale import unittest from test import support @@ -452,6 +453,122 @@ def test_plural_number(self): self.assertRaises(TypeError, f, object()) +class LGettextTestCase(GettextBaseTest): + def setUp(self): + GettextBaseTest.setUp(self) + self.mofile = MOFILE + + def test_lgettext(self): + lgettext = gettext.lgettext + ldgettext = gettext.ldgettext + self.assertEqual(lgettext('mullusk'), b'bacon') + self.assertEqual(lgettext('spam'), b'spam') + self.assertEqual(ldgettext('gettext', 'mullusk'), b'bacon') + self.assertEqual(ldgettext('gettext', 'spam'), b'spam') + + def test_lgettext_2(self): + with open(self.mofile, 'rb') as fp: + t = gettext.GNUTranslations(fp) + lgettext = t.lgettext + self.assertEqual(lgettext('mullusk'), b'bacon') + self.assertEqual(lgettext('spam'), b'spam') + + def test_lgettext_bind_textdomain_codeset(self): + lgettext = gettext.lgettext + ldgettext = gettext.ldgettext + saved_codeset = gettext.bind_textdomain_codeset('gettext') + try: + gettext.bind_textdomain_codeset('gettext', 'utf-16') + self.assertEqual(lgettext('mullusk'), 'bacon'.encode('utf-16')) + self.assertEqual(lgettext('spam'), 'spam'.encode('utf-16')) + self.assertEqual(ldgettext('gettext', 'mullusk'), 'bacon'.encode('utf-16')) + self.assertEqual(ldgettext('gettext', 'spam'), 'spam'.encode('utf-16')) + finally: + del gettext._localecodesets['gettext'] + gettext.bind_textdomain_codeset('gettext', saved_codeset) + + def test_lgettext_output_encoding(self): + with open(self.mofile, 'rb') as fp: + t = gettext.GNUTranslations(fp) + lgettext = t.lgettext + t.set_output_charset('utf-16') + self.assertEqual(lgettext('mullusk'), 'bacon'.encode('utf-16')) + self.assertEqual(lgettext('spam'), 'spam'.encode('utf-16')) + + def test_lngettext(self): + lngettext = gettext.lngettext + ldngettext = gettext.ldngettext + x = lngettext('There is %s file', 'There are %s files', 1) + self.assertEqual(x, b'Hay %s fichero') + x = lngettext('There is %s file', 'There are %s files', 2) + self.assertEqual(x, b'Hay %s ficheros') + x = lngettext('There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, b'There is %s directory') + x = lngettext('There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, b'There are %s directories') + x = ldngettext('gettext', 'There is %s file', 'There are %s files', 1) + self.assertEqual(x, b'Hay %s fichero') + x = ldngettext('gettext', 'There is %s file', 'There are %s files', 2) + self.assertEqual(x, b'Hay %s ficheros') + x = ldngettext('gettext', 'There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, b'There is %s directory') + x = ldngettext('gettext', 'There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, b'There are %s directories') + + def test_lngettext_2(self): + with open(self.mofile, 'rb') as fp: + t = gettext.GNUTranslations(fp) + lngettext = t.lngettext + x = lngettext('There is %s file', 'There are %s files', 1) + self.assertEqual(x, b'Hay %s fichero') + x = lngettext('There is %s file', 'There are %s files', 2) + self.assertEqual(x, b'Hay %s ficheros') + x = lngettext('There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, b'There is %s directory') + x = lngettext('There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, b'There are %s directories') + + def test_lngettext_bind_textdomain_codeset(self): + lngettext = gettext.lngettext + ldngettext = gettext.ldngettext + saved_codeset = gettext.bind_textdomain_codeset('gettext') + try: + gettext.bind_textdomain_codeset('gettext', 'utf-16') + x = lngettext('There is %s file', 'There are %s files', 1) + self.assertEqual(x, 'Hay %s fichero'.encode('utf-16')) + x = lngettext('There is %s file', 'There are %s files', 2) + self.assertEqual(x, 'Hay %s ficheros'.encode('utf-16')) + x = lngettext('There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, 'There is %s directory'.encode('utf-16')) + x = lngettext('There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, 'There are %s directories'.encode('utf-16')) + x = ldngettext('gettext', 'There is %s file', 'There are %s files', 1) + self.assertEqual(x, 'Hay %s fichero'.encode('utf-16')) + x = ldngettext('gettext', 'There is %s file', 'There are %s files', 2) + self.assertEqual(x, 'Hay %s ficheros'.encode('utf-16')) + x = ldngettext('gettext', 'There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, 'There is %s directory'.encode('utf-16')) + x = ldngettext('gettext', 'There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, 'There are %s directories'.encode('utf-16')) + finally: + del gettext._localecodesets['gettext'] + gettext.bind_textdomain_codeset('gettext', saved_codeset) + + def test_lngettext_output_encoding(self): + with open(self.mofile, 'rb') as fp: + t = gettext.GNUTranslations(fp) + lngettext = t.lngettext + t.set_output_charset('utf-16') + x = lngettext('There is %s file', 'There are %s files', 1) + self.assertEqual(x, 'Hay %s fichero'.encode('utf-16')) + x = lngettext('There is %s file', 'There are %s files', 2) + self.assertEqual(x, 'Hay %s ficheros'.encode('utf-16')) + x = lngettext('There is %s directory', 'There are %s directories', 1) + self.assertEqual(x, 'There is %s directory'.encode('utf-16')) + x = lngettext('There is %s directory', 'There are %s directories', 2) + self.assertEqual(x, 'There are %s directories'.encode('utf-16')) + + class GNUTranslationParsingTest(GettextBaseTest): def test_plural_form_error_issue17898(self): with open(MOFILE, 'wb') as fp: @@ -469,13 +586,10 @@ def setUp(self): self._ = self.t.gettext def test_unicode_msgid(self): - unless = self.assertTrue - unless(isinstance(self._(''), str)) - unless(isinstance(self._(''), str)) + self.assertIsInstance(self._(''), str) def test_unicode_msgstr(self): - eq = self.assertEqual - eq(self._('ab\xde'), '\xa4yz') + self.assertEqual(self._('ab\xde'), '\xa4yz') class WeirdMetadataTest(GettextBaseTest): @@ -544,7 +658,7 @@ def test_main(): # The original version was automatically generated from the sources with # pygettext. Later it was manually modified to add plural forms support. -''' +b''' # Dummy translation for the Python test_gettext.py module. # Copyright (C) 2001 Python Software Foundation # Barry Warsaw , 2000. @@ -604,7 +718,7 @@ def test_main(): # Here's the second example po file example, used to generate the UMO_DATA # containing utf-8 encoded Unicode strings -''' +b''' # Dummy translation for the Python test_gettext.py module. # Copyright (C) 2001 Python Software Foundation # Barry Warsaw , 2000. @@ -627,7 +741,7 @@ def test_main(): # Here's the third example po file, used to generate MMO_DATA -''' +b''' msgid "" msgstr "" "Project-Id-Version: No Project 0.0\n" @@ -646,7 +760,7 @@ def test_main(): # messages.po, used for bug 17898 # -''' +b''' # test file for http://bugs.python.org/issue17898 msgid "" msgstr "" diff --git a/Misc/NEWS b/Misc/NEWS index 51edb1debb4a42..d2bdd1e756fd54 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- bpo-29755: Fixed the lgettext() family of functions in the gettext module. + They now always return bytes. + - [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` From 4a66524006852fc982aebafa277f2c043d9ad149 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Jun 2017 16:05:11 +0200 Subject: [PATCH 0467/1345] bpo-30694: Upgrade Modules/expat/ to libexpat 2.2.1 (#2300) (#2313) New file: Modules/expat/siphash.h. (cherry picked from commit 5ff7132313eb651107b179d20218dfe5d4e47f13) --- Misc/NEWS | 9 + Modules/expat/COPYING | 2 +- Modules/expat/expat.h | 22 +- Modules/expat/expat_external.h | 5 +- Modules/expat/siphash.h | 344 +++++++++++++++++++ Modules/expat/winconfig.h | 10 + Modules/expat/xmlparse.c | 609 ++++++++++++++++++++++++++------- Modules/expat/xmlrole.c | 10 +- Modules/expat/xmltok.c | 34 +- 9 files changed, 887 insertions(+), 158 deletions(-) create mode 100644 Modules/expat/siphash.h diff --git a/Misc/NEWS b/Misc/NEWS index d2bdd1e756fd54..70a531c76f3711 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,15 @@ Core and Builtins Library ------- +- [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes + of multiple security vulnerabilities including: CVE-2017-9233 (External + entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), + CVE-2016-0718 (Fix regression bugs from 2.2.0's fix to CVE-2016-0718) + and CVE-2012-0876 (Counter hash flooding with SipHash). + Note: the CVE-2016-5300 (Use os-specific entropy sources like getrandom) + doesn't impact Python, since Python already gets entropy from the OS to set + the expat secret using ``XML_SetHashSalt()``. + - bpo-29755: Fixed the lgettext() family of functions in the gettext module. They now always return bytes. diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 092c83baee13ae..8d288f0f28fddd 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2016 Expat maintainers +Copyright (c) 2001-2017 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 086e24b39c51ae..28b0f954d41963 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -95,7 +95,9 @@ enum XML_Error { /* Added in 2.0. */ XML_ERROR_RESERVED_PREFIX_XML, XML_ERROR_RESERVED_PREFIX_XMLNS, - XML_ERROR_RESERVED_NAMESPACE_URI + XML_ERROR_RESERVED_NAMESPACE_URI, + /* Added in 2.2.1. */ + XML_ERROR_INVALID_ARGUMENT }; enum XML_Content_Type { @@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser); be called, despite an external subset being parsed. Note: If XML_DTD is not defined when Expat is compiled, returns XML_ERROR_FEATURE_REQUIRES_XML_DTD. + Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT. */ XMLPARSEAPI(enum XML_Error) XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD); @@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser); to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus this correspondds to an index into the atts array passed to the - XML_StartElementHandler. + XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to - XML_StartElementHandler, or -1 if there is no ID attribute. Each - attribute/value pair counts as 2; thus this correspondds to an - index into the atts array passed to the XML_StartElementHandler. + XML_StartElementHandler, or -1 if there is no ID attribute or + parser == NULL. Each attribute/value pair counts as 2; thus this + correspondds to an index into the atts array passed to the + XML_StartElementHandler. */ XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); @@ -901,6 +905,7 @@ enum XML_ParamEntityParsing { entities is requested; otherwise it will return non-zero. Note: If XML_SetParamEntityParsing is called after XML_Parse or XML_ParseBuffer, then it has no effect and will always return 0. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetParamEntityParsing(XML_Parser parser, @@ -910,6 +915,7 @@ XML_SetParamEntityParsing(XML_Parser parser, Helps in preventing DoS attacks based on predicting hash function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, @@ -936,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser); the location is the location of the character at which the error was detected; otherwise the location is the location of the last parse event, as described above. + + Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber + return 0 to indicate an error. + Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); @@ -1039,7 +1049,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 0 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 4860c5777aa018..4c9e5eabdee2ce 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -97,7 +97,10 @@ extern "C" { #endif #ifdef XML_UNICODE_WCHAR_T -#define XML_UNICODE +# define XML_UNICODE +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif #endif #ifdef XML_UNICODE /* Information is UTF-16 encoded. */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h new file mode 100644 index 00000000000000..23b56d2ae48754 --- /dev/null +++ b/Modules/expat/siphash.h @@ -0,0 +1,344 @@ +/* ========================================================================== + * siphash.h - SipHash-2-4 in a single header file + * -------------------------------------------------------------------------- + * Derived by William Ahern from the reference implementation[1] published[2] + * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. + * by Jean-Philippe Aumasson and Daniel J. Berstein. + * Minimal changes by Sebastian Pipping on top, details below. + * Licensed under the CC0 Public Domain Dedication license. + * + * 1. https://www.131002.net/siphash/siphash24.c + * 2. https://www.131002.net/siphash/ + * -------------------------------------------------------------------------- + * HISTORY: + * + * 2017-06-10 (Sebastian Pipping) + * - Clarify license note in the header + * - Address C89 issues: + * - Stop using inline keyword (and let compiler decide) + * - Turn integer suffix ULL to UL + * - Replace _Bool by int + * - Turn macro siphash24 into a function + * - Address invalid conversion (void pointer) by explicit cast + * - Always expose sip24_valid (for self-tests) + * + * 2012-11-04 - Born. (William Ahern) + * -------------------------------------------------------------------------- + * USAGE: + * + * SipHash-2-4 takes as input two 64-bit words as the key, some number of + * message bytes, and outputs a 64-bit word as the message digest. This + * implementation employs two data structures: a struct sipkey for + * representing the key, and a struct siphash for representing the hash + * state. + * + * For converting a 16-byte unsigned char array to a key, use either the + * macro sip_keyof or the routine sip_tokey. The former instantiates a + * compound literal key, while the latter requires a key object as a + * parameter. + * + * unsigned char secret[16]; + * arc4random_buf(secret, sizeof secret); + * struct sipkey *key = sip_keyof(secret); + * + * For hashing a message, use either the convenience macro siphash24 or the + * routines sip24_init, sip24_update, and sip24_final. + * + * struct siphash state; + * void *msg; + * size_t len; + * uint64_t hash; + * + * sip24_init(&state, key); + * sip24_update(&state, msg, len); + * hash = sip24_final(&state); + * + * or + * + * hash = siphash24(msg, len, key); + * + * To convert the 64-bit hash value to a canonical 8-byte little-endian + * binary representation, use either the macro sip_binof or the routine + * sip_tobin. The former instantiates and returns a compound literal array, + * while the latter requires an array object as a parameter. + * -------------------------------------------------------------------------- + * NOTES: + * + * o Neither sip_keyof, sip_binof, nor siphash24 will work with compilers + * lacking compound literal support. Instead, you must use the lower-level + * interfaces which take as parameters the temporary state objects. + * + * o Uppercase macros may evaluate parameters more than once. Lowercase + * macros should not exhibit any such side effects. + * ========================================================================== + */ +#ifndef SIPHASH_H +#define SIPHASH_H + +#include /* size_t */ +#include /* uint64_t uint32_t uint8_t */ + + +#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) + +#define SIP_U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v) >> 0); (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24); + +#define SIP_U64TO8_LE(p, v) \ + SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0)); \ + SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define SIP_U8TO64_LE(p) \ + (((uint64_t)((p)[0]) << 0) | \ + ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | \ + ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | \ + ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | \ + ((uint64_t)((p)[7]) << 56)) + + +#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 } + +struct siphash { + uint64_t v0, v1, v2, v3; + + unsigned char buf[8], *p; + uint64_t c; +}; /* struct siphash */ + + +#define SIP_KEYLEN 16 + +struct sipkey { + uint64_t k[2]; +}; /* struct sipkey */ + +#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k)) + +static struct sipkey *sip_tokey(struct sipkey *key, const void *src) { + key->k[0] = SIP_U8TO64_LE((const unsigned char *)src); + key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8); + return key; +} /* sip_tokey() */ + + +#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v)) + +static void *sip_tobin(void *dst, uint64_t u64) { + SIP_U64TO8_LE((unsigned char *)dst, u64); + return dst; +} /* sip_tobin() */ + + +static void sip_round(struct siphash *H, const int rounds) { + int i; + + for (i = 0; i < rounds; i++) { + H->v0 += H->v1; + H->v1 = SIP_ROTL(H->v1, 13); + H->v1 ^= H->v0; + H->v0 = SIP_ROTL(H->v0, 32); + + H->v2 += H->v3; + H->v3 = SIP_ROTL(H->v3, 16); + H->v3 ^= H->v2; + + H->v0 += H->v3; + H->v3 = SIP_ROTL(H->v3, 21); + H->v3 ^= H->v0; + + H->v2 += H->v1; + H->v1 = SIP_ROTL(H->v1, 17); + H->v1 ^= H->v2; + H->v2 = SIP_ROTL(H->v2, 32); + } +} /* sip_round() */ + + +static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { + H->v0 = 0x736f6d6570736575UL ^ key->k[0]; + H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; + H->v2 = 0x6c7967656e657261UL ^ key->k[0]; + H->v3 = 0x7465646279746573UL ^ key->k[1]; + + H->p = H->buf; + H->c = 0; + + return H; +} /* sip24_init() */ + + +#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) + +static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { + const unsigned char *p = (const unsigned char *)src, *pe = p + len; + uint64_t m; + + do { + while (p < pe && H->p < sip_endof(H->buf)) + *H->p++ = *p++; + + if (H->p < sip_endof(H->buf)) + break; + + m = SIP_U8TO64_LE(H->buf); + H->v3 ^= m; + sip_round(H, 2); + H->v0 ^= m; + + H->p = H->buf; + H->c += 8; + } while (p < pe); + + return H; +} /* sip24_update() */ + + +static uint64_t sip24_final(struct siphash *H) { + char left = H->p - H->buf; + uint64_t b = (H->c + left) << 56; + + switch (left) { + case 7: b |= (uint64_t)H->buf[6] << 48; + case 6: b |= (uint64_t)H->buf[5] << 40; + case 5: b |= (uint64_t)H->buf[4] << 32; + case 4: b |= (uint64_t)H->buf[3] << 24; + case 3: b |= (uint64_t)H->buf[2] << 16; + case 2: b |= (uint64_t)H->buf[1] << 8; + case 1: b |= (uint64_t)H->buf[0] << 0; + case 0: break; + } + + H->v3 ^= b; + sip_round(H, 2); + H->v0 ^= b; + H->v2 ^= 0xff; + sip_round(H, 4); + + return H->v0 ^ H->v1 ^ H->v2 ^ H->v3; +} /* sip24_final() */ + + +static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { + struct siphash state = SIPHASH_INITIALIZER; + return sip24_final(sip24_update(sip24_init(&state, key), src, len)); +} /* siphash24() */ + + +/* + * SipHash-2-4 output with + * k = 00 01 02 ... + * and + * in = (empty string) + * in = 00 (1 byte) + * in = 00 01 (2 bytes) + * in = 00 01 02 (3 bytes) + * ... + * in = 00 01 02 ... 3e (63 bytes) + */ +static int sip24_valid(void) { + static const unsigned char vectors[64][8] = { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + unsigned char in[64]; + struct sipkey k; + size_t i; + + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + + for (i = 0; i < sizeof in; ++i) { + in[i] = i; + + if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) + return 0; + } + + return 1; +} /* sip24_valid() */ + + +#if SIPHASH_MAIN + +#include + +int main(void) { + int ok = sip24_valid(); + + if (ok) + puts("OK"); + else + puts("FAIL"); + + return !ok; +} /* main() */ + +#endif /* SIPHASH_MAIN */ + + +#endif /* SIPHASH_H */ diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index c1b791d62d0d0c..9bf014d7fbabd6 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -17,6 +17,12 @@ #include #include + +#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ +# include +#else /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #define XML_NS 1 #define XML_DTD 1 #define XML_CONTEXT_BYTES 1024 @@ -27,4 +33,8 @@ /* Windows has memmove() available. */ #define HAVE_MEMMOVE + +#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b308e67e2d390d..76f078e2505f90 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,13 +1,19 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. + + 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ +#define _GNU_SOURCE /* syscall prototype */ + #include #include /* memset(), memcpy() */ #include #include /* UINT_MAX */ +#include /* fprintf */ +#include /* getenv */ -#ifdef WIN32 +#ifdef _WIN32 #define getpid GetCurrentProcessId #else #include /* gettimeofday() */ @@ -17,20 +23,15 @@ #define XML_BUILDING_EXPAT 1 -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #elif defined(HAVE_EXPAT_CONFIG_H) #include -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "ascii.h" #include "expat.h" +#include "siphash.h" #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX @@ -109,17 +110,11 @@ typedef struct { const XML_Memory_Handling_Suite *mem; } HASH_TABLE; -/* Basic character hash algorithm, taken from Python's string hash: - h = h * 1000003 ^ character, the constant being a prime number. +static size_t +keylen(KEY s); -*/ -#ifdef XML_UNICODE -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned short)(c)) -#else -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned char)(c)) -#endif +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key); /* For probing (after a collision) we need a step size relative prime to the hash table size, which is a power of 2. We use double-hashing, @@ -355,6 +350,8 @@ doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ +static void +freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, BINDING **bindingsPtr); @@ -697,10 +694,84 @@ static const XML_Char implicitContext[] = { ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0' }; + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# include + +# if defined(HAVE_GETRANDOM) +# include /* getrandom */ +# else +# include /* syscall */ +# include /* SYS_getrandom */ +# endif + +/* Obtain entropy on Linux 3.17+ */ +static int +writeRandomBytes_getrandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + const unsigned int getrandomFlags = 0; + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const int bytesWrittenMore = +#if defined(HAVE_GETRANDOM) + getrandom(currentTarget, bytesToWrite, getrandomFlags); +#else + syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); +#endif + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR || errno == EAGAIN)); + + return success; +} + +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + + +#ifdef _WIN32 + +typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); + +/* Obtain entropy on Windows XP / Windows Server 2003 and later. + * Hint on RtlGenRandom and the following article from libsodioum. + * + * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI + * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/ + */ +static int +writeRandomBytes_RtlGenRandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + + if (advapi32) { + const RTLGENRANDOM_FUNC RtlGenRandom + = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036"); + if (RtlGenRandom) { + if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) { + success = 1; + } + } + FreeLibrary(advapi32); + } + + return success; +} + +#endif /* _WIN32 */ + + static unsigned long gather_time_entropy(void) { -#ifdef WIN32 +#ifdef _WIN32 FILETIME ft; GetSystemTimeAsFileTime(&ft); /* never fails */ return ft.dwHighDateTime ^ ft.dwLowDateTime; @@ -716,20 +787,62 @@ gather_time_entropy(void) #endif } +#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) +# include +#endif + +static unsigned long +ENTROPY_DEBUG(const char * label, unsigned long entropy) { + const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); + if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { + fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", + label, + (int)sizeof(entropy) * 2, entropy, + (unsigned long)sizeof(entropy)); + } + return entropy; +} + static unsigned long generate_hash_secret_salt(XML_Parser parser) { - /* Process ID is 0 bits entropy if attacker has local access - * XML_Parser address is few bits of entropy if attacker has local access */ - const unsigned long entropy = - gather_time_entropy() ^ getpid() ^ (unsigned long)parser; + unsigned long entropy; + (void)parser; +#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) + (void)gather_time_entropy; + arc4random_buf(&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random_buf", entropy); +#else + /* Try high quality providers first .. */ +#ifdef _WIN32 + if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("RtlGenRandom", entropy); + } +#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getrandom", entropy); + } +#endif + /* .. and self-made low quality for backup: */ + + /* Process ID is 0 bits entropy if attacker has local access */ + entropy = gather_time_entropy() ^ getpid(); /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return entropy * 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { - return entropy * (unsigned long)2305843009213693951; + return ENTROPY_DEBUG("fallback(8)", + entropy * (unsigned long)2305843009213693951); } +#endif +} + +static unsigned long +get_hash_secret_salt(XML_Parser parser) { + if (parser->m_parentParser != NULL) + return get_hash_secret_salt(parser->m_parentParser); + return parser->m_hash_secret_salt; } static XML_Bool /* only valid for root parser */ @@ -960,6 +1073,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; OPEN_INTERNAL_ENTITY *openEntityList; + + if (parser == NULL) + return XML_FALSE; + if (parentParser) return XML_FALSE; /* move tagStack to freeTagList */ @@ -994,6 +1111,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { + if (parser == NULL) + return XML_STATUS_ERROR; /* Block after XML_Parse()/XML_ParseBuffer() has been called. XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. @@ -1017,52 +1136,88 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, { XML_Parser parser = oldParser; DTD *newDtd = NULL; - DTD *oldDtd = _dtd; - XML_StartElementHandler oldStartElementHandler = startElementHandler; - XML_EndElementHandler oldEndElementHandler = endElementHandler; - XML_CharacterDataHandler oldCharacterDataHandler = characterDataHandler; - XML_ProcessingInstructionHandler oldProcessingInstructionHandler - = processingInstructionHandler; - XML_CommentHandler oldCommentHandler = commentHandler; - XML_StartCdataSectionHandler oldStartCdataSectionHandler - = startCdataSectionHandler; - XML_EndCdataSectionHandler oldEndCdataSectionHandler - = endCdataSectionHandler; - XML_DefaultHandler oldDefaultHandler = defaultHandler; - XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler - = unparsedEntityDeclHandler; - XML_NotationDeclHandler oldNotationDeclHandler = notationDeclHandler; - XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler - = startNamespaceDeclHandler; - XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler - = endNamespaceDeclHandler; - XML_NotStandaloneHandler oldNotStandaloneHandler = notStandaloneHandler; - XML_ExternalEntityRefHandler oldExternalEntityRefHandler - = externalEntityRefHandler; - XML_SkippedEntityHandler oldSkippedEntityHandler = skippedEntityHandler; - XML_UnknownEncodingHandler oldUnknownEncodingHandler - = unknownEncodingHandler; - XML_ElementDeclHandler oldElementDeclHandler = elementDeclHandler; - XML_AttlistDeclHandler oldAttlistDeclHandler = attlistDeclHandler; - XML_EntityDeclHandler oldEntityDeclHandler = entityDeclHandler; - XML_XmlDeclHandler oldXmlDeclHandler = xmlDeclHandler; - ELEMENT_TYPE * oldDeclElementType = declElementType; - - void *oldUserData = userData; - void *oldHandlerArg = handlerArg; - XML_Bool oldDefaultExpandInternalEntities = defaultExpandInternalEntities; - XML_Parser oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; + DTD *oldDtd; + XML_StartElementHandler oldStartElementHandler; + XML_EndElementHandler oldEndElementHandler; + XML_CharacterDataHandler oldCharacterDataHandler; + XML_ProcessingInstructionHandler oldProcessingInstructionHandler; + XML_CommentHandler oldCommentHandler; + XML_StartCdataSectionHandler oldStartCdataSectionHandler; + XML_EndCdataSectionHandler oldEndCdataSectionHandler; + XML_DefaultHandler oldDefaultHandler; + XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler; + XML_NotationDeclHandler oldNotationDeclHandler; + XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler; + XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler; + XML_NotStandaloneHandler oldNotStandaloneHandler; + XML_ExternalEntityRefHandler oldExternalEntityRefHandler; + XML_SkippedEntityHandler oldSkippedEntityHandler; + XML_UnknownEncodingHandler oldUnknownEncodingHandler; + XML_ElementDeclHandler oldElementDeclHandler; + XML_AttlistDeclHandler oldAttlistDeclHandler; + XML_EntityDeclHandler oldEntityDeclHandler; + XML_XmlDeclHandler oldXmlDeclHandler; + ELEMENT_TYPE * oldDeclElementType; + + void *oldUserData; + void *oldHandlerArg; + XML_Bool oldDefaultExpandInternalEntities; + XML_Parser oldExternalEntityRefHandlerArg; #ifdef XML_DTD - enum XML_ParamEntityParsing oldParamEntityParsing = paramEntityParsing; - int oldInEntityValue = prologState.inEntityValue; + enum XML_ParamEntityParsing oldParamEntityParsing; + int oldInEntityValue; #endif - XML_Bool oldns_triplets = ns_triplets; + XML_Bool oldns_triplets; /* Note that the new parser shares the same hash secret as the old parser, so that dtdCopy and copyEntityTable can lookup values from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt = hash_secret_salt; + unsigned long oldhash_secret_salt; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) + return NULL; + + /* Stash the original parser contents on the stack */ + oldDtd = _dtd; + oldStartElementHandler = startElementHandler; + oldEndElementHandler = endElementHandler; + oldCharacterDataHandler = characterDataHandler; + oldProcessingInstructionHandler = processingInstructionHandler; + oldCommentHandler = commentHandler; + oldStartCdataSectionHandler = startCdataSectionHandler; + oldEndCdataSectionHandler = endCdataSectionHandler; + oldDefaultHandler = defaultHandler; + oldUnparsedEntityDeclHandler = unparsedEntityDeclHandler; + oldNotationDeclHandler = notationDeclHandler; + oldStartNamespaceDeclHandler = startNamespaceDeclHandler; + oldEndNamespaceDeclHandler = endNamespaceDeclHandler; + oldNotStandaloneHandler = notStandaloneHandler; + oldExternalEntityRefHandler = externalEntityRefHandler; + oldSkippedEntityHandler = skippedEntityHandler; + oldUnknownEncodingHandler = unknownEncodingHandler; + oldElementDeclHandler = elementDeclHandler; + oldAttlistDeclHandler = attlistDeclHandler; + oldEntityDeclHandler = entityDeclHandler; + oldXmlDeclHandler = xmlDeclHandler; + oldDeclElementType = declElementType; + + oldUserData = userData; + oldHandlerArg = handlerArg; + oldDefaultExpandInternalEntities = defaultExpandInternalEntities; + oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; +#ifdef XML_DTD + oldParamEntityParsing = paramEntityParsing; + oldInEntityValue = prologState.inEntityValue; +#endif + oldns_triplets = ns_triplets; + /* Note that the new parser shares the same hash secret as the old + parser, so that dtdCopy and copyEntityTable can lookup values + from hash tables associated with either parser without us having + to worry which hash secrets each table has. + */ + oldhash_secret_salt = hash_secret_salt; #ifdef XML_DTD if (!context) @@ -1228,12 +1383,15 @@ XML_ParserFree(XML_Parser parser) void XMLCALL XML_UseParserAsHandlerArg(XML_Parser parser) { - handlerArg = parser; + if (parser != NULL) + handlerArg = parser; } enum XML_Error XMLCALL XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) @@ -1248,6 +1406,8 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) void XMLCALL XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { + if (parser == NULL) + return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return; @@ -1257,6 +1417,8 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) void XMLCALL XML_SetUserData(XML_Parser parser, void *p) { + if (parser == NULL) + return; if (handlerArg == userData) handlerArg = userData = p; else @@ -1266,6 +1428,8 @@ XML_SetUserData(XML_Parser parser, void *p) enum XML_Status XMLCALL XML_SetBase(XML_Parser parser, const XML_Char *p) { + if (parser == NULL) + return XML_STATUS_ERROR; if (p) { p = poolCopyString(&_dtd->pool, p); if (!p) @@ -1280,18 +1444,24 @@ XML_SetBase(XML_Parser parser, const XML_Char *p) const XML_Char * XMLCALL XML_GetBase(XML_Parser parser) { + if (parser == NULL) + return NULL; return curBase; } int XMLCALL XML_GetSpecifiedAttributeCount(XML_Parser parser) { + if (parser == NULL) + return -1; return nSpecifiedAtts; } int XMLCALL XML_GetIdAttributeIndex(XML_Parser parser) { + if (parser == NULL) + return -1; return idAttIndex; } @@ -1299,6 +1469,8 @@ XML_GetIdAttributeIndex(XML_Parser parser) const XML_AttrInfo * XMLCALL XML_GetAttributeInfo(XML_Parser parser) { + if (parser == NULL) + return NULL; return attInfo; } #endif @@ -1308,6 +1480,8 @@ XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { + if (parser == NULL) + return; startElementHandler = start; endElementHandler = end; } @@ -1315,34 +1489,39 @@ XML_SetElementHandler(XML_Parser parser, void XMLCALL XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) { - startElementHandler = start; + if (parser != NULL) + startElementHandler = start; } void XMLCALL XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) { - endElementHandler = end; + if (parser != NULL) + endElementHandler = end; } void XMLCALL XML_SetCharacterDataHandler(XML_Parser parser, XML_CharacterDataHandler handler) { - characterDataHandler = handler; + if (parser != NULL) + characterDataHandler = handler; } void XMLCALL XML_SetProcessingInstructionHandler(XML_Parser parser, XML_ProcessingInstructionHandler handler) { - processingInstructionHandler = handler; + if (parser != NULL) + processingInstructionHandler = handler; } void XMLCALL XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) { - commentHandler = handler; + if (parser != NULL) + commentHandler = handler; } void XMLCALL @@ -1350,6 +1529,8 @@ XML_SetCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start, XML_EndCdataSectionHandler end) { + if (parser == NULL) + return; startCdataSectionHandler = start; endCdataSectionHandler = end; } @@ -1357,19 +1538,23 @@ XML_SetCdataSectionHandler(XML_Parser parser, void XMLCALL XML_SetStartCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start) { - startCdataSectionHandler = start; + if (parser != NULL) + startCdataSectionHandler = start; } void XMLCALL XML_SetEndCdataSectionHandler(XML_Parser parser, XML_EndCdataSectionHandler end) { - endCdataSectionHandler = end; + if (parser != NULL) + endCdataSectionHandler = end; } void XMLCALL XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_FALSE; } @@ -1378,6 +1563,8 @@ void XMLCALL XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_TRUE; } @@ -1387,6 +1574,8 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start, XML_EndDoctypeDeclHandler end) { + if (parser == NULL) + return; startDoctypeDeclHandler = start; endDoctypeDeclHandler = end; } @@ -1394,27 +1583,31 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start) { - startDoctypeDeclHandler = start; + if (parser != NULL) + startDoctypeDeclHandler = start; } void XMLCALL XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) { - endDoctypeDeclHandler = end; + if (parser != NULL) + endDoctypeDeclHandler = end; } void XMLCALL XML_SetUnparsedEntityDeclHandler(XML_Parser parser, XML_UnparsedEntityDeclHandler handler) { - unparsedEntityDeclHandler = handler; + if (parser != NULL) + unparsedEntityDeclHandler = handler; } void XMLCALL XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) { - notationDeclHandler = handler; + if (parser != NULL) + notationDeclHandler = handler; } void XMLCALL @@ -1422,6 +1615,8 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start, XML_EndNamespaceDeclHandler end) { + if (parser == NULL) + return; startNamespaceDeclHandler = start; endNamespaceDeclHandler = end; } @@ -1429,32 +1624,38 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start) { - startNamespaceDeclHandler = start; + if (parser != NULL) + startNamespaceDeclHandler = start; } void XMLCALL XML_SetEndNamespaceDeclHandler(XML_Parser parser, XML_EndNamespaceDeclHandler end) { - endNamespaceDeclHandler = end; + if (parser != NULL) + endNamespaceDeclHandler = end; } void XMLCALL XML_SetNotStandaloneHandler(XML_Parser parser, XML_NotStandaloneHandler handler) { - notStandaloneHandler = handler; + if (parser != NULL) + notStandaloneHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandler(XML_Parser parser, XML_ExternalEntityRefHandler handler) { - externalEntityRefHandler = handler; + if (parser != NULL) + externalEntityRefHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) { + if (parser == NULL) + return; if (arg) externalEntityRefHandlerArg = (XML_Parser)arg; else @@ -1465,7 +1666,8 @@ void XMLCALL XML_SetSkippedEntityHandler(XML_Parser parser, XML_SkippedEntityHandler handler) { - skippedEntityHandler = handler; + if (parser != NULL) + skippedEntityHandler = handler; } void XMLCALL @@ -1473,6 +1675,8 @@ XML_SetUnknownEncodingHandler(XML_Parser parser, XML_UnknownEncodingHandler handler, void *data) { + if (parser == NULL) + return; unknownEncodingHandler = handler; unknownEncodingHandlerData = data; } @@ -1481,33 +1685,39 @@ void XMLCALL XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) { - elementDeclHandler = eldecl; + if (parser != NULL) + elementDeclHandler = eldecl; } void XMLCALL XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) { - attlistDeclHandler = attdecl; + if (parser != NULL) + attlistDeclHandler = attdecl; } void XMLCALL XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) { - entityDeclHandler = handler; + if (parser != NULL) + entityDeclHandler = handler; } void XMLCALL XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) { - xmlDeclHandler = handler; + if (parser != NULL) + xmlDeclHandler = handler; } int XMLCALL XML_SetParamEntityParsing(XML_Parser parser, enum XML_ParamEntityParsing peParsing) { + if (parser == NULL) + return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1523,6 +1733,10 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + if (parser == NULL) + return 0; + if (parser->m_parentParser) + return XML_SetHashSalt(parser->m_parentParser, hash_salt); /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1533,6 +1747,10 @@ XML_SetHashSalt(XML_Parser parser, enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { + errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1585,6 +1803,13 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) const char *end; int nLeftOver; enum XML_Status result; + /* Detect overflow (a+b > MAX <==> b > MAX-a) */ + if (len > ((XML_Size)-1) / 2 - parseEndByteIndex) { + errorCode = XML_ERROR_NO_MEMORY; + eventPtr = eventEndPtr = NULL; + processor = errorProcessor; + return XML_STATUS_ERROR; + } parseEndByteIndex += len; positionPtr = s; ps_finalBuffer = (XML_Bool)isFinal; @@ -1617,11 +1842,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) nLeftOver = s + len - end; if (nLeftOver) { if (buffer == NULL || nLeftOver > bufferLim - buffer) { - /* FIXME avoid integer overflow */ - char *temp; - temp = (buffer == NULL - ? (char *)MALLOC(len * 2) - : (char *)REALLOC(buffer, len * 2)); + /* avoid _signed_ integer overflow */ + char *temp = NULL; + const int bytesToAllocate = (int)((unsigned)len * 2U); + if (bytesToAllocate > 0) { + temp = (buffer == NULL + ? (char *)MALLOC(bytesToAllocate) + : (char *)REALLOC(buffer, bytesToAllocate)); + } if (temp == NULL) { errorCode = XML_ERROR_NO_MEMORY; eventPtr = eventEndPtr = NULL; @@ -1629,7 +1857,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) return XML_STATUS_ERROR; } buffer = temp; - bufferLim = buffer + len * 2; + bufferLim = buffer + bytesToAllocate; } memcpy(buffer, end, nLeftOver); } @@ -1659,6 +1887,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) const char *start; enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1712,6 +1942,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) void * XMLCALL XML_GetBuffer(XML_Parser parser, int len) { + if (parser == NULL) + return NULL; if (len < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -1808,6 +2040,8 @@ XML_GetBuffer(XML_Parser parser, int len) enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: if (resumable) { @@ -1840,6 +2074,8 @@ XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; if (ps_parsing != XML_SUSPENDED) { errorCode = XML_ERROR_NOT_SUSPENDED; return XML_STATUS_ERROR; @@ -1876,6 +2112,8 @@ XML_ResumeParser(XML_Parser parser) void XMLCALL XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) { + if (parser == NULL) + return; assert(status != NULL); *status = parser->m_parsingStatus; } @@ -1883,12 +2121,16 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) enum XML_Error XMLCALL XML_GetErrorCode(XML_Parser parser) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; return errorCode; } XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { + if (parser == NULL) + return -1; if (eventPtr) return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; @@ -1897,6 +2139,8 @@ XML_GetCurrentByteIndex(XML_Parser parser) int XMLCALL XML_GetCurrentByteCount(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventEndPtr && eventPtr) return (int)(eventEndPtr - eventPtr); return 0; @@ -1906,11 +2150,19 @@ const char * XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { #ifdef XML_CONTEXT_BYTES + if (parser == NULL) + return NULL; if (eventPtr && buffer) { - *offset = (int)(eventPtr - buffer); - *size = (int)(bufferEnd - buffer); + if (offset != NULL) + *offset = (int)(eventPtr - buffer); + if (size != NULL) + *size = (int)(bufferEnd - buffer); return buffer; } +#else + (void)parser; + (void)offset; + (void)size; #endif /* defined XML_CONTEXT_BYTES */ return (char *) 0; } @@ -1918,6 +2170,8 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) XML_Size XMLCALL XML_GetCurrentLineNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1928,6 +2182,8 @@ XML_GetCurrentLineNumber(XML_Parser parser) XML_Size XMLCALL XML_GetCurrentColumnNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1938,30 +2194,38 @@ XML_GetCurrentColumnNumber(XML_Parser parser) void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - FREE(model); + if (parser != NULL) + FREE(model); } void * XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { + if (parser == NULL) + return NULL; return MALLOC(size); } void * XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { + if (parser == NULL) + return NULL; return REALLOC(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - FREE(ptr); + if (parser != NULL) + FREE(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { + if (parser == NULL) + return; if (defaultHandler) { if (openInternalEntities) reportDefault(parser, @@ -2468,7 +2732,7 @@ doContent(XML_Parser parser, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { + if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2511,8 +2775,10 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; poolFinish(&tempPool); result = storeAtts(parser, enc, s, &name, &bindings); - if (result) + if (result != XML_ERROR_NONE) { + freeBindings(parser, bindings); return result; + } poolFinish(&tempPool); if (startElementHandler) { startElementHandler(handlerArg, name.str, (const XML_Char **)atts); @@ -2527,15 +2793,7 @@ doContent(XML_Parser parser, if (noElmHandlers && defaultHandler) reportDefault(parser, enc, s, next); poolClear(&tempPool); - while (bindings) { - BINDING *b = bindings; - if (endNamespaceDeclHandler) - endNamespaceDeclHandler(handlerArg, b->prefix->name); - bindings = bindings->nextTagBinding; - b->nextTagBinding = freeBindingList; - freeBindingList = b; - b->prefix->binding = b->prevPrefixBinding; - } + freeBindings(parser, bindings); } if (tagLevel == 0) return epilogProcessor(parser, next, end, nextPtr); @@ -2733,6 +2991,29 @@ doContent(XML_Parser parser, /* not reached */ } +/* This function does not call free() on the allocated memory, merely + * moving it to the parser's freeBindingList where it can be freed or + * reused as appropriate. + */ +static void +freeBindings(XML_Parser parser, BINDING *bindings) +{ + while (bindings) { + BINDING *b = bindings; + + /* startNamespaceDeclHandler will have been called for this + * binding in addBindings(), so call the end handler now. + */ + if (endNamespaceDeclHandler) + endNamespaceDeclHandler(handlerArg, b->prefix->name); + + bindings = bindings->nextTagBinding; + b->nextTagBinding = freeBindingList; + freeBindingList = b; + b->prefix->binding = b->prevPrefixBinding; + } +} + /* Precondition: all arguments must be non-NULL; Purpose: - normalize attributes @@ -2957,7 +3238,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (s[-1] == 2) { /* prefixed */ ATTRIBUTE_ID *id; const BINDING *b; - unsigned long uriHash = hash_secret_salt; + unsigned long uriHash; + struct siphash sip_state; + struct sipkey sip_key; + + copy_salt_to_sipkey(parser, &sip_key); + sip24_init(&sip_state, &sip_key); + ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (!id || !id->prefix) @@ -2966,22 +3253,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (!b) return XML_ERROR_UNBOUND_PREFIX; - /* as we expand the name we also calculate its hash value */ for (j = 0; j < b->uriLen; j++) { const XML_Char c = b->uri[j]; if (!poolAppendChar(&tempPool, c)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } + + sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); + while (*s++ != XML_T(ASCII_COLON)) ; + + sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); + do { /* copies null terminator */ - const XML_Char c = *s; if (!poolAppendChar(&tempPool, *s)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } while (*s++); + uriHash = (unsigned long)sip24_final(&sip_state); + { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). */ @@ -3695,6 +3986,14 @@ entityValueInitProcessor(XML_Parser parser, *nextPtr = next; return XML_ERROR_NONE; } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "internalEventEndPtr = NULL; textStart = (char *)entity->textPtr; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4916,6 +5217,8 @@ internalEntityProcessor(XML_Parser parser, entity = openEntity->entity; textStart = ((char *)entity->textPtr) + entity->processed; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -5876,7 +6179,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H newE->defaultAtts = (DEFAULT_ATTRIBUTE *) ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (!newE->defaultAtts) { - ms->free_fcn(newE); return 0; } } @@ -6011,13 +6313,32 @@ keyeq(KEY s1, KEY s2) return XML_FALSE; } +static size_t +keylen(KEY s) +{ + size_t len = 0; + for (; *s; s++, len++); + return len; +} + +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key) +{ + key->k[0] = 0; + key->k[1] = get_hash_secret_salt(parser); +} + static unsigned long FASTCALL hash(XML_Parser parser, KEY s) { - unsigned long h = hash_secret_salt; - while (*s) - h = CHAR_HASH(h, *s++); - return h; + struct siphash state; + struct sipkey key; + (void)sip_tobin; + (void)sip24_valid; + copy_salt_to_sipkey(parser, &key); + sip24_init(&state, &key); + sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + return (unsigned long)sip24_final(&state); } static NAMED * @@ -6260,6 +6581,35 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, return pool->start; } +static size_t +poolBytesToAllocateFor(int blockSize) +{ + /* Unprotected math would be: + ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char); + ** + ** Detect overflow, avoiding _signed_ overflow undefined behavior + ** For a + b * c we check b * c in isolation first, so that addition of a + ** on top has no chance of making us accept a small non-negative number + */ + const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */ + + if (blockSize <= 0) + return 0; + + if (blockSize > (int)(INT_MAX / stretch)) + return 0; + + { + const int stretchedBlockSize = blockSize * (int)stretch; + const int bytesToAllocate = (int)( + offsetof(BLOCK, s) + (unsigned)stretchedBlockSize); + if (bytesToAllocate < 0) + return 0; + + return (size_t)bytesToAllocate; + } +} + static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { @@ -6289,14 +6639,17 @@ poolGrow(STRING_POOL *pool) if (pool->blocks && pool->start == pool->blocks->s) { BLOCK *temp; int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + temp = (BLOCK *) - pool->mem->realloc_fcn(pool->blocks, - (offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char))); + pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -6308,16 +6661,26 @@ poolGrow(STRING_POOL *pool) else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; - else + else { + /* Detect overflow, avoiding _signed_ overflow undefined behavior */ + if ((int)((unsigned)blockSize * 2U) < 0) { + return XML_FALSE; + } blockSize *= 2; - tem = (BLOCK *)pool->mem->malloc_fcn(offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char)); + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate); if (!tem) return XML_FALSE; tem->size = blockSize; diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index fcd0dc6948f442..a7c56302796957 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index a29d9e2f8ac386..cdf0720dd89689 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -369,24 +363,24 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - enum XML_Convert_Result res = XML_CONVERT_COMPLETED; char *to; const char *from; - if (fromLim - *fromP > toLim - *toP) { - /* Avoid copying partial characters. */ - res = XML_CONVERT_OUTPUT_EXHAUSTED; - fromLim = *fromP + (toLim - *toP); - align_limit_to_full_utf8_characters(*fromP, &fromLim); - } + const char *fromLimInitial = fromLim; + + /* Avoid copying partial characters. */ + align_limit_to_full_utf8_characters(*fromP, &fromLim); + for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++) *to = *from; *fromP = from; *toP = to; - if ((to == toLim) && (from < fromLim)) + if (fromLim < fromLimInitial) + return XML_CONVERT_INPUT_INCOMPLETE; + else if ((to == toLim) && (from < fromLim)) return XML_CONVERT_OUTPUT_EXHAUSTED; else - return res; + return XML_CONVERT_COMPLETED; } static enum XML_Convert_Result PTRCALL @@ -402,7 +396,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; @@ -410,7 +404,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD3: if (fromLim - from < 3) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); @@ -441,6 +435,8 @@ utf8_toUtf16(const ENCODING *enc, break; } } + if (from < fromLim) + res = XML_CONVERT_OUTPUT_EXHAUSTED; after: *fromP = from; *toP = to; From 296dc492cc610f349db39a082a18042a79d0dc46 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 21 Jun 2017 21:43:47 -0400 Subject: [PATCH 0468/1345] [3.6] Fix trivial typo in idlelib/config.py (GH-2309) (#2321) Comceptually -> Conceptually (cherry picked from commit f3e8209) --- Lib/idlelib/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 6a9fc61c2b59d7..6d683e2b80ac07 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -7,7 +7,7 @@ configuration files, and if a user file becomes empty, it will be deleted. -The configuration database maps options to values. Comceptually, the +The configuration database maps options to values. Conceptually, the database keys are tuples (config-type, section, item). As implemented, there are separate dicts for default and user values. Each has config-type keys 'main', 'extensions', 'highlight', and 'keys'. The From 396998e0c6ee7e555133c53ce42224046b1255f7 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 21 Jun 2017 22:46:07 -0400 Subject: [PATCH 0469/1345] [3.6] Fix typo in idlelib.config_key.py (GH-2322) (#2323) (cherry picked from commit a0e911b) --- Lib/idlelib/config_key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 26022934c3300a..479d6ad313e35f 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -186,7 +186,7 @@ def ClearKeySeq(self): def LoadFinalKeyList(self): #these tuples are also available for use in validity checks - self.functionKeys=('F1','F2','F2','F4','F5','F6','F7','F8','F9', + self.functionKeys=('F1','F2','F3','F4','F5','F6','F7','F8','F9', 'F10','F11','F12') self.alphanumKeys=tuple(string.ascii_lowercase+string.digits) self.punctuationKeys=tuple('~!@#%^&*()_-+={}[]|;:,.<>/?') From e9b34c11001a0a2d71323e64fe7324fe6455e94f Mon Sep 17 00:00:00 2001 From: larryhastings Date: Thu, 22 Jun 2017 17:38:47 -0700 Subject: [PATCH 0470/1345] Add "Misc/NEWS.d" directory tree for "blurb". GH-2329 CPython workflow is changing! We're going to start using "blurb" to manage Misc/NEWS entries: https://github.com/python/core-workflow (This will be a big win for release managers, honest.) This checkin simply populates the "Misc/NEWS.d" subdirectory tree so that people can start putting their news entries in there. No other changes (yet). --- Misc/NEWS.d/next/Build/README.rst | 1 + Misc/NEWS.d/next/C API/README.rst | 1 + Misc/NEWS.d/next/Core and Builtins/README.rst | 1 + Misc/NEWS.d/next/Documentation/README.rst | 1 + Misc/NEWS.d/next/IDLE/README.rst | 1 + Misc/NEWS.d/next/Library/README.rst | 1 + Misc/NEWS.d/next/Security/README.rst | 1 + Misc/NEWS.d/next/Tests/README.rst | 1 + Misc/NEWS.d/next/Tools-Demos/README.rst | 1 + Misc/NEWS.d/next/Windows/README.rst | 1 + Misc/NEWS.d/next/macOS/README.rst | 1 + 11 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/README.rst create mode 100644 Misc/NEWS.d/next/C API/README.rst create mode 100644 Misc/NEWS.d/next/Core and Builtins/README.rst create mode 100644 Misc/NEWS.d/next/Documentation/README.rst create mode 100644 Misc/NEWS.d/next/IDLE/README.rst create mode 100644 Misc/NEWS.d/next/Library/README.rst create mode 100644 Misc/NEWS.d/next/Security/README.rst create mode 100644 Misc/NEWS.d/next/Tests/README.rst create mode 100644 Misc/NEWS.d/next/Tools-Demos/README.rst create mode 100644 Misc/NEWS.d/next/Windows/README.rst create mode 100644 Misc/NEWS.d/next/macOS/README.rst diff --git a/Misc/NEWS.d/next/Build/README.rst b/Misc/NEWS.d/next/Build/README.rst new file mode 100644 index 00000000000000..0d2d2c1fc38be9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Build* section in this directory. diff --git a/Misc/NEWS.d/next/C API/README.rst b/Misc/NEWS.d/next/C API/README.rst new file mode 100644 index 00000000000000..5a04f76f47b67e --- /dev/null +++ b/Misc/NEWS.d/next/C API/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *C API* section in this directory. diff --git a/Misc/NEWS.d/next/Core and Builtins/README.rst b/Misc/NEWS.d/next/Core and Builtins/README.rst new file mode 100644 index 00000000000000..52b8c3e6263163 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Core and Builtins* section in this directory. diff --git a/Misc/NEWS.d/next/Documentation/README.rst b/Misc/NEWS.d/next/Documentation/README.rst new file mode 100644 index 00000000000000..405f0ac01a7202 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Documentation* section in this directory. diff --git a/Misc/NEWS.d/next/IDLE/README.rst b/Misc/NEWS.d/next/IDLE/README.rst new file mode 100644 index 00000000000000..5475f7b42050cd --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *IDLE* section in this directory. diff --git a/Misc/NEWS.d/next/Library/README.rst b/Misc/NEWS.d/next/Library/README.rst new file mode 100644 index 00000000000000..6d2d30eca0fd05 --- /dev/null +++ b/Misc/NEWS.d/next/Library/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Library* section in this directory. diff --git a/Misc/NEWS.d/next/Security/README.rst b/Misc/NEWS.d/next/Security/README.rst new file mode 100644 index 00000000000000..84c1a3a6ed7a07 --- /dev/null +++ b/Misc/NEWS.d/next/Security/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Security* section in this directory. diff --git a/Misc/NEWS.d/next/Tests/README.rst b/Misc/NEWS.d/next/Tests/README.rst new file mode 100644 index 00000000000000..d2e50e43d84790 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Tests* section in this directory. diff --git a/Misc/NEWS.d/next/Tools-Demos/README.rst b/Misc/NEWS.d/next/Tools-Demos/README.rst new file mode 100644 index 00000000000000..357f82862cb51a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Tools/Demos* section in this directory. diff --git a/Misc/NEWS.d/next/Windows/README.rst b/Misc/NEWS.d/next/Windows/README.rst new file mode 100644 index 00000000000000..1e65de35f0ad97 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Windows* section in this directory. diff --git a/Misc/NEWS.d/next/macOS/README.rst b/Misc/NEWS.d/next/macOS/README.rst new file mode 100644 index 00000000000000..a3adb59b62631e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *macOS* section in this directory. From 8baf78b98f3acd76e51a4381542ff1612911b18f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 22 Jun 2017 18:52:57 -0700 Subject: [PATCH 0471/1345] [3.6] bpo-30619: Clarify typing.Union documentation (GH-2326) (GH-2337) When a class and its subclass are present, the latter is skipped. (cherry picked from commit 6580c19bbbe7bc9bc0884699afd69184f523b32e) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1780739ad1dd43..1e48fecdbf78a4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -939,7 +939,7 @@ The module defines the following classes, functions and decorators: Union[int, str] == Union[str, int] - * When a class and its subclass are present, the former is skipped, e.g.:: + * When a class and its subclass are present, the latter is skipped, e.g.:: Union[int, object] == object From d79ac2c2468f072653ed9096cd33f89acb6ff5bb Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 22 Jun 2017 21:24:43 -0700 Subject: [PATCH 0472/1345] [3.6] bpo-30709: Improve code example in Descriptor HowTo doc (GH-2339) (GH-2342) (cherry picked from commit b066edfb1b268e90ea11f45dd1827f46d7ceec88) --- Doc/howto/descriptor.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 2dd6c34e2ced02..b34937585ea444 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -252,10 +252,10 @@ to wrap access to the value attribute in a property data descriptor:: class Cell(object): . . . - def getvalue(self, obj): - "Recalculate cell before returning value" + def getvalue(self): + "Recalculate the cell before returning value" self.recalc() - return obj._value + return self._value value = property(getvalue) From bae5ddab2c7b602cc846beef7285b4d2ae376d66 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 22 Jun 2017 23:11:52 -0700 Subject: [PATCH 0473/1345] fix some reference to Unicode 8 that should be to Unicode 9 (#2346) --- Doc/library/stdtypes.rst | 2 +- Doc/reference/lexical_analysis.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 083fe5367af9d8..5ab8177490d6ce 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -354,7 +354,7 @@ Notes: The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedNumericType.txt + See http://www.unicode.org/Public/9.0.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 7f9c66481785c1..fff31a3ce4f242 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -313,7 +313,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -875,4 +875,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/9.0.0/ucd/NameAliases.txt From c1d5345679eaa5fccd719b1c130140eecc8ba4c8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 Jun 2017 13:52:06 +0300 Subject: [PATCH 0474/1345] [3.6] bpo-30727: Fix a race condition in test_threading. (GH-2334) (#2351) (cherry picked from commit 32cb968) --- Lib/test/lock_tests.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index a6cb3b169b6d73..4fa154f734d070 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -461,21 +461,28 @@ def _check_notify(self, cond): # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. N = 5 + ready = [] results1 = [] results2 = [] phase_num = 0 def f(): cond.acquire() + ready.append(phase_num) result = cond.wait() cond.release() results1.append((result, phase_num)) cond.acquire() + ready.append(phase_num) result = cond.wait() cond.release() results2.append((result, phase_num)) b = Bunch(f, N) b.wait_for_started() - _wait() + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issues #8799 and #30727. + while len(ready) < 5: + _wait() + ready.clear() self.assertEqual(results1, []) # Notify 3 threads at first cond.acquire() @@ -487,9 +494,9 @@ def f(): _wait() self.assertEqual(results1, [(True, 1)] * 3) self.assertEqual(results2, []) - # first wait, to ensure all workers settle into cond.wait() before - # we continue. See issue #8799 - _wait() + # make sure all awaken workers settle into cond.wait() + while len(ready) < 3: + _wait() # Notify 5 threads: they might be in their first or second wait cond.acquire() cond.notify(5) @@ -500,7 +507,9 @@ def f(): _wait() self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) self.assertEqual(results2, [(True, 2)] * 3) - _wait() # make sure all workers settle into cond.wait() + # make sure all workers settle into cond.wait() + while len(ready) < 5: + _wait() # Notify all threads: they are all in their second wait cond.acquire() cond.notify_all() From c472fb6b2744b36c7a0823c20e0d5ac9be3ea623 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 15:21:24 +0200 Subject: [PATCH 0475/1345] bpo-30602: Fix lastarg in os.spawnve() (#2287) (#2357) Fix a regression introduced by myself in the commit 526b22657cb18fe79118c2ea68511aca09430c2c. (cherry picked from commit c8d6ab2e25ff212702d387e516e258b1d8c52910) --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0a9123b61bdc06..11aaeef02bc50c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5189,7 +5189,7 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, goto fail_1; } if (i == 0 && !argvlist[0][0]) { - lastarg = i; + lastarg = i + 1; PyErr_SetString( PyExc_ValueError, "spawnv() arg 2 first element cannot be empty"); From d32a05953130fb5cc2d3c0c9fcb20ad0859353f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 15:21:33 +0200 Subject: [PATCH 0476/1345] [3.6] bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) (#2349) * bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) bpo-30726, bpo-29591: libexpat 2.2.1 of Modules/expat/ now uses a winconfig.h configuration file which already defines: * XML_NS * XML_DTD * BYTEORDER=1234 * XML_CONTEXT_BYTES=1024 * HAVE_MEMMOVE Remove these defines from PCbuild/_elementtree.vcxproj to prevent compiler warnings. Co-Authored-By: Jeremy Kloth (cherry picked from commit c8fb58bd7917151e63398587a7fc2126db7c26de) * bpo-30726: Fix elementtree warnings on Windows due to expat upgrade (#2319) * bpo-30726: Fix elementtree warnings on Windows Caused by usage of `getenv` which should be safe. And a few integer truncations which should also be ok. * bpo-30726: Don't ignore libexpat warnings which haypo intends to fix upstream (cherry picked from commit 87c65550730a8f85ce339ba197bce4fb7e836619) --- PCbuild/_elementtree.vcxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 725b5a967bebc9..639ba479db5786 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 @@ -103,4 +103,4 @@ - \ No newline at end of file + From cea2174ab7cce01c420b2770562be4c91f1f4e35 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 15:24:28 +0200 Subject: [PATCH 0477/1345] bpo-30604: Skip CoExtra tests if ctypes is missing (#2356) (#2358) (cherry picked from commit a4b091e135ccf345cfafdd8477aef897c5214f82) --- Lib/test/test_code.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 9f9df9dba6b3d6..68537482ac91f3 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -106,6 +106,10 @@ import threading import unittest import weakref +try: + import ctypes +except ImportError: + ctypes = None from test.support import (run_doctest, run_unittest, cpython_only, check_impl_detail) @@ -214,8 +218,7 @@ def callback(code): self.assertTrue(self.called) -if check_impl_detail(cpython=True): - import ctypes +if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) @@ -311,7 +314,7 @@ def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] - if check_impl_detail(cpython=True): + if check_impl_detail(cpython=True) and ctypes is not None: tests.append(CoExtra) run_unittest(*tests) From 1b7474dedcbbd731a362b17abfbd7e5a60b64f63 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 23 Jun 2017 12:59:59 -0400 Subject: [PATCH 0478/1345] [3.6] bpo-24813: IDLE: Add icon to help_about (GH-2335) (#2359) Patch by Cheryl Sabella (cherry picked from commit d352d68) --- Lib/idlelib/help_about.py | 16 ++++++++++++---- Lib/idlelib/idle_test/test_help_about.py | 7 +++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index f0a40e927c3634..e65e67c47ee0d7 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -4,8 +4,8 @@ import os from sys import version -from tkinter import Toplevel, Frame, Label, Button -from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW +from tkinter import Toplevel, Frame, Label, Button, PhotoImage +from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW, E from idlelib import textview @@ -62,7 +62,16 @@ def create_widgets(self): header = Label(frame_background, text='IDLE', fg=self.fg, bg=self.bg, font=('courier', 24, 'bold')) - header.grid(row=0, column=0, sticky=W, padx=10, pady=10) + header.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + tk_patchlevel = self.tk.call('info', 'patchlevel') + ext = '.png' if tk_patchlevel >= '8.6' else '.gif' + icon = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'Icons', f'idle_48{ext}') + self.icon_image = PhotoImage(master=self._root(), file=icon) + logo = Label(frame_background, image=self.icon_image, bg=self.bg) + logo.grid(row=0, column=0, sticky=W, rowspan=2, padx=10, pady=10) + byline_text = "Python's Integrated DeveLopment Environment" + 5*'\n' byline = Label(frame_background, text=byline_text, justify=LEFT, fg=self.fg, bg=self.bg) @@ -82,7 +91,6 @@ def create_widgets(self): pyver = Label(frame_background, text='Python version: ' + release, fg=self.fg, bg=self.bg) pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tk_patchlevel = self.tk.call('info', 'patchlevel') tkver = Label(frame_background, text='Tk version: ' + tk_patchlevel, fg=self.fg, bg=self.bg) tkver.grid(row=9, column=1, sticky=W, padx=2, pady=0) diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 15d1b6b42c797a..f3ca75cf0182bf 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -9,6 +9,7 @@ from idlelib.idle_test.mock_tk import Mbox_func from idlelib.help_about import AboutDialog as About from idlelib import textview +import os.path class LiveDialogTest(unittest.TestCase): """Simulate user clicking buttons other than [Close]. @@ -33,6 +34,12 @@ def test_dialog_title(self): """Test about dialog title""" self.assertEqual(self.dialog.title(), 'About IDLE') + def test_dialog_logo(self): + """Test about dialog logo.""" + path, file = os.path.split(self.dialog.icon_image['file']) + fn, ext = os.path.splitext(file) + self.assertEqual(fn, 'idle_48') + def test_printer_buttons(self): """Test buttons whose commands use printer function.""" dialog = self.dialog From e7135751b8e48af80665e40ac8fa6d0073e5affe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 Jun 2017 20:17:38 +0300 Subject: [PATCH 0479/1345] [3.6] bpo-30730: Prevent environment variables injection in subprocess on Windows. (GH-2325) (#2360) Prevent passing other invalid environment variables and command arguments.. (cherry picked from commit d174d24a5d37d1516b885dc7c82f71ecd5930700) --- Lib/subprocess.py | 8 ++++++-- Lib/test/test_subprocess.py | 40 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_winapi.c | 26 +++++++++++++++++++----- Objects/abstract.c | 4 ++-- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e626a8afdbb7f0..b790cbd65ba208 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1239,8 +1239,12 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # and pass it to fork_exec() if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d8d6d82713cded..9a9e3ca5791256 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -644,6 +644,46 @@ def test_empty_env(self): # environment b"['__CF_USER_TEXT_ENCODING']")) + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' diff --git a/Misc/NEWS b/Misc/NEWS index 70a531c76f3711..94a29d0363c6ad 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- [Security] bpo-30730: Prevent environment variables injection in subprocess on + Windows. Prevent passing other environment variables and command arguments. + - [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security vulnerabilities including: CVE-2017-9233 (External entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 248f4582c60fb4..71426bc12dc655 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -744,6 +744,20 @@ getenvironment(PyObject* environment) "environment can only contain strings"); goto error; } + if (PyUnicode_FindChar(key, '\0', 0, PyUnicode_GET_LENGTH(key), 1) != -1 || + PyUnicode_FindChar(value, '\0', 0, PyUnicode_GET_LENGTH(value), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto error; + } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key) == 0 || + PyUnicode_FindChar(key, '=', 1, PyUnicode_GET_LENGTH(key), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } if (totalsize > PY_SSIZE_T_MAX - PyUnicode_GET_LENGTH(key) - 1) { PyErr_SetString(PyExc_OverflowError, "environment too long"); goto error; @@ -830,7 +844,8 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - wchar_t *wenvironment; + const wchar_t *wenvironment; + Py_ssize_t wenvironment_size; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); @@ -846,12 +861,13 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, if (env_mapping != Py_None) { environment = getenvironment(env_mapping); - if (! environment) + if (environment == NULL) { return NULL; + } + /* contains embedded null characters */ wenvironment = PyUnicode_AsUnicode(environment); - if (wenvironment == NULL) - { - Py_XDECREF(environment); + if (wenvironment == NULL) { + Py_DECREF(environment); return NULL; } } diff --git a/Objects/abstract.c b/Objects/abstract.c index 3585b34ca4d8bd..38de774de3a4db 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -3193,8 +3193,8 @@ _PySequence_BytesToCharpArray(PyObject* self) array[i] = NULL; goto fail; } - data = PyBytes_AsString(item); - if (data == NULL) { + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(item, &data, NULL) < 0) { /* NULL terminate before freeing. */ array[i] = NULL; goto fail; From 9db3ae045dd462a2da2e016c44231de1befd1f87 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 23 Jun 2017 11:23:36 -0700 Subject: [PATCH 0480/1345] [3.6] bpo-30645: don't append to an inner loop path in imp.load_package() (GH-2268) (#2364) Bug didn't manifest itself when importing a module with source as .py files are always the first on the search path. The issue only showed up in bytecode-only packages where the calculated file path would be ``__init__.py/__init__.pyc``. Patch by Alexandru Ardelean. (cherry picked from commit c38e32a10061a7c6d54e7e53ffabf7af7998f045) --- Lib/imp.py | 5 +++-- Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/imp.py b/Lib/imp.py index 781ff23d252161..866464b245b24c 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -203,8 +203,9 @@ def load_package(name, path): extensions = (machinery.SOURCE_SUFFIXES[:] + machinery.BYTECODE_SUFFIXES[:]) for extension in extensions: - path = os.path.join(path, '__init__'+extension) - if os.path.exists(path): + init_path = os.path.join(path, '__init__' + extension) + if os.path.exists(init_path): + path = init_path break else: raise ValueError('{!r} is not a package'.format(path)) diff --git a/Misc/ACKS b/Misc/ACKS index 03f9ea6cc97ae3..c68f9eb9f18867 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -56,6 +56,7 @@ Ankur Ankan Heidi Annexstad Ramchandra Apte Éric Araujo +Alexandru Ardelean Alicia Arlen Jeffrey Armstrong Jason Asbahr diff --git a/Misc/NEWS b/Misc/NEWS index 94a29d0363c6ad..371dcf8ff0d0ac 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -101,6 +101,10 @@ Library variable-argument parameters wrapped with partialmethod. Patch by Dong-hee Na. +- bpo-30645: Fix path calculation in imp.load_package(), fixing it for + cases when a package is only shipped with bytecodes. Patch by + Alexandru Ardelean. + - bpo-29931: Fixed comparison check for ipaddress.ip_interface objects. Patch by Sanjay Sundaresan. From bd570f42110b99bade6e58e3ed2d620f27a92fc3 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 23 Jun 2017 20:19:46 -0400 Subject: [PATCH 0481/1345] [3.6] bpo-24813: IDLE: Add default title to help_about (GH-2366) (#2369) Patch by Cheryl Sabella. (cherry picked from commit 18ede06) --- Lib/idlelib/editor.py | 2 +- Lib/idlelib/help_about.py | 15 ++++++++------- Lib/idlelib/idle_test/test_help_about.py | 23 +++++++++++++++++++++++ Lib/idlelib/macosx.py | 2 +- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index ab4f1a37c168c1..13b4a51c18dff3 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -463,7 +463,7 @@ def rmenu_check_paste(self): def about_dialog(self, event=None): "Handle Help 'About IDLE' event." # Synchronize with macosx.overrideRootMenu.about_dialog. - help_about.AboutDialog(self.top,'About IDLE') + help_about.AboutDialog(self.top) def config_dialog(self, event=None): "Handle Options 'Configure IDLE' event." diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index e65e67c47ee0d7..a2ac3166c34cf3 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -2,7 +2,7 @@ """ import os -from sys import version +from platform import python_version from tkinter import Toplevel, Frame, Label, Button, PhotoImage from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW, E @@ -14,7 +14,7 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title, _htest=False, _utest=False): + def __init__(self, parent, title=None, _htest=False, _utest=False): """Create popup, do not return until tk widget destroyed. parent - parent of this dialog @@ -32,7 +32,7 @@ def __init__(self, parent, title, _htest=False, _utest=False): self.fg = "#ffffff" self.create_widgets() self.resizable(height=False, width=False) - self.title(title) + self.title(title or f'About IDLE {python_version()}') self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.ok) @@ -48,7 +48,6 @@ def __init__(self, parent, title, _htest=False, _utest=False): self.wait_window() def create_widgets(self): - release = version[:version.index(' ')] frame = Frame(self, borderwidth=2, relief=SUNKEN) frame_buttons = Frame(self) frame_buttons.pack(side=BOTTOM, fill=X) @@ -80,7 +79,7 @@ def create_widgets(self): justify=LEFT, fg=self.fg, bg=self.bg) email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) docs = Label(frame_background, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', + python_version()[:3] + '/library/idle.html', justify=LEFT, fg=self.fg, bg=self.bg) docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) @@ -88,7 +87,8 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - pyver = Label(frame_background, text='Python version: ' + release, + pyver = Label(frame_background, + text='Python version: ' + python_version(), fg=self.fg, bg=self.bg) pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) tkver = Label(frame_background, text='Tk version: ' + tk_patchlevel, @@ -113,7 +113,8 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idlever = Label(frame_background, text='IDLE version: ' + release, + idlever = Label(frame_background, + text='IDLE version: ' + python_version(), fg=self.fg, bg=self.bg) idlever.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index f3ca75cf0182bf..be1fadd58f493b 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -10,6 +10,7 @@ from idlelib.help_about import AboutDialog as About from idlelib import textview import os.path +from platform import python_version class LiveDialogTest(unittest.TestCase): """Simulate user clicking buttons other than [Close]. @@ -79,6 +80,28 @@ def test_file_buttons(self): dialog._current_textview.destroy() +class DefaultTitleTest(unittest.TestCase): + "Test default title." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_dialog_title(self): + """Test about dialog title""" + self.assertEqual(self.dialog.title(), f'About IDLE {python_version()}') + + class CloseTest(unittest.TestCase): """Simulate user clicking [Close] button""" diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index d4566cd815a2dd..d85278a0b765ae 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -165,7 +165,7 @@ def about_dialog(event=None): "Handle Help 'About IDLE' event." # Synchronize with editor.EditorWindow.about_dialog. from idlelib import help_about - help_about.AboutDialog(root, 'About IDLE') + help_about.AboutDialog(root) def config_dialog(event=None): "Handle Options 'Configure IDLE' event." From 20f1029584a091959ead30b7f7ef238668745aa0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 23 Jun 2017 22:44:57 -0700 Subject: [PATCH 0482/1345] [3.6] Fix a typo in a comment in coroutines.py (GH-2267) (GH-2370) defiend -> defined (cherry picked from commit cab469245d7635447c5e04fa6ed860b067dfc26b) --- Lib/asyncio/coroutines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 08e94412b33b23..b2adaadfc2cd6d 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -197,7 +197,7 @@ def coroutine(func): """ if _inspect_iscoroutinefunction(func): # In Python 3.5 that's all we need to do for coroutines - # defiend with "async def". + # defined with "async def". # Wrapping in CoroWrapper will happen via # 'sys.set_coroutine_wrapper' function. return func From 0e1f9e8d3ea82262cbb9a403b70a884da5e6a6ac Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 24 Jun 2017 16:28:26 +0300 Subject: [PATCH 0483/1345] [3.6] bpo-30745: Fix compiler warnings introduced in bpo-30730. (GH-2376) (#2378) (cherry picked from commit 0ee32c1) --- Modules/_winapi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 71426bc12dc655..1606f0da772c95 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -844,8 +844,7 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - const wchar_t *wenvironment; - Py_ssize_t wenvironment_size; + wchar_t *wenvironment; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); From 504b95047a8ada06ab630abce55ac2f85566ca37 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 25 Jun 2017 01:12:20 +0900 Subject: [PATCH 0484/1345] [3.6] bpo-30616: Functional API of enum allows to create empty enums. (#2304) (#2324) --- Lib/enum.py | 2 +- Lib/test/test_enum.py | 20 ++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Lib/enum.py b/Lib/enum.py index 056400d04c94a1..73dd613887d22b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -381,7 +381,7 @@ def _create_(cls, class_name, names=None, *, module=None, qualname=None, type=No # special processing needed for names? if isinstance(names, str): names = names.replace(',', ' ').split() - if isinstance(names, (tuple, list)) and isinstance(names[0], str): + if isinstance(names, (tuple, list)) and names and isinstance(names[0], str): original_names, names = names, [] last_values = [] for count, name in enumerate(original_names): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 13a89fc2db6390..eaea8ee3abdfc9 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2291,6 +2291,26 @@ def test_programatic_function_from_dict(self): self.assertIs(type(e), Perm) + def test_programatic_function_from_empty_list(self): + Perm = enum.IntFlag('Perm', []) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 0, Perm) + Thing = enum.Enum('Thing', []) + lst = list(Thing) + self.assertEqual(len(lst), len(Thing)) + self.assertEqual(len(Thing), 0, Thing) + + + def test_programatic_function_from_empty_tuple(self): + Perm = enum.IntFlag('Perm', ()) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 0, Perm) + Thing = enum.Enum('Thing', ()) + self.assertEqual(len(lst), len(Thing)) + self.assertEqual(len(Thing), 0, Thing) + def test_containment(self): Perm = self.Perm R, W, X = Perm diff --git a/Misc/NEWS b/Misc/NEWS index 371dcf8ff0d0ac..3f5798268fa81c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -89,6 +89,9 @@ Core and Builtins Library ------- +- bpo-30616: Functional API of enum allows to create empty enums. + Patched by Dong-hee Na + - bpo-30038: Fix race condition between signal delivery and wakeup file descriptor. Patch by Nathaniel Smith. From 57ee0c8c9e73c3405a0343e1a05cba673ac7f8cb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 24 Jun 2017 17:59:49 -0700 Subject: [PATCH 0485/1345] [3.6] Check the whitespace of pull requests on Travis (GH-2367) (GH-2386) (cherry picked from commit 70cb1875bb5343e31d7268f4b2d231a5fecdf989) --- .travis.yml | 5 +++++ Tools/scripts/patchcheck.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e3620b2379e65..e7b78db48bd829 100644 --- a/.travis.yml +++ b/.travis.yml @@ -88,6 +88,11 @@ before_script: fi script: + # Using the built Python as patchcheck.py is built around the idea of using + # a checkout-build of CPython to know things like what base branch the changes + # should be compared against. + # Only run on Linux as the check only needs to be run once. + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi # `-r -w` implicitly provided through `make buildbottest`. - make buildbottest TESTOPTS="-j4 -uall,-cpu,-tzdata" diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 33a9fead879325..436b277b6ae866 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Check proposed changes for common issues.""" import re import sys import shutil @@ -135,7 +136,7 @@ def report_modified_files(file_paths): return "\n".join(lines) -@status("Fixing whitespace", info=report_modified_files) +@status("Fixing Python file whitespace", info=report_modified_files) def normalize_whitespace(file_paths): """Make sure that the whitespace for .py files have been normalized.""" reindent.makebackup = False # No need to create backups. @@ -212,6 +213,27 @@ def regenerated_pyconfig_h_in(file_paths): else: return "not needed" +def travis(pull_request): + if pull_request == 'false': + print('Not a pull request; skipping') + return + base_branch = get_base_branch() + file_paths = changed_files(base_branch) + python_files = [fn for fn in file_paths if fn.endswith('.py')] + c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] + doc_files = [fn for fn in file_paths if fn.startswith('Doc') and + fn.endswith(('.rst', '.inc'))] + fixed = [] + fixed.extend(normalize_whitespace(python_files)) + fixed.extend(normalize_c_whitespace(c_files)) + fixed.extend(normalize_docs_whitespace(doc_files)) + if not fixed: + print('No whitespace issues found') + else: + print(f'Please fix the {len(fixed)} file(s) with whitespace issues') + print('(on UNIX you can run `make patchcheck` to make the fixes)') + sys.exit(1) + def main(): base_branch = get_base_branch() file_paths = changed_files(base_branch) @@ -246,4 +268,12 @@ def main(): if __name__ == '__main__': - main() + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--travis', + help='Perform pass/fail checks') + args = parser.parse_args() + if args.travis: + travis(args.travis) + else: + main() From 9c2dc0c58a878ac3d1c44dd0048f8e1cfab2790e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 25 Jun 2017 09:49:15 +0300 Subject: [PATCH 0486/1345] [3.6] bpo-30746: Prohibited the '=' character in environment variable names (GH-2382) (#2391) in `os.putenv()` and `os.spawn*()`.. (cherry picked from commit 77703942c5997dff00c48f10df1b29b11645624c) --- Lib/test/test_os.py | 76 ++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_posix.py | 15 +++++++++ Misc/NEWS | 3 ++ Modules/posixmodule.c | 32 +++++++++++++++--- 4 files changed, 122 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 8612ec9edb5356..fdeb9a9647b819 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1553,6 +1553,27 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) + def test_execve_invalid_env(self): + args = [sys.executable, '-c', 'pass'] + + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): @@ -2364,6 +2385,61 @@ def test_spawnve_noargs(self): self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + def _test_invalid_env(self, spawn): + args = [sys.executable, '-c', 'pass'] + + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the enviroment variable value + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + fp.write('import sys, os\n' + 'if os.getenv("FRUIT") != "orange=lemon":\n' + ' raise AssertionError') + args = [sys.executable, filename] + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + self.assertEqual(exitcode, 0) + + @requires_os_func('spawnve') + def test_spawnve_invalid_env(self): + self._test_invalid_env(os.spawnve) + + @requires_os_func('spawnvpe') + def test_spawnvpe_invalid_env(self): + self._test_invalid_env(os.spawnvpe) + + # The introduction of this TestCase caused at least two different errors on # *nix buildbots. Temporarily skip this to let the buildbots move along. @unittest.skip("Skip due to platform/environment differences on *NIX buildbots") diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 029d0815e94589..4ca623050b8494 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -751,6 +751,21 @@ def test_environ(self): self.assertEqual(type(k), item_type) self.assertEqual(type(v), item_type) + @unittest.skipUnless(hasattr(os, "putenv"), "requires os.putenv()") + def test_putenv(self): + with self.assertRaises(ValueError): + os.putenv('FRUIT\0VEGETABLE', 'cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') + with self.assertRaises(ValueError): + os.putenv('FRUIT', 'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv('FRUIT=ORANGE', 'lemon') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT=ORANGE', b'lemon') + @unittest.skipUnless(hasattr(posix, 'getcwd'), 'test needs posix.getcwd()') def test_getcwd_long_pathnames(self): dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef' diff --git a/Misc/NEWS b/Misc/NEWS index 3f5798268fa81c..a3a32ae6c60810 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- bpo-30746: Prohibited the '=' character in environment variable names in + ``os.putenv()`` and ``os.spawn*()``. + - [Security] bpo-30730: Prevent environment variables injection in subprocess on Windows. Prevent passing other environment variables and command arguments. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 11aaeef02bc50c..033789099c3a84 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4817,6 +4817,14 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) Py_DECREF(key2); goto error; } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key2) == 0 || + PyUnicode_FindChar(key2, '=', 1, PyUnicode_GET_LENGTH(key2), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } keyval = PyUnicode_FromFormat("%U=%U", key2, val2); #else if (!PyUnicode_FSConverter(key, &key2)) @@ -4825,6 +4833,12 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) Py_DECREF(key2); goto error; } + if (PyBytes_GET_SIZE(key2) == 0 || + strchr(PyBytes_AS_STRING(key2) + 1, '=') != NULL) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } keyval = PyBytes_FromFormat("%s=%s", PyBytes_AS_STRING(key2), PyBytes_AS_STRING(val2)); #endif @@ -8864,9 +8878,16 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) { const wchar_t *env; + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(name) == 0 || + PyUnicode_FindChar(name, '=', 1, PyUnicode_GET_LENGTH(name), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + return NULL; + } PyObject *unicode = PyUnicode_FromFormat("%U=%U", name, value); if (unicode == NULL) { - PyErr_NoMemory(); return NULL; } if (_MAX_ENV < PyUnicode_GET_LENGTH(unicode)) { @@ -8908,12 +8929,15 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) { PyObject *bytes = NULL; char *env; - const char *name_string = PyBytes_AsString(name); - const char *value_string = PyBytes_AsString(value); + const char *name_string = PyBytes_AS_STRING(name); + const char *value_string = PyBytes_AS_STRING(value); + if (strchr(name_string, '=') != NULL) { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + return NULL; + } bytes = PyBytes_FromFormat("%s=%s", name_string, value_string); if (bytes == NULL) { - PyErr_NoMemory(); return NULL; } From 64a0c266e78c6b5363668012fa7fb614f6018930 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Mon, 26 Jun 2017 01:21:02 -0400 Subject: [PATCH 0487/1345] [3.6] bpo-6739: IDLE: Check for valid keybinding in config_keys (GH-2377) (#2397) Verify user-entered key sequences by trying to bind them with tk. Add tests for all 3 validation functions. Original patch by G Polo. Tests added by Cheryl Sabella. (cherry picked from commit 8c78aa7) --- Lib/idlelib/config_key.py | 62 ++++++++++----- Lib/idlelib/idle_test/test_config_key.py | 79 +++++++++++++++++-- .../2017-06-26-00-28-59.bpo-6739.x5MfhB.rst | 3 + 3 files changed, 118 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-26-00-28-59.bpo-6739.x5MfhB.rst diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 479d6ad313e35f..8d575ec43ca833 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -3,11 +3,16 @@ """ from tkinter import * from tkinter.ttk import Scrollbar -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox import string import sys + class GetKeysDialog(Toplevel): + + # Dialog title for invalid key sequence + keyerror_title = 'Key Sequence Error' + def __init__(self, parent, title, action, currentKeySequences, _htest=False, _utest=False): """ @@ -54,6 +59,10 @@ def __init__(self, parent, title, action, currentKeySequences, self.deiconify() #geometry set, unhide self.wait_window() + def showerror(self, *args, **kwargs): + # Make testing easier. Replace in #30751. + messagebox.showerror(*args, **kwargs) + def CreateWidgets(self): frameMain = Frame(self,borderwidth=2,relief=SUNKEN) frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) @@ -219,53 +228,70 @@ def TranslateKey(self, key, modifiers): return key def OK(self, event=None): - if self.advanced or self.KeysOK(): # doesn't check advanced string yet - self.result=self.keyString.get() - self.destroy() + keys = self.keyString.get().strip() + if not keys: + self.showerror(title=self.keyerror_title, parent=self, + message="No key specified.") + return + if (self.advanced or self.KeysOK(keys)) and self.bind_ok(keys): + self.result = keys + self.destroy() def Cancel(self, event=None): self.result='' self.destroy() - def KeysOK(self): + def KeysOK(self, keys): '''Validity check on user's 'basic' keybinding selection. Doesn't check the string produced by the advanced dialog because 'modifiers' isn't set. ''' - keys = self.keyString.get() - keys.strip() finalKey = self.listKeysFinal.get(ANCHOR) modifiers = self.GetModifiers() # create a key sequence list for overlap check: keySequence = keys.split() keysOK = False - title = 'Key Sequence Error' - if not keys: - tkMessageBox.showerror(title=title, parent=self, - message='No keys specified.') - elif not keys.endswith('>'): - tkMessageBox.showerror(title=title, parent=self, - message='Missing the final Key') + title = self.keyerror_title + if not keys.endswith('>'): + self.showerror(title, parent=self, + message='Missing the final Key') elif (not modifiers and finalKey not in self.functionKeys + self.moveKeys): - tkMessageBox.showerror(title=title, parent=self, - message='No modifier key(s) specified.') + self.showerror(title=title, parent=self, + message='No modifier key(s) specified.') elif (modifiers == ['Shift']) \ and (finalKey not in self.functionKeys + self.moveKeys + ('Tab', 'Space')): msg = 'The shift modifier by itself may not be used with'\ ' this key symbol.' - tkMessageBox.showerror(title=title, parent=self, message=msg) + self.showerror(title=title, parent=self, message=msg) elif keySequence in self.currentKeySequences: msg = 'This key combination is already in use.' - tkMessageBox.showerror(title=title, parent=self, message=msg) + self.showerror(title=title, parent=self, message=msg) else: keysOK = True return keysOK + def bind_ok(self, keys): + "Return True if Tcl accepts the new keys else show message." + + try: + binding = self.bind(keys, lambda: None) + except TclError as err: + self.showerror( + title=self.keyerror_title, parent=self, + message=(f'The entered key sequence is not accepted.\n\n' + f'Error: {err}')) + return False + else: + self.unbind(keys, binding) + return True + if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(GetKeysDialog) diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index 8a24c9632b7224..a4227ed0552b7f 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -4,29 +4,92 @@ ''' from idlelib import config_key from test.support import requires -requires('gui') +import sys import unittest from tkinter import Tk +from idlelib.idle_test.mock_idle import Func +from idlelib.idle_test.mock_tk import Var, Mbox_func -class GetKeysTest(unittest.TestCase): +class ValidationTest(unittest.TestCase): + "Test validation methods: OK, KeysOK, bind_ok." + + class Validator(config_key.GetKeysDialog): + def __init__(self, *args, **kwargs): + config_key.GetKeysDialog.__init__(self, *args, **kwargs) + class listKeysFinal: + get = Func() + self.listKeysFinal = listKeysFinal + GetModifiers = Func() + showerror = Mbox_func() @classmethod def setUpClass(cls): + requires('gui') cls.root = Tk() cls.root.withdraw() + cls.dialog = cls.Validator( + cls.root, 'Title', '<>', [['']], _utest=True) @classmethod def tearDownClass(cls): - cls.root.update() # Stop "can't run event command" warning. + cls.dialog.Cancel() + cls.root.update_idletasks() cls.root.destroy() - del cls.root + del cls.dialog, cls.root + + def setUp(self): + self.dialog.showerror.message = '' + # A test that needs a particular final key value should set it. + # A test that sets a non-blank modifier list should reset it to []. + + def test_ok_empty(self): + self.dialog.keyString.set(' ') + self.dialog.OK() + self.assertEqual(self.dialog.result, '') + self.assertEqual(self.dialog.showerror.message, 'No key specified.') + + def test_ok_good(self): + self.dialog.keyString.set('') + self.dialog.listKeysFinal.get.result = 'F11' + self.dialog.OK() + self.assertEqual(self.dialog.result, '') + self.assertEqual(self.dialog.showerror.message, '') + + def test_keys_no_ending(self): + self.assertFalse(self.dialog.KeysOK('')) + self.assertIn('No modifier', self.dialog.showerror.message) + + def test_keys_no_modifier_ok(self): + self.dialog.listKeysFinal.get.result = 'F11' + self.assertTrue(self.dialog.KeysOK('')) + self.assertEqual(self.dialog.showerror.message, '') + + def test_keys_shift_bad(self): + self.dialog.listKeysFinal.get.result = 'a' + self.dialog.GetModifiers.result = ['Shift'] + self.assertFalse(self.dialog.KeysOK('')) + self.assertIn('shift modifier', self.dialog.showerror.message) + self.dialog.GetModifiers.result = [] + + def test_keys_dup(self): + self.dialog.listKeysFinal.get.result = 'F12' + self.dialog.GetModifiers.result = [] + self.assertFalse(self.dialog.KeysOK('')) + self.assertIn('already in use', self.dialog.showerror.message) + def test_bind_ok(self): + self.assertTrue(self.dialog.bind_ok('')) + self.assertEqual(self.dialog.showerror.message, '') - def test_init(self): - dia = config_key.GetKeysDialog( - self.root, 'test', '<>', [''], _utest=True) - dia.Cancel() + def test_bind_not_ok(self): + self.assertFalse(self.dialog.bind_ok('')) + self.assertIn('not accepted', self.dialog.showerror.message) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/IDLE/2017-06-26-00-28-59.bpo-6739.x5MfhB.rst b/Misc/NEWS.d/next/IDLE/2017-06-26-00-28-59.bpo-6739.x5MfhB.rst new file mode 100644 index 00000000000000..fee904d9e78c50 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-26-00-28-59.bpo-6739.x5MfhB.rst @@ -0,0 +1,3 @@ +IDLE: Verify user-entered key sequences by trying to bind them with tk. Add +tests for all 3 validation functions. Original patch by G Polo. Tests added +by Cheryl Sabella. From 55ab604e3fdb91baabe98d1c407b5910eeb31cef Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 26 Jun 2017 21:57:28 +0200 Subject: [PATCH 0488/1345] [3.6] bpo-30765: Avoid blocking when PyThread_acquire_lock() is asked not to (GH-2403) (#2418) * bpo-30765: Avoid blocking when PyThread_acquire_lock() is asked not to lock This is especially important if PyThread_acquire_lock() is called reentrantly (for example from a signal handler). * Update 2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst * Avoid core logic when taking the mutex failed (cherry picked from commit f84ac420c2af98339678744953869cad3c253281) --- .../2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst | 2 + Python/thread_pthread.h | 95 ++++++++++--------- 2 files changed, 52 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst b/Misc/NEWS.d/next/Core and Builtins/2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst new file mode 100644 index 00000000000000..08d76cb965e369 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-06-26-14-29-50.bpo-30765.Q5iBmf.rst @@ -0,0 +1,2 @@ +Avoid blocking in pthread_mutex_lock() when PyThread_acquire_lock() is asked +not to block. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index ba7393f03de6ae..3607341eaa587b 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -466,61 +466,66 @@ PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, int intr_flag) { - PyLockStatus success; + PyLockStatus success = PY_LOCK_FAILURE; pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", lock, microseconds, intr_flag)); - status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); - - if (thelock->locked == 0) { - success = PY_LOCK_ACQUIRED; - } else if (microseconds == 0) { - success = PY_LOCK_FAILURE; - } else { - struct timespec ts; - if (microseconds > 0) - MICROSECONDS_TO_TIMESPEC(microseconds, ts); - /* continue trying until we get the lock */ - - /* mut must be locked by me -- part of the condition - * protocol */ - success = PY_LOCK_FAILURE; - while (success == PY_LOCK_FAILURE) { - if (microseconds > 0) { - status = pthread_cond_timedwait( - &thelock->lock_released, - &thelock->mut, &ts); - if (status == ETIMEDOUT) + if (microseconds == 0) { + status = pthread_mutex_trylock( &thelock->mut ); + if (status != EBUSY) + CHECK_STATUS_PTHREAD("pthread_mutex_trylock[1]"); + } + else { + status = pthread_mutex_lock( &thelock->mut ); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); + } + if (status == 0) { + if (thelock->locked == 0) { + success = PY_LOCK_ACQUIRED; + } + else if (microseconds != 0) { + struct timespec ts; + if (microseconds > 0) + MICROSECONDS_TO_TIMESPEC(microseconds, ts); + /* continue trying until we get the lock */ + + /* mut must be locked by me -- part of the condition + * protocol */ + while (success == PY_LOCK_FAILURE) { + if (microseconds > 0) { + status = pthread_cond_timedwait( + &thelock->lock_released, + &thelock->mut, &ts); + if (status == ETIMEDOUT) + break; + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); + } + else { + status = pthread_cond_wait( + &thelock->lock_released, + &thelock->mut); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); + } + + if (intr_flag && status == 0 && thelock->locked) { + /* We were woken up, but didn't get the lock. We probably received + * a signal. Return PY_LOCK_INTR to allow the caller to handle + * it and retry. */ + success = PY_LOCK_INTR; break; - CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); - } - else { - status = pthread_cond_wait( - &thelock->lock_released, - &thelock->mut); - CHECK_STATUS_PTHREAD("pthread_cond_wait"); - } - - if (intr_flag && status == 0 && thelock->locked) { - /* We were woken up, but didn't get the lock. We probably received - * a signal. Return PY_LOCK_INTR to allow the caller to handle - * it and retry. */ - success = PY_LOCK_INTR; - break; - } else if (status == 0 && !thelock->locked) { - success = PY_LOCK_ACQUIRED; - } else { - success = PY_LOCK_FAILURE; + } + else if (status == 0 && !thelock->locked) { + success = PY_LOCK_ACQUIRED; + } } } + if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; + status = pthread_mutex_unlock( &thelock->mut ); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); } - if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; - status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", From 9ad50d94599aed0c37beaf78948ec271c8aa3881 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 27 Jun 2017 00:00:33 +0200 Subject: [PATCH 0489/1345] bpo-30764: test_subprocess uses SuppressCrashReport (#2405) (#2410) bpo-30764, bpo-29335: test_child_terminated_in_stopped_state() of test_subprocess now uses support.SuppressCrashReport() to prevent the creation of a core dump on FreeBSD. (cherry picked from commit cdee3f14f7f4c995e7eedb0bf6a67e260c739f7d) --- Lib/test/test_subprocess.py | 51 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 9a9e3ca5791256..4072591b83e9cc 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -21,6 +21,8 @@ import ctypes except ImportError: ctypes = None +else: + import ctypes.util try: import threading @@ -2565,43 +2567,40 @@ def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): proc.communicate(timeout=999) mock_proc_stdin.close.assert_called_once_with() - _libc_file_extensions = { - 'Linux': 'so.6', - 'Darwin': 'dylib', - } - @unittest.skipIf(not ctypes, 'ctypes module required.') - @unittest.skipIf(platform.uname()[0] not in _libc_file_extensions, - 'Test requires a libc this code can load with ctypes.') - @unittest.skipIf(not sys.executable, 'Test requires sys.executable.') + @unittest.skipIf(not ctypes, 'ctypes module required') + @unittest.skipIf(not sys.executable, 'Test requires sys.executable') def test_child_terminated_in_stopped_state(self): """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" PTRACE_TRACEME = 0 # From glibc and MacOS (PT_TRACE_ME). - libc_name = 'libc.' + self._libc_file_extensions[platform.uname()[0]] + libc_name = ctypes.util.find_library('c') libc = ctypes.CDLL(libc_name) if not hasattr(libc, 'ptrace'): - raise unittest.SkipTest('ptrace() required.') - test_ptrace = subprocess.Popen( - [sys.executable, '-c', """if True: - import ctypes - libc = ctypes.CDLL({libc_name!r}) - libc.ptrace({PTRACE_TRACEME}, 0, 0) - """.format(libc_name=libc_name, PTRACE_TRACEME=PTRACE_TRACEME) - ]) - if test_ptrace.wait() != 0: - raise unittest.SkipTest('ptrace() failed - unable to test.') - child = subprocess.Popen( - [sys.executable, '-c', """if True: + raise unittest.SkipTest('ptrace() required') + + code = textwrap.dedent(f""" import ctypes + import faulthandler + from test.support import SuppressCrashReport + libc = ctypes.CDLL({libc_name!r}) libc.ptrace({PTRACE_TRACEME}, 0, 0) - libc.printf(ctypes.c_char_p(0xdeadbeef)) # Crash the process. - """.format(libc_name=libc_name, PTRACE_TRACEME=PTRACE_TRACEME) - ]) + """) + + child = subprocess.Popen([sys.executable, '-c', code]) + if child.wait() != 0: + raise unittest.SkipTest('ptrace() failed - unable to test') + + code += textwrap.dedent(f""" + with SuppressCrashReport(): + # Crash the process + faulthandler._sigsegv() + """) + child = subprocess.Popen([sys.executable, '-c', code]) try: returncode = child.wait() - except Exception as e: + except: child.kill() # Clean up the hung stopped process. - raise e + raise self.assertNotEqual(0, returncode) self.assertLess(returncode, 0) # signal death, likely SIGSEGV. From 938e73809d10f6073c85ecd330c88a85c6095530 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Mon, 26 Jun 2017 20:48:39 -0400 Subject: [PATCH 0490/1345] [3.6] bpo-30728: IDLE: Refactor configdialog to PEP8 names (GH-2307) (#2421) Also, change '*' in the tkinter import to an explicit list of names. Patch by Cheryl Sabella. (cherry picked from commit bac7d33) --- Lib/idlelib/configdialog.py | 1625 +++++++++-------- Lib/idlelib/idle_test/test_configdialog.py | 40 +- .../2017-06-26-15-47-13.bpo-30728.qH4TGL.rst | 4 + 3 files changed, 841 insertions(+), 828 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-26-15-47-13.bpo-30728.qH4TGL.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 8184582a3e2ff3..96682a92214769 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -9,7 +9,12 @@ Refer to comments in EditorWindow autoindent code for details. """ -from tkinter import * +from tkinter import (Toplevel, Frame, LabelFrame, Listbox, Label, Button, + Entry, Text, Scale, Radiobutton, Checkbutton, Canvas, + StringVar, BooleanVar, IntVar, TRUE, FALSE, + TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, + NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, + HORIZONTAL, VERTICAL, ANCHOR, END) from tkinter.ttk import Scrollbar import tkinter.colorchooser as tkColorChooser import tkinter.font as tkFont @@ -34,17 +39,17 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.parent = parent if _htest: parent.instance_dict = {} - self.wm_withdraw() + self.withdraw() self.configure(borderwidth=5) self.title(title or 'IDLE Preferences') - self.geometry( - "+%d+%d" % (parent.winfo_rootx() + 20, - parent.winfo_rooty() + (30 if not _htest else 150))) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (30 if not _htest else 150) + self.geometry(f'+{x}+{y}') #Theme Elements. Each theme element key is its display name. #The first value of the tuple is the sample area tag name. #The second value is the display name list sort index. - self.themeElements={ + self.theme_elements={ 'Normal Text': ('normal', '00'), 'Python Keywords': ('keyword', '01'), 'Python Definitions': ('definition', '02'), @@ -60,152 +65,152 @@ def __init__(self, parent, title='', _htest=False, _utest=False): 'Shell Stdout Text': ('stdout', '12'), 'Shell Stderr Text': ('stderr', '13'), } - self.ResetChangedItems() #load initial values in changed items dict - self.CreateWidgets() + self.reset_changed_items() #load initial values in changed items dict + self.create_widgets() self.resizable(height=FALSE, width=FALSE) self.transient(parent) self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Cancel) - self.tabPages.focus_set() + self.protocol("WM_DELETE_WINDOW", self.cancel) + self.tab_pages.focus_set() #key bindings for this dialog #self.bind('', self.Cancel) #dismiss dialog, no save #self.bind('', self.Apply) #apply changes, save #self.bind('', self.Help) #context help - self.LoadConfigs() - self.AttachVarCallbacks() #avoid callbacks during LoadConfigs + self.load_configs() + self.attach_var_callbacks() #avoid callbacks during load_configs if not _utest: self.wm_deiconify() self.wait_window() - def CreateWidgets(self): - self.tabPages = TabbedPageSet(self, + def create_widgets(self): + self.tab_pages = TabbedPageSet(self, page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', 'Extensions']) - self.tabPages.pack(side=TOP, expand=TRUE, fill=BOTH) - self.CreatePageFontTab() - self.CreatePageHighlight() - self.CreatePageKeys() - self.CreatePageGeneral() - self.CreatePageExtensions() + self.tab_pages.pack(side=TOP, expand=TRUE, fill=BOTH) + self.create_page_font_tab() + self.create_page_highlight() + self.create_page_keys() + self.create_page_general() + self.create_page_extensions() self.create_action_buttons().pack(side=BOTTOM) def create_action_buttons(self): if macosx.isAquaTk(): # Changing the default padding on OSX results in unreadable # text in the buttons - paddingArgs = {} + padding_args = {} else: - paddingArgs = {'padx':6, 'pady':3} + padding_args = {'padx':6, 'pady':3} outer = Frame(self, pady=2) buttons = Frame(outer, pady=2) for txt, cmd in ( - ('Ok', self.Ok), - ('Apply', self.Apply), - ('Cancel', self.Cancel), - ('Help', self.Help)): + ('Ok', self.ok), + ('Apply', self.apply), + ('Cancel', self.cancel), + ('Help', self.help)): Button(buttons, text=txt, command=cmd, takefocus=FALSE, - **paddingArgs).pack(side=LEFT, padx=5) + **padding_args).pack(side=LEFT, padx=5) # add space above buttons Frame(outer, height=2, borderwidth=0).pack(side=TOP) buttons.pack(side=BOTTOM) return outer - def CreatePageFontTab(self): + def create_page_font_tab(self): parent = self.parent - self.fontSize = StringVar(parent) - self.fontBold = BooleanVar(parent) - self.fontName = StringVar(parent) - self.spaceNum = IntVar(parent) - self.editFont = tkFont.Font(parent, ('courier', 10, 'normal')) + self.font_size = StringVar(parent) + self.font_bold = BooleanVar(parent) + self.font_name = StringVar(parent) + self.space_num = IntVar(parent) + self.edit_font = tkFont.Font(parent, ('courier', 10, 'normal')) ##widget creation #body frame - frame = self.tabPages.pages['Fonts/Tabs'].frame + frame = self.tab_pages.pages['Fonts/Tabs'].frame #body section frames - frameFont = LabelFrame( + frame_font = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') - frameIndent = LabelFrame( + frame_indent = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') - #frameFont - frameFontName = Frame(frameFont) - frameFontParam = Frame(frameFont) - labelFontNameTitle = Label( - frameFontName, justify=LEFT, text='Font Face :') - self.listFontName = Listbox( - frameFontName, height=5, takefocus=FALSE, exportselection=FALSE) - self.listFontName.bind( - '', self.OnListFontButtonRelease) - scrollFont = Scrollbar(frameFontName) - scrollFont.config(command=self.listFontName.yview) - self.listFontName.config(yscrollcommand=scrollFont.set) - labelFontSizeTitle = Label(frameFontParam, text='Size :') - self.optMenuFontSize = DynOptionMenu( - frameFontParam, self.fontSize, None, command=self.SetFontSample) - checkFontBold = Checkbutton( - frameFontParam, variable=self.fontBold, onvalue=1, - offvalue=0, text='Bold', command=self.SetFontSample) - frameFontSample = Frame(frameFont, relief=SOLID, borderwidth=1) - self.labelFontSample = Label( - frameFontSample, justify=LEFT, font=self.editFont, + #frame_font + frame_font_name = Frame(frame_font) + frame_font_param = Frame(frame_font) + font_name_title = Label( + frame_font_name, justify=LEFT, text='Font Face :') + self.list_fonts = Listbox( + frame_font_name, height=5, takefocus=FALSE, exportselection=FALSE) + self.list_fonts.bind( + '', self.on_list_fonts_button_release) + scroll_font = Scrollbar(frame_font_name) + scroll_font.config(command=self.list_fonts.yview) + self.list_fonts.config(yscrollcommand=scroll_font.set) + font_size_title = Label(frame_font_param, text='Size :') + self.opt_menu_font_size = DynOptionMenu( + frame_font_param, self.font_size, None, command=self.set_font_sample) + check_font_bold = Checkbutton( + frame_font_param, variable=self.font_bold, onvalue=1, + offvalue=0, text='Bold', command=self.set_font_sample) + frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) + self.font_sample = Label( + frame_font_sample, justify=LEFT, font=self.edit_font, text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') - #frameIndent - frameIndentSize = Frame(frameIndent) - labelSpaceNumTitle = Label( - frameIndentSize, justify=LEFT, + #frame_indent + frame_indent_size = Frame(frame_indent) + indent_size_title = Label( + frame_indent_size, justify=LEFT, text='Python Standard: 4 Spaces!') - self.scaleSpaceNum = Scale( - frameIndentSize, variable=self.spaceNum, + self.scale_indent_size = Scale( + frame_indent_size, variable=self.space_num, orient='horizontal', tickinterval=2, from_=2, to=16) #widget packing #body - frameFont.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameIndent.pack(side=LEFT, padx=5, pady=5, fill=Y) - #frameFont - frameFontName.pack(side=TOP, padx=5, pady=5, fill=X) - frameFontParam.pack(side=TOP, padx=5, pady=5, fill=X) - labelFontNameTitle.pack(side=TOP, anchor=W) - self.listFontName.pack(side=LEFT, expand=TRUE, fill=X) - scrollFont.pack(side=LEFT, fill=Y) - labelFontSizeTitle.pack(side=LEFT, anchor=W) - self.optMenuFontSize.pack(side=LEFT, anchor=W) - checkFontBold.pack(side=LEFT, anchor=W, padx=20) - frameFontSample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - self.labelFontSample.pack(expand=TRUE, fill=BOTH) - #frameIndent - frameIndentSize.pack(side=TOP, fill=X) - labelSpaceNumTitle.pack(side=TOP, anchor=W, padx=5) - self.scaleSpaceNum.pack(side=TOP, padx=5, fill=X) + frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) + #frame_font + frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) + frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) + font_name_title.pack(side=TOP, anchor=W) + self.list_fonts.pack(side=LEFT, expand=TRUE, fill=X) + scroll_font.pack(side=LEFT, fill=Y) + font_size_title.pack(side=LEFT, anchor=W) + self.opt_menu_font_size.pack(side=LEFT, anchor=W) + check_font_bold.pack(side=LEFT, anchor=W, padx=20) + frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.font_sample.pack(expand=TRUE, fill=BOTH) + #frame_indent + frame_indent_size.pack(side=TOP, fill=X) + indent_size_title.pack(side=TOP, anchor=W, padx=5) + self.scale_indent_size.pack(side=TOP, padx=5, fill=X) return frame - def CreatePageHighlight(self): + def create_page_highlight(self): parent = self.parent - self.builtinTheme = StringVar(parent) - self.customTheme = StringVar(parent) - self.fgHilite = BooleanVar(parent) + self.builtin_theme = StringVar(parent) + self.custom_theme = StringVar(parent) + self.fg_bg_toggle = BooleanVar(parent) self.colour = StringVar(parent) - self.fontName = StringVar(parent) - self.themeIsBuiltin = BooleanVar(parent) - self.highlightTarget = StringVar(parent) + self.font_name = StringVar(parent) + self.is_builtin_theme = BooleanVar(parent) + self.highlight_target = StringVar(parent) ##widget creation #body frame - frame = self.tabPages.pages['Highlighting'].frame + frame = self.tab_pages.pages['Highlighting'].frame #body section frames - frameCustom = LabelFrame(frame, borderwidth=2, relief=GROOVE, + frame_custom = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Custom Highlighting ') - frameTheme = LabelFrame(frame, borderwidth=2, relief=GROOVE, + frame_theme = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Highlighting Theme ') - #frameCustom - self.textHighlightSample=Text( - frameCustom, relief=SOLID, borderwidth=1, + #frame_custom + self.text_highlight_sample=Text( + frame_custom, relief=SOLID, borderwidth=1, font=('courier', 12, ''), cursor='hand2', width=21, height=11, takefocus=FALSE, highlightthickness=0, wrap=NONE) - text=self.textHighlightSample + text=self.text_highlight_sample text.bind('', lambda e: 'break') text.bind('', lambda e: 'break') - textAndTags=( + text_and_tags=( ('#you can click here', 'comment'), ('\n', 'normal'), ('#to choose items', 'comment'), ('\n', 'normal'), ('def', 'keyword'), (' ', 'normal'), @@ -222,1004 +227,1004 @@ def CreatePageHighlight(self): ('shell', 'console'), (' ', 'normal'), ('stdout', 'stdout'), (' ', 'normal'), ('stderr', 'stderr'), ('\n', 'normal')) - for txTa in textAndTags: - text.insert(END, txTa[0], txTa[1]) - for element in self.themeElements: + for texttag in text_and_tags: + text.insert(END, texttag[0], texttag[1]) + for element in self.theme_elements: def tem(event, elem=element): - event.widget.winfo_toplevel().highlightTarget.set(elem) + event.widget.winfo_toplevel().highlight_target.set(elem) text.tag_bind( - self.themeElements[element][0], '', tem) + self.theme_elements[element][0], '', tem) text.config(state=DISABLED) - self.frameColourSet = Frame(frameCustom, relief=SOLID, borderwidth=1) - frameFgBg = Frame(frameCustom) - buttonSetColour = Button( - self.frameColourSet, text='Choose Colour for :', - command=self.GetColour, highlightthickness=0) - self.optMenuHighlightTarget = DynOptionMenu( - self.frameColourSet, self.highlightTarget, None, - highlightthickness=0) #, command=self.SetHighlightTargetBinding - self.radioFg = Radiobutton( - frameFgBg, variable=self.fgHilite, value=1, - text='Foreground', command=self.SetColourSampleBinding) - self.radioBg=Radiobutton( - frameFgBg, variable=self.fgHilite, value=0, - text='Background', command=self.SetColourSampleBinding) - self.fgHilite.set(1) - buttonSaveCustomTheme = Button( - frameCustom, text='Save as New Custom Theme', - command=self.SaveAsNewTheme) - #frameTheme - labelTypeTitle = Label(frameTheme, text='Select : ') - self.radioThemeBuiltin = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=1, - command=self.SetThemeType, text='a Built-in Theme') - self.radioThemeCustom = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=0, - command=self.SetThemeType, text='a Custom Theme') - self.optMenuThemeBuiltin = DynOptionMenu( - frameTheme, self.builtinTheme, None, command=None) - self.optMenuThemeCustom=DynOptionMenu( - frameTheme, self.customTheme, None, command=None) - self.buttonDeleteCustomTheme=Button( - frameTheme, text='Delete Custom Theme', - command=self.DeleteCustomTheme) - self.new_custom_theme = Label(frameTheme, bd=2) + self.frame_colour_set = Frame(frame_custom, relief=SOLID, borderwidth=1) + frame_fg_bg_toggle = Frame(frame_custom) + button_set_colour = Button( + self.frame_colour_set, text='Choose Colour for :', + command=self.get_colour, highlightthickness=0) + self.opt_menu_highlight_target = DynOptionMenu( + self.frame_colour_set, self.highlight_target, None, + highlightthickness=0) #, command=self.set_highlight_targetBinding + self.radio_fg = Radiobutton( + frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=1, + text='Foreground', command=self.set_colour_sample_binding) + self.radio_bg=Radiobutton( + frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=0, + text='Background', command=self.set_colour_sample_binding) + self.fg_bg_toggle.set(1) + button_save_custom_theme = Button( + frame_custom, text='Save as New Custom Theme', + command=self.save_as_new_theme) + #frame_theme + theme_type_title = Label(frame_theme, text='Select : ') + self.radio_theme_builtin = Radiobutton( + frame_theme, variable=self.is_builtin_theme, value=1, + command=self.set_theme_type, text='a Built-in Theme') + self.radio_theme_custom = Radiobutton( + frame_theme, variable=self.is_builtin_theme, value=0, + command=self.set_theme_type, text='a Custom Theme') + self.opt_menu_theme_builtin = DynOptionMenu( + frame_theme, self.builtin_theme, None, command=None) + self.opt_menu_theme_custom=DynOptionMenu( + frame_theme, self.custom_theme, None, command=None) + self.button_delete_custom_theme=Button( + frame_theme, text='Delete Custom Theme', + command=self.delete_custom_theme) + self.new_custom_theme = Label(frame_theme, bd=2) ##widget packing #body - frameCustom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameTheme.pack(side=LEFT, padx=5, pady=5, fill=Y) - #frameCustom - self.frameColourSet.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) - frameFgBg.pack(side=TOP, padx=5, pady=0) - self.textHighlightSample.pack( + frame_custom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_theme.pack(side=LEFT, padx=5, pady=5, fill=Y) + #frame_custom + self.frame_colour_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) + frame_fg_bg_toggle.pack(side=TOP, padx=5, pady=0) + self.text_highlight_sample.pack( side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - buttonSetColour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) - self.optMenuHighlightTarget.pack( + button_set_colour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) + self.opt_menu_highlight_target.pack( side=TOP, expand=TRUE, fill=X, padx=8, pady=3) - self.radioFg.pack(side=LEFT, anchor=E) - self.radioBg.pack(side=RIGHT, anchor=W) - buttonSaveCustomTheme.pack(side=BOTTOM, fill=X, padx=5, pady=5) - #frameTheme - labelTypeTitle.pack(side=TOP, anchor=W, padx=5, pady=5) - self.radioThemeBuiltin.pack(side=TOP, anchor=W, padx=5) - self.radioThemeCustom.pack(side=TOP, anchor=W, padx=5, pady=2) - self.optMenuThemeBuiltin.pack(side=TOP, fill=X, padx=5, pady=5) - self.optMenuThemeCustom.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) - self.buttonDeleteCustomTheme.pack(side=TOP, fill=X, padx=5, pady=5) + self.radio_fg.pack(side=LEFT, anchor=E) + self.radio_bg.pack(side=RIGHT, anchor=W) + button_save_custom_theme.pack(side=BOTTOM, fill=X, padx=5, pady=5) + #frame_theme + theme_type_title.pack(side=TOP, anchor=W, padx=5, pady=5) + self.radio_theme_builtin.pack(side=TOP, anchor=W, padx=5) + self.radio_theme_custom.pack(side=TOP, anchor=W, padx=5, pady=2) + self.opt_menu_theme_builtin.pack(side=TOP, fill=X, padx=5, pady=5) + self.opt_menu_theme_custom.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) + self.button_delete_custom_theme.pack(side=TOP, fill=X, padx=5, pady=5) self.new_custom_theme.pack(side=TOP, fill=X, pady=5) return frame - def CreatePageKeys(self): + def create_page_keys(self): parent = self.parent - self.bindingTarget = StringVar(parent) - self.builtinKeys = StringVar(parent) - self.customKeys = StringVar(parent) - self.keysAreBuiltin = BooleanVar(parent) - self.keyBinding = StringVar(parent) + self.binding_target = StringVar(parent) + self.builtin_keys = StringVar(parent) + self.custom_keys = StringVar(parent) + self.are_keys_builtin = BooleanVar(parent) + self.keybinding = StringVar(parent) ##widget creation #body frame - frame = self.tabPages.pages['Keys'].frame + frame = self.tab_pages.pages['Keys'].frame #body section frames - frameCustom = LabelFrame( + frame_custom = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Custom Key Bindings ') - frameKeySets = LabelFrame( + frame_key_sets = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Key Set ') - #frameCustom - frameTarget = Frame(frameCustom) - labelTargetTitle = Label(frameTarget, text='Action - Key(s)') - scrollTargetY = Scrollbar(frameTarget) - scrollTargetX = Scrollbar(frameTarget, orient=HORIZONTAL) - self.listBindings = Listbox( - frameTarget, takefocus=FALSE, exportselection=FALSE) - self.listBindings.bind('', self.KeyBindingSelected) - scrollTargetY.config(command=self.listBindings.yview) - scrollTargetX.config(command=self.listBindings.xview) - self.listBindings.config(yscrollcommand=scrollTargetY.set) - self.listBindings.config(xscrollcommand=scrollTargetX.set) - self.buttonNewKeys = Button( - frameCustom, text='Get New Keys for Selection', - command=self.GetNewKeys, state=DISABLED) - #frameKeySets - frames = [Frame(frameKeySets, padx=2, pady=2, borderwidth=0) + #frame_custom + frame_target = Frame(frame_custom) + target_title = Label(frame_target, text='Action - Key(s)') + scroll_target_y = Scrollbar(frame_target) + scroll_target_x = Scrollbar(frame_target, orient=HORIZONTAL) + self.list_bindings = Listbox( + frame_target, takefocus=FALSE, exportselection=FALSE) + self.list_bindings.bind('', self.keybinding_selected) + scroll_target_y.config(command=self.list_bindings.yview) + scroll_target_x.config(command=self.list_bindings.xview) + self.list_bindings.config(yscrollcommand=scroll_target_y.set) + self.list_bindings.config(xscrollcommand=scroll_target_x.set) + self.button_new_keys = Button( + frame_custom, text='Get New Keys for Selection', + command=self.get_new_keys, state=DISABLED) + #frame_key_sets + frames = [Frame(frame_key_sets, padx=2, pady=2, borderwidth=0) for i in range(2)] - self.radioKeysBuiltin = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=1, - command=self.SetKeysType, text='Use a Built-in Key Set') - self.radioKeysCustom = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=0, - command=self.SetKeysType, text='Use a Custom Key Set') - self.optMenuKeysBuiltin = DynOptionMenu( - frames[0], self.builtinKeys, None, command=None) - self.optMenuKeysCustom = DynOptionMenu( - frames[0], self.customKeys, None, command=None) - self.buttonDeleteCustomKeys = Button( + self.radio_keys_builtin = Radiobutton( + frames[0], variable=self.are_keys_builtin, value=1, + command=self.set_keys_type, text='Use a Built-in Key Set') + self.radio_keys_custom = Radiobutton( + frames[0], variable=self.are_keys_builtin, value=0, + command=self.set_keys_type, text='Use a Custom Key Set') + self.opt_menu_keys_builtin = DynOptionMenu( + frames[0], self.builtin_keys, None, command=None) + self.opt_menu_keys_custom = DynOptionMenu( + frames[0], self.custom_keys, None, command=None) + self.button_delete_custom_keys = Button( frames[1], text='Delete Custom Key Set', - command=self.DeleteCustomKeys) - buttonSaveCustomKeys = Button( + command=self.delete_custom_keys) + button_save_custom_keys = Button( frames[1], text='Save as New Custom Key Set', - command=self.SaveAsNewKeySet) + command=self.save_as_new_key_set) self.new_custom_keys = Label(frames[0], bd=2) ##widget packing #body - frameCustom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameKeySets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) - #frameCustom - self.buttonNewKeys.pack(side=BOTTOM, fill=X, padx=5, pady=5) - frameTarget.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_custom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_key_sets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) + #frame_custom + self.button_new_keys.pack(side=BOTTOM, fill=X, padx=5, pady=5) + frame_target.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) #frame target - frameTarget.columnconfigure(0, weight=1) - frameTarget.rowconfigure(1, weight=1) - labelTargetTitle.grid(row=0, column=0, columnspan=2, sticky=W) - self.listBindings.grid(row=1, column=0, sticky=NSEW) - scrollTargetY.grid(row=1, column=1, sticky=NS) - scrollTargetX.grid(row=2, column=0, sticky=EW) - #frameKeySets - self.radioKeysBuiltin.grid(row=0, column=0, sticky=W+NS) - self.radioKeysCustom.grid(row=1, column=0, sticky=W+NS) - self.optMenuKeysBuiltin.grid(row=0, column=1, sticky=NSEW) - self.optMenuKeysCustom.grid(row=1, column=1, sticky=NSEW) + frame_target.columnconfigure(0, weight=1) + frame_target.rowconfigure(1, weight=1) + target_title.grid(row=0, column=0, columnspan=2, sticky=W) + self.list_bindings.grid(row=1, column=0, sticky=NSEW) + scroll_target_y.grid(row=1, column=1, sticky=NS) + scroll_target_x.grid(row=2, column=0, sticky=EW) + #frame_key_sets + self.radio_keys_builtin.grid(row=0, column=0, sticky=W+NS) + self.radio_keys_custom.grid(row=1, column=0, sticky=W+NS) + self.opt_menu_keys_builtin.grid(row=0, column=1, sticky=NSEW) + self.opt_menu_keys_custom.grid(row=1, column=1, sticky=NSEW) self.new_custom_keys.grid(row=0, column=2, sticky=NSEW, padx=5, pady=5) - self.buttonDeleteCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) - buttonSaveCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) + self.button_delete_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) + button_save_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) frames[0].pack(side=TOP, fill=BOTH, expand=True) frames[1].pack(side=TOP, fill=X, expand=True, pady=2) return frame - def CreatePageGeneral(self): + def create_page_general(self): parent = self.parent - self.winWidth = StringVar(parent) - self.winHeight = StringVar(parent) - self.startupEdit = IntVar(parent) - self.autoSave = IntVar(parent) + self.win_width = StringVar(parent) + self.win_height = StringVar(parent) + self.startup_edit = IntVar(parent) + self.autosave = IntVar(parent) self.encoding = StringVar(parent) - self.userHelpBrowser = BooleanVar(parent) - self.helpBrowser = StringVar(parent) + self.user_help_browser = BooleanVar(parent) + self.help_browser = StringVar(parent) #widget creation #body - frame = self.tabPages.pages['General'].frame + frame = self.tab_pages.pages['General'].frame #body section frames - frameRun = LabelFrame(frame, borderwidth=2, relief=GROOVE, + frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Startup Preferences ') - frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Autosave Preferences ') - frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE) - frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE, + frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' autosave Preferences ') + frame_win_size = Frame(frame, borderwidth=2, relief=GROOVE) + frame_help = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Additional Help Sources ') - #frameRun - labelRunChoiceTitle = Label(frameRun, text='At Startup') - self.radioStartupEdit = Radiobutton( - frameRun, variable=self.startupEdit, value=1, + #frame_run + startup_title = Label(frame_run, text='At Startup') + self.radio_startup_edit = Radiobutton( + frame_run, variable=self.startup_edit, value=1, text="Open Edit Window") - self.radioStartupShell = Radiobutton( - frameRun, variable=self.startupEdit, value=0, + self.radio_startup_shell = Radiobutton( + frame_run, variable=self.startup_edit, value=0, text='Open Shell Window') - #frameSave - labelRunSaveTitle = Label(frameSave, text='At Start of Run (F5) ') - self.radioSaveAsk = Radiobutton( - frameSave, variable=self.autoSave, value=0, + #frame_save + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + self.radio_save_ask = Radiobutton( + frame_save, variable=self.autosave, value=0, text="Prompt to Save") - self.radioSaveAuto = Radiobutton( - frameSave, variable=self.autoSave, value=1, + self.radio_save_auto = Radiobutton( + frame_save, variable=self.autosave, value=1, text='No Prompt') - #frameWinSize - labelWinSizeTitle = Label( - frameWinSize, text='Initial Window Size (in characters)') - labelWinWidthTitle = Label(frameWinSize, text='Width') - self.entryWinWidth = Entry( - frameWinSize, textvariable=self.winWidth, width=3) - labelWinHeightTitle = Label(frameWinSize, text='Height') - self.entryWinHeight = Entry( - frameWinSize, textvariable=self.winHeight, width=3) - #frameHelp - frameHelpList = Frame(frameHelp) - frameHelpListButtons = Frame(frameHelpList) - scrollHelpList = Scrollbar(frameHelpList) - self.listHelp = Listbox( - frameHelpList, height=5, takefocus=FALSE, + #frame_win_size + win_size_title = Label( + frame_win_size, text='Initial Window Size (in characters)') + win_width_title = Label(frame_win_size, text='Width') + self.entry_win_width = Entry( + frame_win_size, textvariable=self.win_width, width=3) + win_height_title = Label(frame_win_size, text='Height') + self.entry_win_height = Entry( + frame_win_size, textvariable=self.win_height, width=3) + #frame_help + frame_helplist = Frame(frame_help) + frame_helplist_buttons = Frame(frame_helplist) + scroll_helplist = Scrollbar(frame_helplist) + self.list_help = Listbox( + frame_helplist, height=5, takefocus=FALSE, exportselection=FALSE) - scrollHelpList.config(command=self.listHelp.yview) - self.listHelp.config(yscrollcommand=scrollHelpList.set) - self.listHelp.bind('', self.HelpSourceSelected) - self.buttonHelpListEdit = Button( - frameHelpListButtons, text='Edit', state=DISABLED, - width=8, command=self.HelpListItemEdit) - self.buttonHelpListAdd = Button( - frameHelpListButtons, text='Add', - width=8, command=self.HelpListItemAdd) - self.buttonHelpListRemove = Button( - frameHelpListButtons, text='Remove', state=DISABLED, - width=8, command=self.HelpListItemRemove) + scroll_helplist.config(command=self.list_help.yview) + self.list_help.config(yscrollcommand=scroll_helplist.set) + self.list_help.bind('', self.help_source_selected) + self.button_helplist_edit = Button( + frame_helplist_buttons, text='Edit', state=DISABLED, + width=8, command=self.helplist_item_edit) + self.button_helplist_add = Button( + frame_helplist_buttons, text='Add', + width=8, command=self.helplist_item_add) + self.button_helplist_remove = Button( + frame_helplist_buttons, text='Remove', state=DISABLED, + width=8, command=self.helplist_item_remove) #widget packing #body - frameRun.pack(side=TOP, padx=5, pady=5, fill=X) - frameSave.pack(side=TOP, padx=5, pady=5, fill=X) - frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X) - frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - #frameRun - labelRunChoiceTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.radioStartupShell.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.radioStartupEdit.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frameSave - labelRunSaveTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.radioSaveAuto.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.radioSaveAsk.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frameWinSize - labelWinSizeTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.entryWinHeight.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5) - self.entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5) - #frameHelp - frameHelpListButtons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frameHelpList.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scrollHelpList.pack(side=RIGHT, anchor=W, fill=Y) - self.listHelp.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.buttonHelpListEdit.pack(side=TOP, anchor=W, pady=5) - self.buttonHelpListAdd.pack(side=TOP, anchor=W) - self.buttonHelpListRemove.pack(side=TOP, anchor=W, pady=5) + frame_run.pack(side=TOP, padx=5, pady=5, fill=X) + frame_save.pack(side=TOP, padx=5, pady=5, fill=X) + frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) + frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + #frame_run + startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.radio_startup_shell.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.radio_startup_edit.pack(side=RIGHT, anchor=W, padx=5, pady=5) + #frame_save + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.radio_save_auto.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.radio_save_ask.pack(side=RIGHT, anchor=W, padx=5, pady=5) + #frame_win_size + win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.entry_win_height.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_height_title.pack(side=RIGHT, anchor=E, pady=5) + self.entry_win_width.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_width_title.pack(side=RIGHT, anchor=E, pady=5) + #frame_help + frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) + frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) + self.list_help.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) + self.button_helplist_add.pack(side=TOP, anchor=W) + self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) return frame - def AttachVarCallbacks(self): - self.fontSize.trace_add('write', self.VarChanged_font) - self.fontName.trace_add('write', self.VarChanged_font) - self.fontBold.trace_add('write', self.VarChanged_font) - self.spaceNum.trace_add('write', self.VarChanged_spaceNum) - self.colour.trace_add('write', self.VarChanged_colour) - self.builtinTheme.trace_add('write', self.VarChanged_builtinTheme) - self.customTheme.trace_add('write', self.VarChanged_customTheme) - self.themeIsBuiltin.trace_add('write', self.VarChanged_themeIsBuiltin) - self.highlightTarget.trace_add('write', self.VarChanged_highlightTarget) - self.keyBinding.trace_add('write', self.VarChanged_keyBinding) - self.builtinKeys.trace_add('write', self.VarChanged_builtinKeys) - self.customKeys.trace_add('write', self.VarChanged_customKeys) - self.keysAreBuiltin.trace_add('write', self.VarChanged_keysAreBuiltin) - self.winWidth.trace_add('write', self.VarChanged_winWidth) - self.winHeight.trace_add('write', self.VarChanged_winHeight) - self.startupEdit.trace_add('write', self.VarChanged_startupEdit) - self.autoSave.trace_add('write', self.VarChanged_autoSave) - self.encoding.trace_add('write', self.VarChanged_encoding) + def attach_var_callbacks(self): + self.font_size.trace_add('write', self.var_changed_font) + self.font_name.trace_add('write', self.var_changed_font) + self.font_bold.trace_add('write', self.var_changed_font) + self.space_num.trace_add('write', self.var_changed_space_num) + self.colour.trace_add('write', self.var_changed_colour) + self.builtin_theme.trace_add('write', self.var_changed_builtin_theme) + self.custom_theme.trace_add('write', self.var_changed_custom_theme) + self.is_builtin_theme.trace_add('write', self.var_changed_is_builtin_theme) + self.highlight_target.trace_add('write', self.var_changed_highlight_target) + self.keybinding.trace_add('write', self.var_changed_keybinding) + self.builtin_keys.trace_add('write', self.var_changed_builtin_keys) + self.custom_keys.trace_add('write', self.var_changed_custom_keys) + self.are_keys_builtin.trace_add('write', self.var_changed_are_keys_builtin) + self.win_width.trace_add('write', self.var_changed_win_width) + self.win_height.trace_add('write', self.var_changed_win_height) + self.startup_edit.trace_add('write', self.var_changed_startup_edit) + self.autosave.trace_add('write', self.var_changed_autosave) + self.encoding.trace_add('write', self.var_changed_encoding) def remove_var_callbacks(self): "Remove callbacks to prevent memory leaks." for var in ( - self.fontSize, self.fontName, self.fontBold, - self.spaceNum, self.colour, self.builtinTheme, - self.customTheme, self.themeIsBuiltin, self.highlightTarget, - self.keyBinding, self.builtinKeys, self.customKeys, - self.keysAreBuiltin, self.winWidth, self.winHeight, - self.startupEdit, self.autoSave, self.encoding,): + self.font_size, self.font_name, self.font_bold, + self.space_num, self.colour, self.builtin_theme, + self.custom_theme, self.is_builtin_theme, self.highlight_target, + self.keybinding, self.builtin_keys, self.custom_keys, + self.are_keys_builtin, self.win_width, self.win_height, + self.startup_edit, self.autosave, self.encoding,): var.trace_remove('write', var.trace_info()[0][1]) - def VarChanged_font(self, *params): + def var_changed_font(self, *params): '''When one font attribute changes, save them all, as they are not independent from each other. In particular, when we are overriding the default font, we need to write out everything. ''' - value = self.fontName.get() - self.AddChangedItem('main', 'EditorWindow', 'font', value) - value = self.fontSize.get() - self.AddChangedItem('main', 'EditorWindow', 'font-size', value) - value = self.fontBold.get() - self.AddChangedItem('main', 'EditorWindow', 'font-bold', value) - - def VarChanged_spaceNum(self, *params): - value = self.spaceNum.get() - self.AddChangedItem('main', 'Indent', 'num-spaces', value) - - def VarChanged_colour(self, *params): - self.OnNewColourSet() - - def VarChanged_builtinTheme(self, *params): - oldthemes = ('IDLE Classic', 'IDLE New') - value = self.builtinTheme.get() - if value not in oldthemes: - if idleConf.GetOption('main', 'Theme', 'name') not in oldthemes: - self.AddChangedItem('main', 'Theme', 'name', oldthemes[0]) - self.AddChangedItem('main', 'Theme', 'name2', value) + value = self.font_name.get() + self.add_changed_item('main', 'EditorWindow', 'font', value) + value = self.font_size.get() + self.add_changed_item('main', 'EditorWindow', 'font-size', value) + value = self.font_bold.get() + self.add_changed_item('main', 'EditorWindow', 'font-bold', value) + + def var_changed_space_num(self, *params): + value = self.space_num.get() + self.add_changed_item('main', 'Indent', 'num-spaces', value) + + def var_changed_colour(self, *params): + self.on_new_colour_set() + + def var_changed_builtin_theme(self, *params): + old_themes = ('IDLE Classic', 'IDLE New') + value = self.builtin_theme.get() + if value not in old_themes: + if idleConf.GetOption('main', 'Theme', 'name') not in old_themes: + self.add_changed_item('main', 'Theme', 'name', old_themes[0]) + self.add_changed_item('main', 'Theme', 'name2', value) self.new_custom_theme.config(text='New theme, see Help', fg='#500000') else: - self.AddChangedItem('main', 'Theme', 'name', value) - self.AddChangedItem('main', 'Theme', 'name2', '') + self.add_changed_item('main', 'Theme', 'name', value) + self.add_changed_item('main', 'Theme', 'name2', '') self.new_custom_theme.config(text='', fg='black') - self.PaintThemeSample() + self.paint_theme_sample() - def VarChanged_customTheme(self, *params): - value = self.customTheme.get() + def var_changed_custom_theme(self, *params): + value = self.custom_theme.get() if value != '- no custom themes -': - self.AddChangedItem('main', 'Theme', 'name', value) - self.PaintThemeSample() + self.add_changed_item('main', 'Theme', 'name', value) + self.paint_theme_sample() - def VarChanged_themeIsBuiltin(self, *params): - value = self.themeIsBuiltin.get() - self.AddChangedItem('main', 'Theme', 'default', value) + def var_changed_is_builtin_theme(self, *params): + value = self.is_builtin_theme.get() + self.add_changed_item('main', 'Theme', 'default', value) if value: - self.VarChanged_builtinTheme() + self.var_changed_builtin_theme() else: - self.VarChanged_customTheme() + self.var_changed_custom_theme() - def VarChanged_highlightTarget(self, *params): - self.SetHighlightTarget() + def var_changed_highlight_target(self, *params): + self.set_highlight_target() - def VarChanged_keyBinding(self, *params): - value = self.keyBinding.get() - keySet = self.customKeys.get() - event = self.listBindings.get(ANCHOR).split()[0] + def var_changed_keybinding(self, *params): + value = self.keybinding.get() + key_set = self.custom_keys.get() + event = self.list_bindings.get(ANCHOR).split()[0] if idleConf.IsCoreBinding(event): #this is a core keybinding - self.AddChangedItem('keys', keySet, event, value) + self.add_changed_item('keys', key_set, event, value) else: #this is an extension key binding - extName = idleConf.GetExtnNameForEvent(event) - extKeybindSection = extName + '_cfgBindings' - self.AddChangedItem('extensions', extKeybindSection, event, value) + ext_name = idleConf.GetExtnNameForEvent(event) + ext_keybind_section = ext_name + '_cfgBindings' + self.add_changed_item('extensions', ext_keybind_section, event, value) - def VarChanged_builtinKeys(self, *params): - oldkeys = ( + def var_changed_builtin_keys(self, *params): + old_keys = ( 'IDLE Classic Windows', 'IDLE Classic Unix', 'IDLE Classic Mac', 'IDLE Classic OSX', ) - value = self.builtinKeys.get() - if value not in oldkeys: - if idleConf.GetOption('main', 'Keys', 'name') not in oldkeys: - self.AddChangedItem('main', 'Keys', 'name', oldkeys[0]) - self.AddChangedItem('main', 'Keys', 'name2', value) + value = self.builtin_keys.get() + if value not in old_keys: + if idleConf.GetOption('main', 'Keys', 'name') not in old_keys: + self.add_changed_item('main', 'Keys', 'name', old_keys[0]) + self.add_changed_item('main', 'Keys', 'name2', value) self.new_custom_keys.config(text='New key set, see Help', fg='#500000') else: - self.AddChangedItem('main', 'Keys', 'name', value) - self.AddChangedItem('main', 'Keys', 'name2', '') + self.add_changed_item('main', 'Keys', 'name', value) + self.add_changed_item('main', 'Keys', 'name2', '') self.new_custom_keys.config(text='', fg='black') - self.LoadKeysList(value) + self.load_keys_list(value) - def VarChanged_customKeys(self, *params): - value = self.customKeys.get() + def var_changed_custom_keys(self, *params): + value = self.custom_keys.get() if value != '- no custom keys -': - self.AddChangedItem('main', 'Keys', 'name', value) - self.LoadKeysList(value) + self.add_changed_item('main', 'Keys', 'name', value) + self.load_keys_list(value) - def VarChanged_keysAreBuiltin(self, *params): - value = self.keysAreBuiltin.get() - self.AddChangedItem('main', 'Keys', 'default', value) + def var_changed_are_keys_builtin(self, *params): + value = self.are_keys_builtin.get() + self.add_changed_item('main', 'Keys', 'default', value) if value: - self.VarChanged_builtinKeys() + self.var_changed_builtin_keys() else: - self.VarChanged_customKeys() + self.var_changed_custom_keys() - def VarChanged_winWidth(self, *params): - value = self.winWidth.get() - self.AddChangedItem('main', 'EditorWindow', 'width', value) + def var_changed_win_width(self, *params): + value = self.win_width.get() + self.add_changed_item('main', 'EditorWindow', 'width', value) - def VarChanged_winHeight(self, *params): - value = self.winHeight.get() - self.AddChangedItem('main', 'EditorWindow', 'height', value) + def var_changed_win_height(self, *params): + value = self.win_height.get() + self.add_changed_item('main', 'EditorWindow', 'height', value) - def VarChanged_startupEdit(self, *params): - value = self.startupEdit.get() - self.AddChangedItem('main', 'General', 'editor-on-startup', value) + def var_changed_startup_edit(self, *params): + value = self.startup_edit.get() + self.add_changed_item('main', 'General', 'editor-on-startup', value) - def VarChanged_autoSave(self, *params): - value = self.autoSave.get() - self.AddChangedItem('main', 'General', 'autosave', value) + def var_changed_autosave(self, *params): + value = self.autosave.get() + self.add_changed_item('main', 'General', 'autosave', value) - def VarChanged_encoding(self, *params): + def var_changed_encoding(self, *params): value = self.encoding.get() - self.AddChangedItem('main', 'EditorWindow', 'encoding', value) + self.add_changed_item('main', 'EditorWindow', 'encoding', value) - def ResetChangedItems(self): + def reset_changed_items(self): #When any config item is changed in this dialog, an entry #should be made in the relevant section (config type) of this #dictionary. The key should be the config file section name and the #value a dictionary, whose key:value pairs are item=value pairs for #that config file section. - self.changedItems = {'main':{}, 'highlight':{}, 'keys':{}, + self.changed_items = {'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} - def AddChangedItem(self, typ, section, item, value): + def add_changed_item(self, typ, section, item, value): value = str(value) #make sure we use a string - if section not in self.changedItems[typ]: - self.changedItems[typ][section] = {} - self.changedItems[typ][section][item] = value + if section not in self.changed_items[typ]: + self.changed_items[typ][section] = {} + self.changed_items[typ][section][item] = value def GetDefaultItems(self): - dItems={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} - for configType in dItems: - sections = idleConf.GetSectionList('default', configType) + d_items={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} + for config_type in d_items: + sections = idleConf.GetSectionList('default', config_type) for section in sections: - dItems[configType][section] = {} - options = idleConf.defaultCfg[configType].GetOptionList(section) + d_items[config_type][section] = {} + options = idleConf.defaultCfg[config_type].GetOptionList(section) for option in options: - dItems[configType][section][option] = ( - idleConf.defaultCfg[configType].Get(section, option)) - return dItems - - def SetThemeType(self): - if self.themeIsBuiltin.get(): - self.optMenuThemeBuiltin.config(state=NORMAL) - self.optMenuThemeCustom.config(state=DISABLED) - self.buttonDeleteCustomTheme.config(state=DISABLED) + d_items[config_type][section][option] = ( + idleConf.defaultCfg[config_type].Get(section, option)) + return d_items + + def set_theme_type(self): + if self.is_builtin_theme.get(): + self.opt_menu_theme_builtin.config(state=NORMAL) + self.opt_menu_theme_custom.config(state=DISABLED) + self.button_delete_custom_theme.config(state=DISABLED) else: - self.optMenuThemeBuiltin.config(state=DISABLED) - self.radioThemeCustom.config(state=NORMAL) - self.optMenuThemeCustom.config(state=NORMAL) - self.buttonDeleteCustomTheme.config(state=NORMAL) - - def SetKeysType(self): - if self.keysAreBuiltin.get(): - self.optMenuKeysBuiltin.config(state=NORMAL) - self.optMenuKeysCustom.config(state=DISABLED) - self.buttonDeleteCustomKeys.config(state=DISABLED) + self.opt_menu_theme_builtin.config(state=DISABLED) + self.radio_theme_custom.config(state=NORMAL) + self.opt_menu_theme_custom.config(state=NORMAL) + self.button_delete_custom_theme.config(state=NORMAL) + + def set_keys_type(self): + if self.are_keys_builtin.get(): + self.opt_menu_keys_builtin.config(state=NORMAL) + self.opt_menu_keys_custom.config(state=DISABLED) + self.button_delete_custom_keys.config(state=DISABLED) else: - self.optMenuKeysBuiltin.config(state=DISABLED) - self.radioKeysCustom.config(state=NORMAL) - self.optMenuKeysCustom.config(state=NORMAL) - self.buttonDeleteCustomKeys.config(state=NORMAL) - - def GetNewKeys(self): - listIndex = self.listBindings.index(ANCHOR) - binding = self.listBindings.get(listIndex) - bindName = binding.split()[0] #first part, up to first space - if self.keysAreBuiltin.get(): - currentKeySetName = self.builtinKeys.get() + self.opt_menu_keys_builtin.config(state=DISABLED) + self.radio_keys_custom.config(state=NORMAL) + self.opt_menu_keys_custom.config(state=NORMAL) + self.button_delete_custom_keys.config(state=NORMAL) + + def get_new_keys(self): + list_index = self.list_bindings.index(ANCHOR) + binding = self.list_bindings.get(list_index) + bind_name = binding.split()[0] #first part, up to first space + if self.are_keys_builtin.get(): + current_key_set_name = self.builtin_keys.get() else: - currentKeySetName = self.customKeys.get() - currentBindings = idleConf.GetCurrentKeySet() - if currentKeySetName in self.changedItems['keys']: #unsaved changes - keySetChanges = self.changedItems['keys'][currentKeySetName] - for event in keySetChanges: - currentBindings[event] = keySetChanges[event].split() - currentKeySequences = list(currentBindings.values()) - newKeys = GetKeysDialog(self, 'Get New Keys', bindName, - currentKeySequences).result - if newKeys: #new keys were specified - if self.keysAreBuiltin.get(): #current key set is a built-in + current_key_set_name = self.custom_keys.get() + current_bindings = idleConf.GetCurrentKeySet() + if current_key_set_name in self.changed_items['keys']: #unsaved changes + key_set_changes = self.changed_items['keys'][current_key_set_name] + for event in key_set_changes: + current_bindings[event] = key_set_changes[event].split() + current_key_sequences = list(current_bindings.values()) + new_keys = GetKeysDialog(self, 'Get New Keys', bind_name, + current_key_sequences).result + if new_keys: #new keys were specified + if self.are_keys_builtin.get(): #current key set is a built-in message = ('Your changes will be saved as a new Custom Key Set.' ' Enter a name for your new Custom Key Set below.') - newKeySet = self.GetNewKeysName(message) - if not newKeySet: #user cancelled custom key set creation - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) + new_keyset = self.get_new_keys_name(message) + if not new_keyset: #user cancelled custom key set creation + self.list_bindings.select_set(list_index) + self.list_bindings.select_anchor(list_index) return else: #create new custom key set based on previously active key set - self.CreateNewKeySet(newKeySet) - self.listBindings.delete(listIndex) - self.listBindings.insert(listIndex, bindName+' - '+newKeys) - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) - self.keyBinding.set(newKeys) + self.create_new_key_set(new_keyset) + self.list_bindings.delete(list_index) + self.list_bindings.insert(list_index, bind_name+' - '+new_keys) + self.list_bindings.select_set(list_index) + self.list_bindings.select_anchor(list_index) + self.keybinding.set(new_keys) else: - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) + self.list_bindings.select_set(list_index) + self.list_bindings.select_anchor(list_index) - def GetNewKeysName(self, message): - usedNames = (idleConf.GetSectionList('user', 'keys') + + def get_new_keys_name(self, message): + used_names = (idleConf.GetSectionList('user', 'keys') + idleConf.GetSectionList('default', 'keys')) - newKeySet = SectionName( - self, 'New Custom Key Set', message, usedNames).result - return newKeySet + new_keyset = SectionName( + self, 'New Custom Key Set', message, used_names).result + return new_keyset - def SaveAsNewKeySet(self): - newKeysName = self.GetNewKeysName('New Key Set Name:') - if newKeysName: - self.CreateNewKeySet(newKeysName) + def save_as_new_key_set(self): + new_keys_name = self.get_new_keys_name('New Key Set Name:') + if new_keys_name: + self.create_new_key_set(new_keys_name) - def KeyBindingSelected(self, event): - self.buttonNewKeys.config(state=NORMAL) + def keybinding_selected(self, event): + self.button_new_keys.config(state=NORMAL) - def CreateNewKeySet(self, newKeySetName): + def create_new_key_set(self, new_key_set_name): #creates new custom key set based on the previously active key set, #and makes the new key set active - if self.keysAreBuiltin.get(): - prevKeySetName = self.builtinKeys.get() + if self.are_keys_builtin.get(): + prev_key_set_name = self.builtin_keys.get() else: - prevKeySetName = self.customKeys.get() - prevKeys = idleConf.GetCoreKeys(prevKeySetName) - newKeys = {} - for event in prevKeys: #add key set to changed items - eventName = event[2:-2] #trim off the angle brackets - binding = ' '.join(prevKeys[event]) - newKeys[eventName] = binding + prev_key_set_name = self.custom_keys.get() + prev_keys = idleConf.GetCoreKeys(prev_key_set_name) + new_keys = {} + for event in prev_keys: #add key set to changed items + event_name = event[2:-2] #trim off the angle brackets + binding = ' '.join(prev_keys[event]) + new_keys[event_name] = binding #handle any unsaved changes to prev key set - if prevKeySetName in self.changedItems['keys']: - keySetChanges = self.changedItems['keys'][prevKeySetName] - for event in keySetChanges: - newKeys[event] = keySetChanges[event] + if prev_key_set_name in self.changed_items['keys']: + key_set_changes = self.changed_items['keys'][prev_key_set_name] + for event in key_set_changes: + new_keys[event] = key_set_changes[event] #save the new theme - self.SaveNewKeySet(newKeySetName, newKeys) + self.save_new_key_set(new_key_set_name, new_keys) #change gui over to the new key set - customKeyList = idleConf.GetSectionList('user', 'keys') - customKeyList.sort() - self.optMenuKeysCustom.SetMenu(customKeyList, newKeySetName) - self.keysAreBuiltin.set(0) - self.SetKeysType() + custom_key_list = idleConf.GetSectionList('user', 'keys') + custom_key_list.sort() + self.opt_menu_keys_custom.SetMenu(custom_key_list, new_key_set_name) + self.are_keys_builtin.set(0) + self.set_keys_type() - def LoadKeysList(self, keySetName): + def load_keys_list(self, keyset_name): reselect = 0 - newKeySet = 0 - if self.listBindings.curselection(): + new_keyset = 0 + if self.list_bindings.curselection(): reselect = 1 - listIndex = self.listBindings.index(ANCHOR) - keySet = idleConf.GetKeySet(keySetName) - bindNames = list(keySet.keys()) - bindNames.sort() - self.listBindings.delete(0, END) - for bindName in bindNames: - key = ' '.join(keySet[bindName]) #make key(s) into a string - bindName = bindName[2:-2] #trim off the angle brackets - if keySetName in self.changedItems['keys']: + list_index = self.list_bindings.index(ANCHOR) + keyset = idleConf.GetKeySet(keyset_name) + bind_names = list(keyset.keys()) + bind_names.sort() + self.list_bindings.delete(0, END) + for bind_name in bind_names: + key = ' '.join(keyset[bind_name]) #make key(s) into a string + bind_name = bind_name[2:-2] #trim off the angle brackets + if keyset_name in self.changed_items['keys']: #handle any unsaved changes to this key set - if bindName in self.changedItems['keys'][keySetName]: - key = self.changedItems['keys'][keySetName][bindName] - self.listBindings.insert(END, bindName+' - '+key) + if bind_name in self.changed_items['keys'][keyset_name]: + key = self.changed_items['keys'][keyset_name][bind_name] + self.list_bindings.insert(END, bind_name+' - '+key) if reselect: - self.listBindings.see(listIndex) - self.listBindings.select_set(listIndex) - self.listBindings.select_anchor(listIndex) + self.list_bindings.see(list_index) + self.list_bindings.select_set(list_index) + self.list_bindings.select_anchor(list_index) - def DeleteCustomKeys(self): - keySetName=self.customKeys.get() + def delete_custom_keys(self): + keyset_name=self.custom_keys.get() delmsg = 'Are you sure you wish to delete the key set %r ?' if not tkMessageBox.askyesno( - 'Delete Key Set', delmsg % keySetName, parent=self): + 'Delete Key Set', delmsg % keyset_name, parent=self): return - self.DeactivateCurrentConfig() + self.deactivate_current_config() #remove key set from config - idleConf.userCfg['keys'].remove_section(keySetName) - if keySetName in self.changedItems['keys']: - del(self.changedItems['keys'][keySetName]) + idleConf.userCfg['keys'].remove_section(keyset_name) + if keyset_name in self.changed_items['keys']: + del(self.changed_items['keys'][keyset_name]) #write changes idleConf.userCfg['keys'].Save() #reload user key set list - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - if not itemList: - self.radioKeysCustom.config(state=DISABLED) - self.optMenuKeysCustom.SetMenu(itemList, '- no custom keys -') + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + if not item_list: + self.radio_keys_custom.config(state=DISABLED) + self.opt_menu_keys_custom.SetMenu(item_list, '- no custom keys -') else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) #revert to default key set - self.keysAreBuiltin.set(idleConf.defaultCfg['main'] + self.are_keys_builtin.set(idleConf.defaultCfg['main'] .Get('Keys', 'default')) - self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') + self.builtin_keys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') or idleConf.default_keys()) #user can't back out of these changes, they must be applied now - self.SaveAllChangedConfigs() - self.ActivateConfigChanges() - self.SetKeysType() + self.save_all_changed_configs() + self.activate_config_changes() + self.set_keys_type() - def DeleteCustomTheme(self): - themeName = self.customTheme.get() + def delete_custom_theme(self): + theme_name = self.custom_theme.get() delmsg = 'Are you sure you wish to delete the theme %r ?' if not tkMessageBox.askyesno( - 'Delete Theme', delmsg % themeName, parent=self): + 'Delete Theme', delmsg % theme_name, parent=self): return - self.DeactivateCurrentConfig() + self.deactivate_current_config() #remove theme from config - idleConf.userCfg['highlight'].remove_section(themeName) - if themeName in self.changedItems['highlight']: - del(self.changedItems['highlight'][themeName]) + idleConf.userCfg['highlight'].remove_section(theme_name) + if theme_name in self.changed_items['highlight']: + del(self.changed_items['highlight'][theme_name]) #write changes idleConf.userCfg['highlight'].Save() #reload user theme list - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - if not itemList: - self.radioThemeCustom.config(state=DISABLED) - self.optMenuThemeCustom.SetMenu(itemList, '- no custom themes -') + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.radio_theme_custom.config(state=DISABLED) + self.opt_menu_theme_custom.SetMenu(item_list, '- no custom themes -') else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) #revert to default theme - self.themeIsBuiltin.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) - self.builtinTheme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) + self.is_builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) + self.builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) #user can't back out of these changes, they must be applied now - self.SaveAllChangedConfigs() - self.ActivateConfigChanges() - self.SetThemeType() - - def GetColour(self): - target = self.highlightTarget.get() - prevColour = self.frameColourSet.cget('bg') - rgbTuplet, colourString = tkColorChooser.askcolor( + self.save_all_changed_configs() + self.activate_config_changes() + self.set_theme_type() + + def get_colour(self): + target = self.highlight_target.get() + prev_colour = self.frame_colour_set.cget('bg') + rgbTuplet, colour_string = tkColorChooser.askcolor( parent=self, title='Pick new colour for : '+target, - initialcolor=prevColour) - if colourString and (colourString != prevColour): + initialcolor=prev_colour) + if colour_string and (colour_string != prev_colour): #user didn't cancel, and they chose a new colour - if self.themeIsBuiltin.get(): #current theme is a built-in + if self.is_builtin_theme.get(): #current theme is a built-in message = ('Your changes will be saved as a new Custom Theme. ' 'Enter a name for your new Custom Theme below.') - newTheme = self.GetNewThemeName(message) - if not newTheme: #user cancelled custom theme creation + new_theme = self.get_new_theme_name(message) + if not new_theme: #user cancelled custom theme creation return else: #create new custom theme based on previously active theme - self.CreateNewTheme(newTheme) - self.colour.set(colourString) + self.create_new_theme(new_theme) + self.colour.set(colour_string) else: #current theme is user defined - self.colour.set(colourString) - - def OnNewColourSet(self): - newColour=self.colour.get() - self.frameColourSet.config(bg=newColour) #set sample - plane ='foreground' if self.fgHilite.get() else 'background' - sampleElement = self.themeElements[self.highlightTarget.get()][0] - self.textHighlightSample.tag_config(sampleElement, **{plane:newColour}) - theme = self.customTheme.get() - themeElement = sampleElement + '-' + plane - self.AddChangedItem('highlight', theme, themeElement, newColour) - - def GetNewThemeName(self, message): - usedNames = (idleConf.GetSectionList('user', 'highlight') + + self.colour.set(colour_string) + + def on_new_colour_set(self): + new_colour=self.colour.get() + self.frame_colour_set.config(bg=new_colour) #set sample + plane ='foreground' if self.fg_bg_toggle.get() else 'background' + sample_element = self.theme_elements[self.highlight_target.get()][0] + self.text_highlight_sample.tag_config(sample_element, **{plane:new_colour}) + theme = self.custom_theme.get() + theme_element = sample_element + '-' + plane + self.add_changed_item('highlight', theme, theme_element, new_colour) + + def get_new_theme_name(self, message): + used_names = (idleConf.GetSectionList('user', 'highlight') + idleConf.GetSectionList('default', 'highlight')) - newTheme = SectionName( - self, 'New Custom Theme', message, usedNames).result - return newTheme + new_theme = SectionName( + self, 'New Custom Theme', message, used_names).result + return new_theme - def SaveAsNewTheme(self): - newThemeName = self.GetNewThemeName('New Theme Name:') - if newThemeName: - self.CreateNewTheme(newThemeName) + def save_as_new_theme(self): + new_theme_name = self.get_new_theme_name('New Theme Name:') + if new_theme_name: + self.create_new_theme(new_theme_name) - def CreateNewTheme(self, newThemeName): + def create_new_theme(self, new_theme_name): #creates new custom theme based on the previously active theme, #and makes the new theme active - if self.themeIsBuiltin.get(): - themeType = 'default' - themeName = self.builtinTheme.get() + if self.is_builtin_theme.get(): + theme_type = 'default' + theme_name = self.builtin_theme.get() else: - themeType = 'user' - themeName = self.customTheme.get() - newTheme = idleConf.GetThemeDict(themeType, themeName) + theme_type = 'user' + theme_name = self.custom_theme.get() + new_theme = idleConf.GetThemeDict(theme_type, theme_name) #apply any of the old theme's unsaved changes to the new theme - if themeName in self.changedItems['highlight']: - themeChanges = self.changedItems['highlight'][themeName] - for element in themeChanges: - newTheme[element] = themeChanges[element] + if theme_name in self.changed_items['highlight']: + theme_changes = self.changed_items['highlight'][theme_name] + for element in theme_changes: + new_theme[element] = theme_changes[element] #save the new theme - self.SaveNewTheme(newThemeName, newTheme) + self.save_new_theme(new_theme_name, new_theme) #change gui over to the new theme - customThemeList = idleConf.GetSectionList('user', 'highlight') - customThemeList.sort() - self.optMenuThemeCustom.SetMenu(customThemeList, newThemeName) - self.themeIsBuiltin.set(0) - self.SetThemeType() - - def OnListFontButtonRelease(self, event): - font = self.listFontName.get(ANCHOR) - self.fontName.set(font.lower()) - self.SetFontSample() - - def SetFontSample(self, event=None): - fontName = self.fontName.get() - fontWeight = tkFont.BOLD if self.fontBold.get() else tkFont.NORMAL - newFont = (fontName, self.fontSize.get(), fontWeight) - self.labelFontSample.config(font=newFont) - self.textHighlightSample.configure(font=newFont) - - def SetHighlightTarget(self): - if self.highlightTarget.get() == 'Cursor': #bg not possible - self.radioFg.config(state=DISABLED) - self.radioBg.config(state=DISABLED) - self.fgHilite.set(1) + custom_theme_list = idleConf.GetSectionList('user', 'highlight') + custom_theme_list.sort() + self.opt_menu_theme_custom.SetMenu(custom_theme_list, new_theme_name) + self.is_builtin_theme.set(0) + self.set_theme_type() + + def on_list_fonts_button_release(self, event): + font = self.list_fonts.get(ANCHOR) + self.font_name.set(font.lower()) + self.set_font_sample() + + def set_font_sample(self, event=None): + font_name = self.font_name.get() + font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + new_font = (font_name, self.font_size.get(), font_weight) + self.font_sample.config(font=new_font) + self.text_highlight_sample.configure(font=new_font) + + def set_highlight_target(self): + if self.highlight_target.get() == 'Cursor': #bg not possible + self.radio_fg.config(state=DISABLED) + self.radio_bg.config(state=DISABLED) + self.fg_bg_toggle.set(1) else: #both fg and bg can be set - self.radioFg.config(state=NORMAL) - self.radioBg.config(state=NORMAL) - self.fgHilite.set(1) - self.SetColourSample() + self.radio_fg.config(state=NORMAL) + self.radio_bg.config(state=NORMAL) + self.fg_bg_toggle.set(1) + self.set_colour_sample() - def SetColourSampleBinding(self, *args): - self.SetColourSample() + def set_colour_sample_binding(self, *args): + self.set_colour_sample() - def SetColourSample(self): + def set_colour_sample(self): #set the colour smaple area - tag = self.themeElements[self.highlightTarget.get()][0] - plane = 'foreground' if self.fgHilite.get() else 'background' - colour = self.textHighlightSample.tag_cget(tag, plane) - self.frameColourSet.config(bg=colour) - - def PaintThemeSample(self): - if self.themeIsBuiltin.get(): #a default theme - theme = self.builtinTheme.get() + tag = self.theme_elements[self.highlight_target.get()][0] + plane = 'foreground' if self.fg_bg_toggle.get() else 'background' + colour = self.text_highlight_sample.tag_cget(tag, plane) + self.frame_colour_set.config(bg=colour) + + def paint_theme_sample(self): + if self.is_builtin_theme.get(): #a default theme + theme = self.builtin_theme.get() else: #a user theme - theme = self.customTheme.get() - for elementTitle in self.themeElements: - element = self.themeElements[elementTitle][0] + theme = self.custom_theme.get() + for element_title in self.theme_elements: + element = self.theme_elements[element_title][0] colours = idleConf.GetHighlight(theme, element) if element == 'cursor': #cursor sample needs special painting colours['background'] = idleConf.GetHighlight( theme, 'normal', fgBg='bg') #handle any unsaved changes to this theme - if theme in self.changedItems['highlight']: - themeDict = self.changedItems['highlight'][theme] - if element + '-foreground' in themeDict: - colours['foreground'] = themeDict[element + '-foreground'] - if element + '-background' in themeDict: - colours['background'] = themeDict[element + '-background'] - self.textHighlightSample.tag_config(element, **colours) - self.SetColourSample() - - def HelpSourceSelected(self, event): - self.SetHelpListButtonStates() - - def SetHelpListButtonStates(self): - if self.listHelp.size() < 1: #no entries in list - self.buttonHelpListEdit.config(state=DISABLED) - self.buttonHelpListRemove.config(state=DISABLED) + if theme in self.changed_items['highlight']: + theme_dict = self.changed_items['highlight'][theme] + if element + '-foreground' in theme_dict: + colours['foreground'] = theme_dict[element + '-foreground'] + if element + '-background' in theme_dict: + colours['background'] = theme_dict[element + '-background'] + self.text_highlight_sample.tag_config(element, **colours) + self.set_colour_sample() + + def help_source_selected(self, event): + self.set_helplist_button_states() + + def set_helplist_button_states(self): + if self.list_help.size() < 1: #no entries in list + self.button_helplist_edit.config(state=DISABLED) + self.button_helplist_remove.config(state=DISABLED) else: #there are some entries - if self.listHelp.curselection(): #there currently is a selection - self.buttonHelpListEdit.config(state=NORMAL) - self.buttonHelpListRemove.config(state=NORMAL) + if self.list_help.curselection(): #there currently is a selection + self.button_helplist_edit.config(state=NORMAL) + self.button_helplist_remove.config(state=NORMAL) else: #there currently is not a selection - self.buttonHelpListEdit.config(state=DISABLED) - self.buttonHelpListRemove.config(state=DISABLED) + self.button_helplist_edit.config(state=DISABLED) + self.button_helplist_remove.config(state=DISABLED) - def HelpListItemAdd(self): - helpSource = HelpSource(self, 'New Help Source', + def helplist_item_add(self): + help_source = HelpSource(self, 'New Help Source', ).result - if helpSource: - self.userHelpList.append((helpSource[0], helpSource[1])) - self.listHelp.insert(END, helpSource[0]) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def HelpListItemEdit(self): - itemIndex = self.listHelp.index(ANCHOR) - helpSource = self.userHelpList[itemIndex] - newHelpSource = HelpSource( + if help_source: + self.user_helplist.append((help_source[0], help_source[1])) + self.list_help.insert(END, help_source[0]) + self.update_user_help_changed_items() + self.set_helplist_button_states() + + def helplist_item_edit(self): + item_index = self.list_help.index(ANCHOR) + help_source = self.user_helplist[item_index] + new_help_source = HelpSource( self, 'Edit Help Source', - menuitem=helpSource[0], - filepath=helpSource[1], + menuitem=help_source[0], + filepath=help_source[1], ).result - if newHelpSource and newHelpSource != helpSource: - self.userHelpList[itemIndex] = newHelpSource - self.listHelp.delete(itemIndex) - self.listHelp.insert(itemIndex, newHelpSource[0]) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def HelpListItemRemove(self): - itemIndex = self.listHelp.index(ANCHOR) - del(self.userHelpList[itemIndex]) - self.listHelp.delete(itemIndex) - self.UpdateUserHelpChangedItems() - self.SetHelpListButtonStates() - - def UpdateUserHelpChangedItems(self): - "Clear and rebuild the HelpFiles section in self.changedItems" - self.changedItems['main']['HelpFiles'] = {} - for num in range(1, len(self.userHelpList) + 1): - self.AddChangedItem( + if new_help_source and new_help_source != help_source: + self.user_helplist[item_index] = new_help_source + self.list_help.delete(item_index) + self.list_help.insert(item_index, new_help_source[0]) + self.update_user_help_changed_items() + self.set_helplist_button_states() + + def helplist_item_remove(self): + item_index = self.list_help.index(ANCHOR) + del(self.user_helplist[item_index]) + self.list_help.delete(item_index) + self.update_user_help_changed_items() + self.set_helplist_button_states() + + def update_user_help_changed_items(self): + "Clear and rebuild the HelpFiles section in self.changed_items" + self.changed_items['main']['HelpFiles'] = {} + for num in range(1, len(self.user_helplist) + 1): + self.add_changed_item( 'main', 'HelpFiles', str(num), - ';'.join(self.userHelpList[num-1][:2])) + ';'.join(self.user_helplist[num-1][:2])) - def LoadFontCfg(self): + def load_font_cfg(self): ##base editor font selection list fonts = list(tkFont.families(self)) fonts.sort() for font in fonts: - self.listFontName.insert(END, font) - configuredFont = idleConf.GetFont(self, 'main', 'EditorWindow') - fontName = configuredFont[0].lower() - fontSize = configuredFont[1] - fontBold = configuredFont[2]=='bold' - self.fontName.set(fontName) + self.list_fonts.insert(END, font) + configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') + font_name = configured_font[0].lower() + font_size = configured_font[1] + font_bold = configured_font[2]=='bold' + self.font_name.set(font_name) lc_fonts = [s.lower() for s in fonts] try: - currentFontIndex = lc_fonts.index(fontName) - self.listFontName.see(currentFontIndex) - self.listFontName.select_set(currentFontIndex) - self.listFontName.select_anchor(currentFontIndex) + current_font_index = lc_fonts.index(font_name) + self.list_fonts.see(current_font_index) + self.list_fonts.select_set(current_font_index) + self.list_fonts.select_anchor(current_font_index) except ValueError: pass ##font size dropdown - self.optMenuFontSize.SetMenu(('7', '8', '9', '10', '11', '12', '13', + self.opt_menu_font_size.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', '16', '18', '20', '22', - '25', '29', '34', '40'), fontSize ) - ##fontWeight - self.fontBold.set(fontBold) + '25', '29', '34', '40'), font_size ) + ##font_weight + self.font_bold.set(font_bold) ##font sample - self.SetFontSample() + self.set_font_sample() - def LoadTabCfg(self): + def load_tab_cfg(self): ##indent sizes - spaceNum = idleConf.GetOption( + space_num = idleConf.GetOption( 'main', 'Indent', 'num-spaces', default=4, type='int') - self.spaceNum.set(spaceNum) + self.space_num.set(space_num) - def LoadThemeCfg(self): + def load_theme_cfg(self): ##current theme type radiobutton - self.themeIsBuiltin.set(idleConf.GetOption( + self.is_builtin_theme.set(idleConf.GetOption( 'main', 'Theme', 'default', type='bool', default=1)) ##currently set theme - currentOption = idleConf.CurrentTheme() + current_option = idleConf.CurrentTheme() ##load available theme option menus - if self.themeIsBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'highlight') - itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - if not itemList: - self.radioThemeCustom.config(state=DISABLED) - self.customTheme.set('- no custom themes -') + if self.is_builtin_theme.get(): #default theme selected + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.opt_menu_theme_builtin.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.radio_theme_custom.config(state=DISABLED) + self.custom_theme.set('- no custom themes -') else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) else: #user theme selected - itemList = idleConf.GetSectionList('user', 'highlight') - itemList.sort() - self.optMenuThemeCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'highlight') - itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, itemList[0]) - self.SetThemeType() + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + self.opt_menu_theme_custom.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.opt_menu_theme_builtin.SetMenu(item_list, item_list[0]) + self.set_theme_type() ##load theme element option menu - themeNames = list(self.themeElements.keys()) - themeNames.sort(key=lambda x: self.themeElements[x][1]) - self.optMenuHighlightTarget.SetMenu(themeNames, themeNames[0]) - self.PaintThemeSample() - self.SetHighlightTarget() + theme_names = list(self.theme_elements.keys()) + theme_names.sort(key=lambda x: self.theme_elements[x][1]) + self.opt_menu_highlight_target.SetMenu(theme_names, theme_names[0]) + self.paint_theme_sample() + self.set_highlight_target() - def LoadKeyCfg(self): + def load_key_cfg(self): ##current keys type radiobutton - self.keysAreBuiltin.set(idleConf.GetOption( + self.are_keys_builtin.set(idleConf.GetOption( 'main', 'Keys', 'default', type='bool', default=1)) ##currently set keys - currentOption = idleConf.CurrentKeys() + current_option = idleConf.CurrentKeys() ##load available keyset option menus - if self.keysAreBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'keys') - itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - if not itemList: - self.radioKeysCustom.config(state=DISABLED) - self.customKeys.set('- no custom keys -') + if self.are_keys_builtin.get(): #default theme selected + item_list = idleConf.GetSectionList('default', 'keys') + item_list.sort() + self.opt_menu_keys_builtin.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + if not item_list: + self.radio_keys_custom.config(state=DISABLED) + self.custom_keys.set('- no custom keys -') else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) else: #user key set selected - itemList = idleConf.GetSectionList('user', 'keys') - itemList.sort() - self.optMenuKeysCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'keys') - itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, idleConf.default_keys()) - self.SetKeysType() + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + self.opt_menu_keys_custom.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('default', 'keys') + item_list.sort() + self.opt_menu_keys_builtin.SetMenu(item_list, idleConf.default_keys()) + self.set_keys_type() ##load keyset element list - keySetName = idleConf.CurrentKeys() - self.LoadKeysList(keySetName) + keyset_name = idleConf.CurrentKeys() + self.load_keys_list(keyset_name) - def LoadGeneralCfg(self): + def load_general_cfg(self): #startup state - self.startupEdit.set(idleConf.GetOption( + self.startup_edit.set(idleConf.GetOption( 'main', 'General', 'editor-on-startup', default=1, type='bool')) #autosave state - self.autoSave.set(idleConf.GetOption( + self.autosave.set(idleConf.GetOption( 'main', 'General', 'autosave', default=0, type='bool')) #initial window size - self.winWidth.set(idleConf.GetOption( + self.win_width.set(idleConf.GetOption( 'main', 'EditorWindow', 'width', type='int')) - self.winHeight.set(idleConf.GetOption( + self.win_height.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) # default source encoding self.encoding.set(idleConf.GetOption( 'main', 'EditorWindow', 'encoding', default='none')) # additional help sources - self.userHelpList = idleConf.GetAllExtraHelpSourcesList() - for helpItem in self.userHelpList: - self.listHelp.insert(END, helpItem[0]) - self.SetHelpListButtonStates() + self.user_helplist = idleConf.GetAllExtraHelpSourcesList() + for help_item in self.user_helplist: + self.list_help.insert(END, help_item[0]) + self.set_helplist_button_states() - def LoadConfigs(self): + def load_configs(self): """ load configuration from default and user config files and populate the widgets on the config dialog pages. """ ### fonts / tabs page - self.LoadFontCfg() - self.LoadTabCfg() + self.load_font_cfg() + self.load_tab_cfg() ### highlighting page - self.LoadThemeCfg() + self.load_theme_cfg() ### keys page - self.LoadKeyCfg() + self.load_key_cfg() ### general page - self.LoadGeneralCfg() + self.load_general_cfg() # note: extension page handled separately - def SaveNewKeySet(self, keySetName, keySet): + def save_new_key_set(self, keyset_name, keyset): """ save a newly created core key set. - keySetName - string, the name of the new key set - keySet - dictionary containing the new key set + keyset_name - string, the name of the new key set + keyset - dictionary containing the new key set """ - if not idleConf.userCfg['keys'].has_section(keySetName): - idleConf.userCfg['keys'].add_section(keySetName) - for event in keySet: - value = keySet[event] - idleConf.userCfg['keys'].SetOption(keySetName, event, value) + if not idleConf.userCfg['keys'].has_section(keyset_name): + idleConf.userCfg['keys'].add_section(keyset_name) + for event in keyset: + value = keyset[event] + idleConf.userCfg['keys'].SetOption(keyset_name, event, value) - def SaveNewTheme(self, themeName, theme): + def save_new_theme(self, theme_name, theme): """ save a newly created theme. - themeName - string, the name of the new theme + theme_name - string, the name of the new theme theme - dictionary containing the new theme """ - if not idleConf.userCfg['highlight'].has_section(themeName): - idleConf.userCfg['highlight'].add_section(themeName) + if not idleConf.userCfg['highlight'].has_section(theme_name): + idleConf.userCfg['highlight'].add_section(theme_name) for element in theme: value = theme[element] - idleConf.userCfg['highlight'].SetOption(themeName, element, value) + idleConf.userCfg['highlight'].SetOption(theme_name, element, value) - def SetUserValue(self, configType, section, item, value): - if idleConf.defaultCfg[configType].has_option(section, item): - if idleConf.defaultCfg[configType].Get(section, item) == value: + def set_user_value(self, config_type, section, item, value): + if idleConf.defaultCfg[config_type].has_option(section, item): + if idleConf.defaultCfg[config_type].Get(section, item) == value: #the setting equals a default setting, remove it from user cfg - return idleConf.userCfg[configType].RemoveOption(section, item) + return idleConf.userCfg[config_type].RemoveOption(section, item) #if we got here set the option - return idleConf.userCfg[configType].SetOption(section, item, value) + return idleConf.userCfg[config_type].SetOption(section, item, value) - def SaveAllChangedConfigs(self): + def save_all_changed_configs(self): "Save configuration changes to the user config file." idleConf.userCfg['main'].Save() - for configType in self.changedItems: - cfgTypeHasChanges = False - for section in self.changedItems[configType]: + for config_type in self.changed_items: + cfg_type_changed = False + for section in self.changed_items[config_type]: if section == 'HelpFiles': #this section gets completely replaced idleConf.userCfg['main'].remove_section('HelpFiles') - cfgTypeHasChanges = True - for item in self.changedItems[configType][section]: - value = self.changedItems[configType][section][item] - if self.SetUserValue(configType, section, item, value): - cfgTypeHasChanges = True - if cfgTypeHasChanges: - idleConf.userCfg[configType].Save() - for configType in ['keys', 'highlight']: + cfg_type_changed = True + for item in self.changed_items[config_type][section]: + value = self.changed_items[config_type][section][item] + if self.set_user_value(config_type, section, item, value): + cfg_type_changed = True + if cfg_type_changed: + idleConf.userCfg[config_type].Save() + for config_type in ['keys', 'highlight']: # save these even if unchanged! - idleConf.userCfg[configType].Save() - self.ResetChangedItems() #clear the changed items dict + idleConf.userCfg[config_type].Save() + self.reset_changed_items() #clear the changed items dict self.save_all_changed_extensions() # uses a different mechanism - def DeactivateCurrentConfig(self): + def deactivate_current_config(self): #Before a config is saved, some cleanup of current #config must be done - remove the previous keybindings - winInstances = self.parent.instance_dict.keys() - for instance in winInstances: + win_instances = self.parent.instance_dict.keys() + for instance in win_instances: instance.RemoveKeybindings() - def ActivateConfigChanges(self): + def activate_config_changes(self): "Dynamically apply configuration changes" - winInstances = self.parent.instance_dict.keys() - for instance in winInstances: + win_instances = self.parent.instance_dict.keys() + for instance in win_instances: instance.ResetColorizer() instance.ResetFont() instance.set_notabs_indentwidth() instance.ApplyKeybindings() instance.reset_help_menu_entries() - def Cancel(self): + def cancel(self): self.destroy() - def Ok(self): - self.Apply() + def ok(self): + self.apply() self.destroy() - def Apply(self): - self.DeactivateCurrentConfig() - self.SaveAllChangedConfigs() - self.ActivateConfigChanges() + def apply(self): + self.deactivate_current_config() + self.save_all_changed_configs() + self.activate_config_changes() - def Help(self): - page = self.tabPages._current_page + def help(self): + page = self.tab_pages._current_page view_text(self, title='Help for IDLE preferences', text=help_common+help_pages.get(page, '')) - def CreatePageExtensions(self): + def create_page_extensions(self): """Part of the config dialog used for configuring IDLE extensions. This code is generic - it works for any and all IDLE extensions. @@ -1237,7 +1242,7 @@ def CreatePageExtensions(self): which are boolean, and can be toggled with a True/False button. """ parent = self.parent - frame = self.tabPages.pages['Extensions'].frame + frame = self.tab_pages.pages['Extensions'].frame self.ext_defaultCfg = idleConf.defaultCfg['extensions'] self.ext_userCfg = idleConf.userCfg['extensions'] self.is_int = self.register(is_int) diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 3f94493733ba0b..4946f02306a786 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -21,14 +21,19 @@ 'extensions': config.IdleUserConfParser(''), } -# ConfigDialog.changedItems is a 3-level hierarchical dictionary of +# ConfigDialog.changed_items is a 3-level hierarchical dictionary of # pending changes that mirrors the multilevel user config dict. # For testing, record args in a list for comparison with expected. changes = [] +root = None +configure = None + + class TestDialog(ConfigDialog): - def AddChangedItem(self, *args): + def add_changed_item(self, *args): changes.append(args) + def setUpModule(): global root, configure idleConf.userCfg = testcfg @@ -49,40 +54,39 @@ def tearDownModule(): class FontTabTest(unittest.TestCase): - def setUp(self): changes.clear() def test_font(self): # Set values guaranteed not to be defaults. - dfont = idleConf.GetFont(root, 'main', 'EditorWindow') - dsize = str(dfont[1]) - dbold = dfont[2] == 'bold' - configure.fontName.set('Test Font') + default_font = idleConf.GetFont(root, 'main', 'EditorWindow') + default_size = str(default_font[1]) + default_bold = default_font[2] == 'bold' + configure.font_name.set('Test Font') expected = [ ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', dsize), - ('main', 'EditorWindow', 'font-bold', dbold)] + ('main', 'EditorWindow', 'font-size', default_size), + ('main', 'EditorWindow', 'font-bold', default_bold)] self.assertEqual(changes, expected) changes.clear() - configure.fontSize.set(20) + configure.font_size.set(20) expected = [ ('main', 'EditorWindow', 'font', 'Test Font'), ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', dbold)] + ('main', 'EditorWindow', 'font-bold', default_bold)] self.assertEqual(changes, expected) changes.clear() - configure.fontBold.set(not dbold) + configure.font_bold.set(not default_bold) expected = [ ('main', 'EditorWindow', 'font', 'Test Font'), ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', not dbold)] + ('main', 'EditorWindow', 'font-bold', not default_bold)] self.assertEqual(changes, expected) #def test_sample(self): pass # TODO def test_tabspace(self): - configure.spaceNum.set(6) + configure.space_num.set(6) self.assertEqual(changes, [('main', 'Indent', 'num-spaces', 6)]) @@ -106,19 +110,19 @@ def setUp(self): changes.clear() def test_startup(self): - configure.radioStartupEdit.invoke() + configure.radio_startup_edit.invoke() self.assertEqual(changes, [('main', 'General', 'editor-on-startup', 1)]) def test_autosave(self): - configure.radioSaveAuto.invoke() + configure.radio_save_auto.invoke() self.assertEqual(changes, [('main', 'General', 'autosave', 1)]) def test_editor_size(self): - configure.entryWinHeight.insert(0, '1') + configure.entry_win_height.insert(0, '1') self.assertEqual(changes, [('main', 'EditorWindow', 'height', '140')]) changes.clear() - configure.entryWinWidth.insert(0, '1') + configure.entry_win_width.insert(0, '1') self.assertEqual(changes, [('main', 'EditorWindow', 'width', '180')]) #def test_help_sources(self): pass # TODO diff --git a/Misc/NEWS.d/next/IDLE/2017-06-26-15-47-13.bpo-30728.qH4TGL.rst b/Misc/NEWS.d/next/IDLE/2017-06-26-15-47-13.bpo-30728.qH4TGL.rst new file mode 100644 index 00000000000000..014eff369de735 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-26-15-47-13.bpo-30728.qH4TGL.rst @@ -0,0 +1,4 @@ +Review and change idlelib.configdialog names. +Lowercase method and attribute names. +Replace 'colour' with 'color', expand overly cryptic names, delete unneeded underscores. +Replace ``import *`` with specific imports. Patches by Cheryl Sabella. From 8047f02a4b0db81cb023df1f5ce4cc1c42d17821 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Mon, 26 Jun 2017 23:09:45 -0400 Subject: [PATCH 0491/1345] [3.6] bpo-24813: IDLE: Add build bitness to About Idle title (GH-2380) (#2426) Patch by Cheryl Sabella. (cherry picked from commit 9a02ae3) --- Lib/idlelib/help_about.py | 18 ++++++++++++++---- Lib/idlelib/idle_test/test_help_about.py | 12 ++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index a2ac3166c34cf3..967f0342a101c2 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -2,7 +2,8 @@ """ import os -from platform import python_version +import sys +from platform import python_version, architecture from tkinter import Toplevel, Frame, Label, Button, PhotoImage from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW, E @@ -10,6 +11,14 @@ from idlelib import textview +def build_bits(): + "Return bits for platform." + if sys.platform == 'darwin': + return '64' if sys.maxsize > 2**32 else '32' + else: + return architecture()[0][:2] + + class AboutDialog(Toplevel): """Modal about dialog for idle @@ -28,11 +37,12 @@ def __init__(self, parent, title=None, _htest=False, _utest=False): self.geometry("+%d+%d" % ( parent.winfo_rootx()+30, parent.winfo_rooty()+(30 if not _htest else 100))) - self.bg = "#707070" - self.fg = "#ffffff" + self.bg = "#bbbbbb" + self.fg = "#000000" self.create_widgets() self.resizable(height=False, width=False) - self.title(title or f'About IDLE {python_version()}') + self.title(title or + f'About IDLE {python_version()} ({build_bits()} bit)') self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.ok) diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index be1fadd58f493b..08c38a515da10d 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -5,12 +5,15 @@ from test.support import requires, findfile from tkinter import Tk, TclError import unittest +from unittest import mock from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func from idlelib.help_about import AboutDialog as About +from idlelib import help_about from idlelib import textview import os.path -from platform import python_version +from platform import python_version, architecture + class LiveDialogTest(unittest.TestCase): """Simulate user clicking buttons other than [Close]. @@ -31,6 +34,9 @@ def tearDownClass(cls): cls.root.destroy() del cls.root + def test_build_bits(self): + self.assertIn(help_about.build_bits(), ('32', '64')) + def test_dialog_title(self): """Test about dialog title""" self.assertEqual(self.dialog.title(), 'About IDLE') @@ -99,7 +105,9 @@ def tearDownClass(cls): def test_dialog_title(self): """Test about dialog title""" - self.assertEqual(self.dialog.title(), f'About IDLE {python_version()}') + self.assertEqual(self.dialog.title(), + f'About IDLE {python_version()}' + f' ({help_about.build_bits()} bit)') class CloseTest(unittest.TestCase): From 2d348f7a723db839aa18ce8213b8663ccb0a3d35 Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Mon, 26 Jun 2017 22:59:25 -0600 Subject: [PATCH 0492/1345] [3.6] bpo-30769: Fix reference leak introduced in 77703942c59 (GH-2416) (#2425) New error condition paths were introduced, which did not decrement `key2` and `val2` objects. Therefore, decrement references before jumping to the error label. Signed-off-by: Eric N. Vander Weele (cherry picked from commit a7874c73c0c729bbec2fd4b077bd0eec276cfff4) --- Misc/ACKS | 1 + Modules/posixmodule.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Misc/ACKS b/Misc/ACKS index c68f9eb9f18867..b8ae14be6914fc 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1590,6 +1590,7 @@ Andi Vajda Case Van Horsen John Mark Vandenberg Kyle VanderBeek +Eric N. Vander Weele Andrew Vant Atul Varma Dmitry Vasiliev diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 033789099c3a84..bf8254311e02dd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4823,6 +4823,8 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) PyUnicode_FindChar(key2, '=', 1, PyUnicode_GET_LENGTH(key2), 1) != -1) { PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + Py_DECREF(key2); + Py_DECREF(val2); goto error; } keyval = PyUnicode_FromFormat("%U=%U", key2, val2); @@ -4837,6 +4839,8 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) strchr(PyBytes_AS_STRING(key2) + 1, '=') != NULL) { PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + Py_DECREF(key2); + Py_DECREF(val2); goto error; } keyval = PyBytes_FromFormat("%s=%s", PyBytes_AS_STRING(key2), From 8bdc3bd3d66fefdc07d32bd19c41c6f902f16111 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 27 Jun 2017 01:53:40 -0400 Subject: [PATCH 0493/1345] [3.6] bpo-29910: IDLE no longer deletes a character after commenting out a region (GH-825) (#2429) This happened because shortcut has a class binding and 'break' was not returned. Fix other potential conflicts between IDLE and default key bindings. * Add news item * Update NEWS (cherry picked from commit 213ce12) --- Lib/idlelib/autocomplete.py | 1 + Lib/idlelib/calltip_w.py | 7 +++-- Lib/idlelib/calltips.py | 1 + Lib/idlelib/codecontext.py | 1 + Lib/idlelib/editor.py | 30 +++++++++++++++---- Lib/idlelib/idle_test/test_parenmatch.py | 8 ++--- Lib/idlelib/parenmatch.py | 10 ++++--- Lib/idlelib/runscript.py | 1 + Lib/idlelib/scrolledlist.py | 1 + Lib/idlelib/zoomheight.py | 1 + .../2017-06-26-22-45-27.bpo-29910.mqHh7u.rst | 3 ++ 11 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-26-22-45-27.bpo-29910.mqHh7u.rst diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index 1e44fa5bc66e8a..cd212ccb143a3c 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -60,6 +60,7 @@ def force_open_completions_event(self, event): if a function call is needed. """ self.open_completions(True, False, True) + return "break" def try_open_completions_event(self, event): """Happens when it would be nice to open a completion list, but not diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py index c7361d18d66efd..bf967f40b6ab1c 100644 --- a/Lib/idlelib/calltip_w.py +++ b/Lib/idlelib/calltip_w.py @@ -89,24 +89,27 @@ def checkhide_event(self, event=None): # If the event was triggered by the same event that unbinded # this function, the function will be called nevertheless, # so do nothing in this case. - return + return None curline, curcol = map(int, self.widget.index("insert").split('.')) if curline < self.parenline or \ (curline == self.parenline and curcol <= self.parencol) or \ self.widget.compare("insert", ">", MARK_RIGHT): self.hidetip() + return "break" else: self.position_window() if self.checkhide_after_id is not None: self.widget.after_cancel(self.checkhide_after_id) self.checkhide_after_id = \ self.widget.after(CHECKHIDE_TIME, self.checkhide_event) + return None def hide_event(self, event): if not self.tipwindow: # See the explanation in checkhide_event. - return + return None self.hidetip() + return "break" def hidetip(self): if not self.tipwindow: diff --git a/Lib/idlelib/calltips.py b/Lib/idlelib/calltips.py index 4c5aea2acbf40b..a8a3abe6c6982a 100644 --- a/Lib/idlelib/calltips.py +++ b/Lib/idlelib/calltips.py @@ -47,6 +47,7 @@ def _remove_calltip_window(self, event=None): def force_open_calltip_event(self, event): "The user selected the menu entry or hotkey, open the tip." self.open_calltip(True) + return "break" def try_open_calltip_event(self, event): """Happens when it would be nice to open a CallTip, but not really diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index f25e1b33a086e8..09dc078d63f191 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -89,6 +89,7 @@ def toggle_code_context_event(self, event=None): idleConf.SetOption("extensions", "CodeContext", "visible", str(self.label is not None)) idleConf.SaveUserCfgFiles() + return "break" def get_line_info(self, linenum): """Get the line indent value, text, and any block start keyword diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 13b4a51c18dff3..5b64ef4630e737 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1,4 +1,3 @@ -import importlib import importlib.abc import importlib.util import os @@ -147,7 +146,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) - text.bind("<>", self.open_module) + text.bind("<>", self.open_module_event) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) text.bind("<>", self.remove_selection) @@ -294,7 +293,7 @@ def new_callback(self, event): def home_callback(self, event): if (event.state & 4) != 0 and event.keysym == "Home": # state&4==Control. If , use the Tk binding. - return + return None if self.text.index("iomark") and \ self.text.compare("iomark", "<=", "insert lineend") and \ self.text.compare("insert linestart", "<=", "iomark"): @@ -423,6 +422,7 @@ def right_menu_event(self, event): rmenu.tk_popup(event.x_root, event.y_root) if iswin: self.text.config(cursor="ibeam") + return "break" rmenu_specs = [ # ("Label", "<>", "statefuncname"), ... @@ -464,11 +464,13 @@ def about_dialog(self, event=None): "Handle Help 'About IDLE' event." # Synchronize with macosx.overrideRootMenu.about_dialog. help_about.AboutDialog(self.top) + return "break" def config_dialog(self, event=None): "Handle Options 'Configure IDLE' event." # Synchronize with macosx.overrideRootMenu.config_dialog. configdialog.ConfigDialog(self.top,'Settings') + return "break" def help_dialog(self, event=None): "Handle Help 'IDLE Help' event." @@ -478,6 +480,7 @@ def help_dialog(self, event=None): else: parent = self.top help.show_idlehelp(parent) + return "break" def python_docs(self, event=None): if sys.platform[:3] == 'win': @@ -497,7 +500,7 @@ def cut(self,event): def copy(self,event): if not self.text.tag_ranges("sel"): # There is no selection, so do nothing and maybe interrupt. - return + return None self.text.event_generate("<>") return "break" @@ -515,6 +518,7 @@ def select_all(self, event=None): def remove_selection(self, event=None): self.text.tag_remove("sel", "1.0", "end") self.text.see("insert") + return "break" def move_at_edge_if_selection(self, edge_index): """Cursor move begins at start or end of selection @@ -575,8 +579,9 @@ def goto_line_event(self, event): return "break" text.mark_set("insert", "%d.0" % lineno) text.see("insert") + return "break" - def open_module(self, event=None): + def open_module(self): """Get module name from user and open it. Return module path or None for calls by open_class_browser @@ -600,21 +605,27 @@ def open_module(self, event=None): self.io.loadfile(file_path) return file_path + def open_module_event(self, event): + self.open_module() + return "break" + def open_class_browser(self, event=None): filename = self.io.filename if not (self.__class__.__name__ == 'PyShellEditorWindow' and filename): filename = self.open_module() if filename is None: - return + return "break" head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import browser browser.ClassBrowser(self.flist, base, [head]) + return "break" def open_path_browser(self, event=None): from idlelib import pathbrowser pathbrowser.PathBrowser(self.flist) + return "break" def open_turtle_demo(self, event = None): import subprocess @@ -623,6 +634,7 @@ def open_turtle_demo(self, event = None): '-c', 'from turtledemo.__main__ import main; main()'] subprocess.Popen(cmd, shell=False) + return "break" def gotoline(self, lineno): if lineno is not None and lineno > 0: @@ -879,6 +891,7 @@ def long_title(self): def center_insert_event(self, event): self.center() + return "break" def center(self, mark="insert"): text = self.text @@ -910,6 +923,7 @@ def get_geometry(self): def close_event(self, event): self.close() + return "break" def maybesave(self): if self.io: @@ -1357,6 +1371,7 @@ def comment_region_event(self, event): line = lines[pos] lines[pos] = '##' + line self.set_region(head, tail, chars, lines) + return "break" def uncomment_region_event(self, event): head, tail, chars, lines = self.get_region() @@ -1370,6 +1385,7 @@ def uncomment_region_event(self, event): line = line[1:] lines[pos] = line self.set_region(head, tail, chars, lines) + return "break" def tabify_region_event(self, event): head, tail, chars, lines = self.get_region() @@ -1382,6 +1398,7 @@ def tabify_region_event(self, event): ntabs, nspaces = divmod(effective, tabwidth) lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:] self.set_region(head, tail, chars, lines) + return "break" def untabify_region_event(self, event): head, tail, chars, lines = self.get_region() @@ -1390,6 +1407,7 @@ def untabify_region_event(self, event): for pos in range(len(lines)): lines[pos] = lines[pos].expandtabs(tabwidth) self.set_region(head, tail, chars, lines) + return "break" def toggle_tabs_event(self, event): if self.askyesno( diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index b5fd0d1c69e4f1..cbec35018b990f 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -95,14 +95,14 @@ def test_paren_corner(self): pm = self.get_parenmatch() text.insert('insert', '# this is a commen)') - self.assertIsNone(pm.paren_closed_event('event')) + pm.paren_closed_event('event') text.insert('insert', '\ndef') - self.assertIsNone(pm.flash_paren_event('event')) - self.assertIsNone(pm.paren_closed_event('event')) + pm.flash_paren_event('event') + pm.paren_closed_event('event') text.insert('insert', ' a, *arg)') - self.assertIsNone(pm.paren_closed_event('event')) + pm.paren_closed_event('event') def test_handle_restore_timer(self): pm = self.get_parenmatch() diff --git a/Lib/idlelib/parenmatch.py b/Lib/idlelib/parenmatch.py index ccec708f31f436..dcec34c18f9287 100644 --- a/Lib/idlelib/parenmatch.py +++ b/Lib/idlelib/parenmatch.py @@ -94,26 +94,28 @@ def flash_paren_event(self, event): .get_surrounding_brackets()) if indices is None: self.bell() - return + return "break" self.activate_restore() self.create_tag(indices) self.set_timeout_last() + return "break" def paren_closed_event(self, event): # If it was a shortcut and not really a closing paren, quit. closer = self.text.get("insert-1c") if closer not in _openers: - return + return "break" hp = HyperParser(self.editwin, "insert-1c") if not hp.is_in_code(): - return + return "break" indices = hp.get_surrounding_brackets(_openers[closer], True) if indices is None: self.bell() - return + return "break" self.activate_restore() self.create_tag(indices) self.set_timeout() + return "break" def restore_event(self, event=None): self.text.tag_delete("paren") diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py index 79d86adabc85a0..3355f17d90bdb7 100644 --- a/Lib/idlelib/runscript.py +++ b/Lib/idlelib/runscript.py @@ -63,6 +63,7 @@ def check_module_event(self, event): return 'break' if not self.tabnanny(filename): return 'break' + return "break" def tabnanny(self, filename): # XXX: tabnanny should work on binary files as well diff --git a/Lib/idlelib/scrolledlist.py b/Lib/idlelib/scrolledlist.py index cc08c26e5eefaa..cdf658404ab643 100644 --- a/Lib/idlelib/scrolledlist.py +++ b/Lib/idlelib/scrolledlist.py @@ -76,6 +76,7 @@ def popup_event(self, event): index = self.listbox.index("active") self.select(index) menu.tk_popup(event.x_root, event.y_root) + return "break" def make_menu(self): menu = Menu(self.listbox, tearoff=0) diff --git a/Lib/idlelib/zoomheight.py b/Lib/idlelib/zoomheight.py index aa4a427eab737f..d01c9e964aa60f 100644 --- a/Lib/idlelib/zoomheight.py +++ b/Lib/idlelib/zoomheight.py @@ -20,6 +20,7 @@ def __init__(self, editwin): def zoom_height_event(self, event): top = self.editwin.top zoom_height(top) + return "break" def zoom_height(top): diff --git a/Misc/NEWS.d/next/IDLE/2017-06-26-22-45-27.bpo-29910.mqHh7u.rst b/Misc/NEWS.d/next/IDLE/2017-06-26-22-45-27.bpo-29910.mqHh7u.rst new file mode 100644 index 00000000000000..5157f8bb923c07 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-26-22-45-27.bpo-29910.mqHh7u.rst @@ -0,0 +1,3 @@ +IDLE no longer deletes a character after commenting out a region by a key +shortcut. Add ``return 'break'`` for this and other potential conflicts +between IDLE and default key bindings. From 93b88e995373c48713c9f7d4b32fe1d0166709e5 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 27 Jun 2017 01:58:18 -0400 Subject: [PATCH 0494/1345] [3.6] bpo-21519: IDLE basic custom key entry better detects duplicates. (GH-2428) (#2433) Original patch by Saimadhav Heblikar. (cherry picked from commit 44913e5) --- Lib/idlelib/config_key.py | 6 +++--- Lib/idlelib/idle_test/test_config_key.py | 14 ++++++++++---- .../IDLE/2017-06-27-00-29-56.bpo-21519.fTj9T0.rst | 2 ++ 3 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-27-00-29-56.bpo-21519.fTj9T0.rst diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 8d575ec43ca833..fc6e341acad0ee 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -250,10 +250,10 @@ def KeysOK(self, keys): ''' finalKey = self.listKeysFinal.get(ANCHOR) modifiers = self.GetModifiers() - # create a key sequence list for overlap check: - keySequence = keys.split() keysOK = False title = self.keyerror_title + key_sequences = [key for keylist in self.currentKeySequences + for key in keylist] if not keys.endswith('>'): self.showerror(title, parent=self, message='Missing the final Key') @@ -267,7 +267,7 @@ def KeysOK(self, keys): msg = 'The shift modifier by itself may not be used with'\ ' this key symbol.' self.showerror(title=title, parent=self, message=msg) - elif keySequence in self.currentKeySequences: + elif keys in key_sequences: msg = 'This key combination is already in use.' self.showerror(title=title, parent=self, message=msg) else: diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index a4227ed0552b7f..9074e23aab35d1 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -28,8 +28,9 @@ def setUpClass(cls): requires('gui') cls.root = Tk() cls.root.withdraw() + keylist = [[''], ['', '']] cls.dialog = cls.Validator( - cls.root, 'Title', '<>', [['']], _utest=True) + cls.root, 'Title', '<>', keylist, _utest=True) @classmethod def tearDownClass(cls): @@ -78,10 +79,15 @@ def test_keys_shift_bad(self): self.dialog.GetModifiers.result = [] def test_keys_dup(self): - self.dialog.listKeysFinal.get.result = 'F12' + for mods, final, seq in (([], 'F12', ''), + (['Control'], 'x', ''), + (['Control'], 'X', '')): + with self.subTest(m=mods, f=final, s=seq): + self.dialog.listKeysFinal.get.result = final + self.dialog.GetModifiers.result = mods + self.assertFalse(self.dialog.KeysOK(seq)) + self.assertIn('already in use', self.dialog.showerror.message) self.dialog.GetModifiers.result = [] - self.assertFalse(self.dialog.KeysOK('')) - self.assertIn('already in use', self.dialog.showerror.message) def test_bind_ok(self): self.assertTrue(self.dialog.bind_ok('')) diff --git a/Misc/NEWS.d/next/IDLE/2017-06-27-00-29-56.bpo-21519.fTj9T0.rst b/Misc/NEWS.d/next/IDLE/2017-06-27-00-29-56.bpo-21519.fTj9T0.rst new file mode 100644 index 00000000000000..b224f6b892b443 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-27-00-29-56.bpo-21519.fTj9T0.rst @@ -0,0 +1,2 @@ +IDLE's basic custom key entry dialog now detects duplicates properly. +Original patch by Saimadhav Heblikar. From 39e501a2913eff047f7a644e2b20eb4278f97d50 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 27 Jun 2017 03:01:58 -0400 Subject: [PATCH 0495/1345] [3.6] bpo-30674: IDLE: add docstrings to grep.py (GH-2213) (#2434) Patch by Cheryl Sabella (cherry picked from commit 65474b9) --- Lib/idlelib/grep.py | 66 +++++++++++++++---- .../2017-06-27-01-40-34.bpo-30674.ppK_q8.rst | 1 + 2 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-27-01-40-34.bpo-30674.ppK_q8.rst diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 64ba28d94a5c14..c55c48cf2d69bc 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -1,3 +1,8 @@ +"""Grep dialog for Find in Files functionality. + + Inherits from SearchDialogBase for GUI and uses searchengine + to prepare search pattern. +""" import fnmatch import os import sys @@ -11,7 +16,17 @@ # Importing OutputWindow here fails due to import loop # EditorWindow -> GrepDialop -> OutputWindow -> EditorWindow + def grep(text, io=None, flist=None): + """Create or find singleton GrepDialog instance. + + Args: + text: Text widget that contains the selected text for + default search phrase. + io: iomenu.IOBinding instance with default path to search. + flist: filelist.FileList instance for OutputWindow parent. + """ + root = text._root() engine = searchengine.get(root) if not hasattr(engine, "_grepdialog"): @@ -20,19 +35,32 @@ def grep(text, io=None, flist=None): searchphrase = text.get("sel.first", "sel.last") dialog.open(text, searchphrase, io) + class GrepDialog(SearchDialogBase): + "Dialog for searching multiple files." title = "Find in Files Dialog" icon = "Grep" needwrapbutton = 0 def __init__(self, root, engine, flist): + """Create search dialog for searching for a phrase in the file system. + + Uses SearchDialogBase as the basis for the GUI and a + searchengine instance to prepare the search. + + Attributes: + globvar: Value of Text Entry widget for path to search. + recvar: Boolean value of Checkbutton widget + for traversing through subdirectories. + """ SearchDialogBase.__init__(self, root, engine) self.flist = flist self.globvar = StringVar(root) self.recvar = BooleanVar(root) def open(self, text, searchphrase, io=None): + "Make dialog visible on top of others and ready to use." SearchDialogBase.open(self, text, searchphrase) if io: path = io.filename or "" @@ -45,20 +73,30 @@ def open(self, text, searchphrase, io=None): self.globvar.set(os.path.join(dir, "*" + tail)) def create_entries(self): + "Create base entry widgets and add widget for search path." SearchDialogBase.create_entries(self) self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): + "Add check button to recurse down subdirectories." btn = Checkbutton( self.make_frame()[0], variable=self.recvar, text="Recurse down subdirectories") btn.pack(side="top", fill="both") def create_command_buttons(self): + "Create base command buttons and add button for search." SearchDialogBase.create_command_buttons(self) self.make_button("Search Files", self.default_command, 1) def default_command(self, event=None): + """Grep for search pattern in file path. The default command is bound + to . + + If entry values are populated, set OutputWindow as stdout + and perform search. The search dialog is closed automatically + when the search begins. + """ prog = self.engine.getprog() if not prog: return @@ -75,12 +113,19 @@ def default_command(self, event=None): sys.stdout = save def grep_it(self, prog, path): + """Search for prog within the lines of the files in path. + + For the each file in the path directory, open the file and + search each line for the matching pattern. If the pattern is + found, write the file and line information to stdout (which + is an OutputWindow). + """ dir, base = os.path.split(path) list = self.findfiles(dir, base, self.recvar.get()) list.sort() self.close() pat = self.engine.getpat() - print("Searching %r in %s ..." % (pat, path)) + print(f"Searching {pat!r} in {path} ...") hits = 0 try: for fn in list: @@ -90,20 +135,22 @@ def grep_it(self, prog, path): if line[-1:] == '\n': line = line[:-1] if prog.search(line): - sys.stdout.write("%s: %s: %s\n" % - (fn, lineno, line)) + sys.stdout.write(f"{fn}: {lineno}: {line}\n") hits += 1 except OSError as msg: print(msg) - print(("Hits found: %s\n" - "(Hint: right-click to open locations.)" - % hits) if hits else "No hits.") + print(f"Hits found: {hits}\n(Hint: right-click to open locations.)" + if hits else "No hits.") except AttributeError: # Tk window has been closed, OutputWindow.text = None, # so in OW.write, OW.text.insert fails. pass def findfiles(self, dir, base, rec): + """Return list of files in the dir that match the base pattern. + + If rec is True, recursively iterate through subdirectories. + """ try: names = os.listdir(dir or os.curdir) except OSError as msg: @@ -123,11 +170,6 @@ def findfiles(self, dir, base, rec): list.extend(self.findfiles(subdir, base, rec)) return list - def close(self, event=None): - if self.top: - self.top.grab_release() - self.top.withdraw() - def _grep_dialog(parent): # htest # from tkinter import Toplevel, Text, SEL, END @@ -136,7 +178,7 @@ def _grep_dialog(parent): # htest # top = Toplevel(parent) top.title("Test GrepDialog") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("+%d+%d" % (x, y + 175)) + top.geometry(f"+{x}+{y + 175}") flist = PyShellFileList(top) text = Text(top, height=5) diff --git a/Misc/NEWS.d/next/IDLE/2017-06-27-01-40-34.bpo-30674.ppK_q8.rst b/Misc/NEWS.d/next/IDLE/2017-06-27-01-40-34.bpo-30674.ppK_q8.rst new file mode 100644 index 00000000000000..4b718fef017799 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-27-01-40-34.bpo-30674.ppK_q8.rst @@ -0,0 +1 @@ +IDLE: add docstrings to grep module. Patch by Cheryl Sabella From 35d2ca2b94a6ff29e763ddb7727166f0592edfa2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 27 Jun 2017 16:04:15 +0200 Subject: [PATCH 0496/1345] [3.6] bpo-30523, bpo-30764, bpo-30776: Sync regrtest from master (#2441) * bpo-30523: regrtest --list-cases --match (#2401) * regrtest --list-cases now supports --match and --match-file options. Example: ./python -m test --list-cases -m FileTests test_os * --list-cases now also sets support.verbose to False to prevent messages to stdout when loading test modules. * Add support._match_test() private function. (cherry picked from commit ace56d583664f855d89d1219ece7c21c2fddcf30) * bpo-30764: regrtest: add --fail-env-changed option (#2402) * bpo-30764: regrtest: change exit code on failure * Exit code 2 if failed tests ("bad") * Exit code 3 if interrupted * bpo-30764: regrtest: add --fail-env-changed option If the option is set, mark a test as failed if it alters the environment, for example if it creates a file without removing it. (cherry picked from commit 63f54c68936d648c70ca411661e4208329edcf26) * bpo-30776: reduce regrtest -R false positives (#2422) * Change the regrtest --huntrleaks checker to decide if a test file leaks or not. Require that each run leaks at least 1 reference. * Warmup runs are now completely ignored: ignored in the checker test and not used anymore to compute the sum. * Add an unit test for a reference leak. Example of reference differences previously considered a failure (leak) and now considered as success (success, no leak): [3, 0, 0] [0, 1, 0] [8, -8, 1] (cherry picked from commit 48b5c422ffb03affb00c184b9a99e5537be92732) --- Lib/test/libregrtest/cmdline.py | 3 + Lib/test/libregrtest/main.py | 16 ++++- Lib/test/libregrtest/refleak.py | 25 ++++++- Lib/test/support/__init__.py | 32 +++++---- Lib/test/test_regrtest.py | 113 +++++++++++++++++++++++--------- 5 files changed, 140 insertions(+), 49 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index bf64062ef964b8..2315cd59b4e679 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -255,6 +255,9 @@ def _create_parser(): ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization training') + group.add_argument('--fail-env-changed', action='store_true', + help='if a test file alters the environment, mark ' + 'the test as failed') return parser diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 527de177792ce5..571eb6112520eb 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -256,9 +256,13 @@ def _list_cases(self, suite): if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - print(test.id()) + if support._match_test(test): + print(test.id()) def list_cases(self): + support.verbose = False + support.match_tests = self.ns.match_tests + for test in self.selected: abstest = get_abs_module(self.ns, test) try: @@ -474,6 +478,8 @@ def finalize(self): result = "FAILURE" elif self.interrupted: result = "INTERRUPTED" + elif self.environment_changed and self.ns.fail_env_changed: + result = "ENV CHANGED" else: result = "SUCCESS" print("Tests result: %s" % result) @@ -534,7 +540,13 @@ def _main(self, tests, kwargs): self.rerun_failed_tests() self.finalize() - sys.exit(len(self.bad) > 0 or self.interrupted) + if self.bad: + sys.exit(2) + if self.interrupted: + sys.exit(130) + if self.ns.fail_env_changed and self.environment_changed: + sys.exit(3) + sys.exit(0) def removepy(names): diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index c69fc0f147706b..8e18d75b217e6e 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -93,9 +93,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks): rc_before = rc_after fd_before = fd_after print(file=sys.stderr) + # These checkers return False on success, True on failure def check_rc_deltas(deltas): - return any(deltas) + # bpo-30776: Try to ignore false positives: + # + # [3, 0, 0] + # [0, 1, 0] + # [8, -8, 1] + # + # Expected leaks: + # + # [5, 5, 6] + # [10, 1, 1] + return all(delta >= 1 for delta in deltas) + def check_alloc_deltas(deltas): # At least 1/3rd of 0s if 3 * deltas.count(0) < len(deltas): @@ -104,14 +116,21 @@ def check_alloc_deltas(deltas): if not set(deltas) <= {1,0,-1}: return True return False + + def check_fd_deltas(deltas): + return any(deltas) + failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), (alloc_deltas, 'memory blocks', check_alloc_deltas), - (fd_deltas, 'file descriptors', check_rc_deltas)]: + (fd_deltas, 'file descriptors', check_fd_deltas) + ]: + # ignore warmup runs + deltas = deltas[nwarmup:] if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) + test, deltas, item_name, sum(deltas)) print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index be926ea6c71d2a..677743bb389802 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1898,6 +1898,23 @@ def _run_suite(suite): raise TestFailed(err) +def _match_test(test): + global match_tests + + if match_tests is None: + return True + test_id = test.id() + + for match_test in match_tests: + if fnmatch.fnmatchcase(test_id, match_test): + return True + + for name in test_id.split("."): + if fnmatch.fnmatchcase(name, match_test): + return True + return False + + def run_unittest(*classes): """Run tests from unittest.TestCase-derived classes.""" valid_types = (unittest.TestSuite, unittest.TestCase) @@ -1912,20 +1929,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - def case_pred(test): - if match_tests is None: - return True - test_id = test.id() - - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): - return True - return False - _filter_suite(suite, case_pred) + _filter_suite(suite, _match_test) _run_suite(suite) #======================================================================= diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 5c6154a347d332..a544f880d2be01 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -377,19 +377,19 @@ def parse_executed_tests(self, output): return list(match.group(1) for match in parser) def check_executed_tests(self, output, tests, skipped=(), failed=(), - omitted=(), randomize=False, interrupted=False): + env_changed=(), omitted=(), + randomize=False, interrupted=False, + fail_env_changed=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] if isinstance(failed, str): failed = [failed] + if isinstance(env_changed, str): + env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - ntest = len(tests) - nskipped = len(skipped) - nfailed = len(failed) - nomitted = len(omitted) executed = self.parse_executed_tests(output) if randomize: @@ -415,11 +415,17 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) + if env_changed: + regex = list_regex('%s test%s altered the execution environment', + env_changed) + self.check_line(output, regex) + if omitted: regex = list_regex('%s test%s omitted', omitted) self.check_line(output, regex) - good = ntest - nskipped - nfailed - nomitted + good = (len(tests) - len(skipped) - len(failed) + - len(omitted) - len(env_changed)) if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -429,10 +435,12 @@ def list_regex(line_format, tests): if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') - if nfailed: + if failed: result = 'FAILURE' elif interrupted: result = 'INTERRUPTED' + elif fail_env_changed and env_changed: + result = 'ENV CHANGED' else: result = 'SUCCESS' self.check_line(output, 'Tests result: %s' % result) @@ -604,7 +612,7 @@ def test_failing(self): test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] - output = self.run_tests(*tests, exitcode=1) + output = self.run_tests(*tests, exitcode=2) self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): @@ -703,7 +711,7 @@ def test_fromfile(self): def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) - output = self.run_tests(test, exitcode=1) + output = self.run_tests(test, exitcode=130) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -732,7 +740,7 @@ def test_slow_interrupted(self): args = ("--slowest", "-j2", test) else: args = ("--slowest", test) - output = self.run_tests(*args, exitcode=1) + output = self.run_tests(*args, exitcode=130) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -772,9 +780,43 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) - output = self.run_tests('--forever', test, exitcode=1) + output = self.run_tests('--forever', test, exitcode=2) self.check_executed_tests(output, [test]*3, failed=test) + def check_leak(self, code, what): + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=2, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) + self.assertIn(line2, output) + + with open(filename) as fp: + reflog = fp.read() + self.assertIn(line2, reflog) + + @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks(self): + # test --huntrleaks + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test_leak(self): + GLOBAL_LIST.append(object()) + """) + self.check_leak(code, 'references') + @unittest.skipUnless(Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak @@ -799,24 +841,7 @@ def test_leak(self): fd = os.open(__file__, os.O_RDONLY) # bug: never cloes the file descriptor """) - test = self.create_test('huntrleaks', code=code) - - filename = 'reflog.txt' - self.addCleanup(support.unlink, filename) - output = self.run_tests('--huntrleaks', '3:3:', test, - exitcode=1, - stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test) - - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) - - line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test - self.assertIn(line2, output) - - with open(filename) as fp: - reflog = fp.read() - self.assertIn(line2, reflog) + self.check_leak(code, 'file descriptors') def test_list_tests(self): # test --list-tests @@ -837,11 +862,20 @@ def test_method2(self): pass """) testname = self.create_test(code=code) + + # Test --list-cases all_methods = ['%s.Tests.test_method1' % testname, '%s.Tests.test_method2' % testname] output = self.run_tests('--list-cases', testname) self.assertEqual(output.splitlines(), all_methods) + # Test --list-cases with --match + all_methods = ['%s.Tests.test_method1' % testname] + output = self.run_tests('--list-cases', + '-m', 'test_method1', + testname) + self.assertEqual(output.splitlines(), all_methods) + def test_crashed(self): # Any code which causes a crash code = 'import faulthandler; faulthandler._sigsegv()' @@ -849,7 +883,7 @@ def test_crashed(self): ok_test = self.create_test(name="ok") tests = [crash_test, ok_test] - output = self.run_tests("-j2", *tests, exitcode=1) + output = self.run_tests("-j2", *tests, exitcode=2) self.check_executed_tests(output, tests, failed=crash_test, randomize=True) @@ -898,6 +932,25 @@ def test_method4(self): subset = ['test_method1', 'test_method3'] self.assertEqual(methods, subset) + def test_env_changed(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_env_changed(self): + open("env_changed", "w").close() + """) + testname = self.create_test(code=code) + + # don't fail by default + output = self.run_tests(testname) + self.check_executed_tests(output, [testname], env_changed=testname) + + # fail with --fail-env-changed + output = self.run_tests("--fail-env-changed", testname, exitcode=3) + self.check_executed_tests(output, [testname], env_changed=testname, + fail_env_changed=True) + if __name__ == '__main__': unittest.main() From 0edffa3073b551ffeca34952529e7b292f1bd350 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 27 Jun 2017 21:08:58 +0300 Subject: [PATCH 0497/1345] [3.6] bpo-30708: Check for null characters in PyUnicode_AsWideCharString(). (GH-2285) (#2443) Raise a ValueError if the second argument is NULL and the wchar_t\* string contains null characters.. (cherry picked from commit e613e6add5f07ff6aad5802924596b631b707d2a) --- Include/unicodeobject.h | 6 ++++++ Lib/ctypes/test/test_slicing.py | 4 ++-- Modules/_ctypes/callproc.c | 2 +- Modules/_ctypes/cfield.c | 2 +- Modules/_cursesmodule.c | 2 +- Modules/_io/winconsoleio.c | 11 ++--------- Modules/_localemodule.c | 4 ++-- Modules/_tkinter.c | 2 +- Modules/overlapped.c | 2 +- Modules/timemodule.c | 2 +- Objects/unicodeobject.c | 31 +++++++++++++++++++++++++++++++ PC/winsound.c | 2 +- 12 files changed, 50 insertions(+), 20 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 5b877185f258d3..cec2b7f4b07e71 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1063,6 +1063,12 @@ PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString( ); #ifndef Py_LIMITED_API +/* Similar to PyUnicode_AsWideCharString(unicode, NULL), but check if + the string contains null characters. */ +PyAPI_FUNC(wchar_t*) _PyUnicode_AsWideCharString( + PyObject *unicode /* Unicode object */ + ); + PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); #endif diff --git a/Lib/ctypes/test/test_slicing.py b/Lib/ctypes/test/test_slicing.py index 240dc0cdda2462..a3932f176728af 100644 --- a/Lib/ctypes/test/test_slicing.py +++ b/Lib/ctypes/test/test_slicing.py @@ -134,7 +134,7 @@ def test_wchar_ptr(self): dll.my_wcsdup.restype = POINTER(c_wchar) dll.my_wcsdup.argtypes = POINTER(c_wchar), dll.my_free.restype = None - res = dll.my_wcsdup(s) + res = dll.my_wcsdup(s[:-1]) self.assertEqual(res[:len(s)], s) self.assertEqual(res[:len(s):], s) self.assertEqual(res[len(s)-1:-1:-1], s[::-1]) @@ -153,7 +153,7 @@ def test_wchar_ptr(self): dll.my_wcsdup.restype = POINTER(c_long) else: self.skipTest('Pointers to c_wchar are not supported') - res = dll.my_wcsdup(s) + res = dll.my_wcsdup(s[:-1]) tmpl = list(range(ord("a"), ord("z")+1)) self.assertEqual(res[:len(s)-1], tmpl) self.assertEqual(res[:len(s)-1:], tmpl) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 7d542fb50d8767..358f287bb7cd45 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -668,7 +668,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) #ifdef CTYPES_UNICODE if (PyUnicode_Check(obj)) { pa->ffi_type = &ffi_type_pointer; - pa->value.p = PyUnicode_AsWideCharString(obj, NULL); + pa->value.p = _PyUnicode_AsWideCharString(obj); if (pa->value.p == NULL) return -1; pa->keep = PyCapsule_New(pa->value.p, CTYPES_CAPSULE_NAME_PYMEM, pymem_destructor); diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a43585f1b83f13..113a815db151b2 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -1372,7 +1372,7 @@ Z_set(void *ptr, PyObject *value, Py_ssize_t size) /* We must create a wchar_t* buffer from the unicode object, and keep it alive */ - buffer = PyUnicode_AsWideCharString(value, NULL); + buffer = _PyUnicode_AsWideCharString(value); if (!buffer) return NULL; keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d88d06e78220ec..41b831e58f35a6 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -345,7 +345,7 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, if (PyUnicode_Check(obj)) { #ifdef HAVE_NCURSESW assert (wstr != NULL); - *wstr = PyUnicode_AsWideCharString(obj, NULL); + *wstr = _PyUnicode_AsWideCharString(obj); if (*wstr == NULL) return 0; return 2; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 94dfe3e4c63146..346c38637f9e1b 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -79,7 +79,7 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { PyErr_Clear(); return '\0'; } - decoded_wstr = PyUnicode_AsWideCharString(decoded, NULL); + decoded_wstr = _PyUnicode_AsWideCharString(decoded); Py_CLEAR(decoded); if (!decoded_wstr) { PyErr_Clear(); @@ -311,8 +311,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, if (!d) return -1; - Py_ssize_t length; - name = PyUnicode_AsWideCharString(decodedname, &length); + name = _PyUnicode_AsWideCharString(decodedname); console_type = _PyIO_get_console_type(decodedname); Py_CLEAR(decodedname); if (name == NULL) @@ -322,12 +321,6 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, "Cannot open non-console file"); return -1; } - - if (wcslen(name) != length) { - PyMem_Free(name); - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return -1; - } } s = mode; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index feb38029008e30..0c7c3cd507fc68 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -215,10 +215,10 @@ PyLocale_strcoll(PyObject* self, PyObject* args) if (!PyArg_ParseTuple(args, "UU:strcoll", &os1, &os2)) return NULL; /* Convert the unicode strings to wchar[]. */ - ws1 = PyUnicode_AsWideCharString(os1, NULL); + ws1 = _PyUnicode_AsWideCharString(os1); if (ws1 == NULL) goto done; - ws2 = PyUnicode_AsWideCharString(os2, NULL); + ws2 = _PyUnicode_AsWideCharString(os2); if (ws2 == NULL) goto done; /* Collate the strings. */ diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 3e96aa0cd50c64..4a1324cb7e621b 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3615,7 +3615,7 @@ PyInit__tkinter(void) return NULL; } if (str_path != NULL) { - wcs_path = PyUnicode_AsWideCharString(str_path, NULL); + wcs_path = _PyUnicode_AsWideCharString(str_path); if (wcs_path == NULL) { return NULL; } diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 0aa8657898da13..17581e0a80d0d7 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1151,7 +1151,7 @@ ConnectPipe(OverlappedObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "U", &AddressObj)) return NULL; - Address = PyUnicode_AsWideCharString(AddressObj, NULL); + Address = _PyUnicode_AsWideCharString(AddressObj); if (Address == NULL) return NULL; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 328b84f3338ea3..d6c34b9225c1b9 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -607,7 +607,7 @@ time_strftime(PyObject *self, PyObject *args) buf.tm_isdst = 1; #ifdef HAVE_WCSFTIME - format = PyUnicode_AsWideCharString(format_arg, NULL); + format = _PyUnicode_AsWideCharString(format_arg); if (format == NULL) return NULL; fmt = format; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4e0c663e338a52..3767064e504981 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3017,6 +3017,37 @@ PyUnicode_AsWideCharString(PyObject *unicode, return buffer; } +wchar_t* +_PyUnicode_AsWideCharString(PyObject *unicode) +{ + const wchar_t *wstr; + wchar_t *buffer; + Py_ssize_t buflen; + + if (unicode == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + wstr = PyUnicode_AsUnicodeAndSize(unicode, &buflen); + if (wstr == NULL) { + return NULL; + } + if (wcslen(wstr) != (size_t)buflen) { + PyErr_SetString(PyExc_ValueError, + "embedded null character"); + return NULL; + } + + buffer = PyMem_NEW(wchar_t, buflen + 1); + if (buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + memcpy(buffer, wstr, (buflen + 1) * sizeof(wchar_t)); + return buffer; +} + #endif /* HAVE_WCHAR_H */ PyObject * diff --git a/PC/winsound.c b/PC/winsound.c index 7feebcbcf4377f..0d3f04584335ce 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -101,7 +101,7 @@ winsound_PlaySound_impl(PyObject *module, PyObject *sound, int flags) Py_TYPE(sound)->tp_name); return NULL; } - wsound = PyUnicode_AsWideCharString(sound, NULL); + wsound = _PyUnicode_AsWideCharString(sound); if (wsound == NULL) { return NULL; } From 26daad4ee14693381d84a5235709d22aed1c22ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 28 Jun 2017 02:28:51 +0200 Subject: [PATCH 0498/1345] bpo-30704, bpo-30604: Fix memleak in code_dealloc() (#2455) (#2456) Free also co_extra->ce_extras, not only co_extra. (cherry picked from commit 23e7944eba1968bb8432fdc4cc96d4fdd2c1a230) --- Objects/codeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d38f185ba3b8e4..6de697ae3fd31c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -422,7 +422,8 @@ code_dealloc(PyCodeObject *co) } } - PyMem_FREE(co->co_extra); + PyMem_Free(co_extra->ce_extras); + PyMem_Free(co_extra); } Py_XDECREF(co->co_code); From c4cc5534fd32dfde5564dd44f867f2fd66d3e1da Mon Sep 17 00:00:00 2001 From: regexaurus Date: Tue, 27 Jun 2017 22:38:22 -0400 Subject: [PATCH 0499/1345] Clarification to the `break` statement (GH-2453) (#GH-2457) Clarify that the break statement breaks out of the innermost enclosing for or while loop. (cherry picked from commit 36fc896740319d2c03aa2054a90a999c162517ef) --- Doc/tutorial/controlflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 54171bc96f0f08..36b093950e10d8 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -157,7 +157,7 @@ Later we will see more functions that return iterables and take iterables as arg :keyword:`break` and :keyword:`continue` Statements, and :keyword:`else` Clauses on Loops ========================================================================================= -The :keyword:`break` statement, like in C, breaks out of the smallest enclosing +The :keyword:`break` statement, like in C, breaks out of the innermost enclosing :keyword:`for` or :keyword:`while` loop. Loop statements may have an ``else`` clause; it is executed when the loop From af68382f68b08a383e7064777cf817375681e434 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 27 Jun 2017 23:02:19 -0400 Subject: [PATCH 0500/1345] [3.6] bpo-30723: IDLE -- Enhance parenmatch; add style, flash, and help (GH-2306) (#2460) * Add 'parens' style to highlight both opener and closer. * Make 'default' style, which is not default, a synonym for 'opener'. * Make time-delay work the same with all styles. * Add help for config dialog extensions tab, including parenmatch. * Add new tests. Original patch by Charles Wohlganger. (cherry picked from commit fae2c35) --- Lib/idlelib/configdialog.py | 15 ++++ Lib/idlelib/idle_test/test_parenmatch.py | 68 +++++++--------- Lib/idlelib/parenmatch.py | 80 +++++++++++-------- Misc/ACKS | 1 + .../2017-06-27-19-05-40.bpo-30723.rQh06y.rst | 5 ++ 5 files changed, 97 insertions(+), 72 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-27-19-05-40.bpo-30723.rQh06y.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 96682a92214769..4a25846238f6e1 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1407,6 +1407,21 @@ def save_all_changed_extensions(self): be used with older IDLE releases if it is saved as a custom key set, with a different name. ''', + 'Extensions': ''' +Extensions: + +Autocomplete: Popupwait is milleseconds to wait after key char, without +cursor movement, before popping up completion box. Key char is '.' after +identifier or a '/' (or '\\' on Windows) within a string. + +FormatParagraph: Max-width is max chars in lines after re-formatting. +Use with paragraphs in both strings and comment blocks. + +ParenMatch: Style indicates what is highlighted when closer is entered: +'opener' - opener '({[' corresponding to closer; 'parens' - both chars; +'expression' (default) - also everything in between. Flash-delay is how +long to highlight if cursor is not moved (0 means forever). +''' } diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index cbec35018b990f..6943a70c997c1c 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -3,13 +3,14 @@ This must currently be a gui test because ParenMatch methods use several text methods not defined on idlelib.idle_test.mock_tk.Text. ''' +from idlelib.parenmatch import ParenMatch from test.support import requires requires('gui') import unittest from unittest.mock import Mock from tkinter import Tk, Text -from idlelib.parenmatch import ParenMatch + class DummyEditwin: def __init__(self, text): @@ -44,46 +45,39 @@ def get_parenmatch(self): pm.bell = lambda: None return pm - def test_paren_expression(self): + def test_paren_styles(self): """ - Test ParenMatch with 'expression' style. + Test ParenMatch with each style. """ text = self.text pm = self.get_parenmatch() - pm.set_style('expression') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.15')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) - - # paren_closed_event can only be tested as below - pm.paren_closed_event('event') - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.16')) - - def test_paren_default(self): - """ - Test ParenMatch with 'default' style. - """ - text = self.text - pm = self.get_parenmatch() - pm.set_style('default') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.11')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) + for style, range1, range2 in ( + ('opener', ('1.10', '1.11'), ('1.10', '1.11')), + ('default',('1.10', '1.11'),('1.10', '1.11')), + ('parens', ('1.14', '1.15'), ('1.15', '1.16')), + ('expression', ('1.10', '1.15'), ('1.10', '1.16'))): + with self.subTest(style=style): + text.delete('1.0', 'end') + pm.set_style(style) + text.insert('insert', 'def foobar(a, b') + + pm.flash_paren_event('event') + self.assertIn('<>', text.event_info()) + if style == 'parens': + self.assertTupleEqual(text.tag_nextrange('paren', '1.0'), + ('1.10', '1.11')) + self.assertTupleEqual( + text.tag_prevrange('paren', 'end'), range1) + + text.insert('insert', ')') + pm.restore_event() + self.assertNotIn('<>', + text.event_info()) + self.assertEqual(text.tag_prevrange('paren', 'end'), ()) + + pm.paren_closed_event('event') + self.assertTupleEqual( + text.tag_prevrange('paren', 'end'), range2) def test_paren_corner(self): """ diff --git a/Lib/idlelib/parenmatch.py b/Lib/idlelib/parenmatch.py index dcec34c18f9287..c15cb818cd8fbd 100644 --- a/Lib/idlelib/parenmatch.py +++ b/Lib/idlelib/parenmatch.py @@ -11,43 +11,37 @@ CHECK_DELAY = 100 # milliseconds class ParenMatch: - """Highlight matching parentheses + """Highlight matching openers and closers, (), [], and {}. - There are three supported style of paren matching, based loosely - on the Emacs options. The style is select based on the - HILITE_STYLE attribute; it can be changed used the set_style - method. + There are three supported styles of paren matching. When a right + paren (opener) is typed: - The supported styles are: + opener -- highlight the matching left paren (closer); + parens -- highlight the left and right parens (opener and closer); + expression -- highlight the entire expression from opener to closer. + (For back compatibility, 'default' is a synonym for 'opener'). - default -- When a right paren is typed, highlight the matching - left paren for 1/2 sec. - - expression -- When a right paren is typed, highlight the entire - expression from the left paren to the right paren. + Flash-delay is the maximum milliseconds the highlighting remains. + Any cursor movement (key press or click) before that removes the + highlight. If flash-delay is 0, there is no maximum. TODO: - - extend IDLE with configuration dialog to change options - - implement rest of Emacs highlight styles (see below) - - print mismatch warning in IDLE status window - - Note: In Emacs, there are several styles of highlight where the - matching paren is highlighted whenever the cursor is immediately - to the right of a right paren. I don't know how to do that in Tk, - so I haven't bothered. + - Augment bell() with mismatch warning in status window. + - Highlight when cursor is moved to the right of a closer. + This might be too expensive to check. """ menudefs = [ ('edit', [ ("Show surrounding parens", "<>"), ]) ] - STYLE = idleConf.GetOption('extensions','ParenMatch','style', - default='expression') - FLASH_DELAY = idleConf.GetOption('extensions','ParenMatch','flash-delay', - type='int',default=500) + STYLE = idleConf.GetOption( + 'extensions','ParenMatch','style', default='expression') + FLASH_DELAY = idleConf.GetOption( + 'extensions','ParenMatch','flash-delay', type='int',default=500) + BELL = idleConf.GetOption( + 'extensions','ParenMatch','bell', type='bool',default=1) HILITE_CONFIG = idleConf.GetHighlight(idleConf.CurrentTheme(),'hilite') - BELL = idleConf.GetOption('extensions','ParenMatch','bell', - type='bool',default=1) RESTORE_VIRTUAL_EVENT_NAME = "<>" # We want the restore event be called before the usual return and @@ -69,27 +63,32 @@ def __init__(self, editwin): self.set_style(self.STYLE) def activate_restore(self): + "Activate mechanism to restore text from highlighting." if not self.is_restore_active: for seq in self.RESTORE_SEQUENCES: self.text.event_add(self.RESTORE_VIRTUAL_EVENT_NAME, seq) self.is_restore_active = True def deactivate_restore(self): + "Remove restore event bindings." if self.is_restore_active: for seq in self.RESTORE_SEQUENCES: self.text.event_delete(self.RESTORE_VIRTUAL_EVENT_NAME, seq) self.is_restore_active = False def set_style(self, style): + "Set tag and timeout functions." self.STYLE = style - if style == "default": - self.create_tag = self.create_tag_default - self.set_timeout = self.set_timeout_last - elif style == "expression": - self.create_tag = self.create_tag_expression - self.set_timeout = self.set_timeout_none + self.create_tag = ( + self.create_tag_opener if style in {"opener", "default"} else + self.create_tag_parens if style == "parens" else + self.create_tag_expression) # "expression" or unknown + + self.set_timeout = (self.set_timeout_last if self.FLASH_DELAY else + self.set_timeout_none) def flash_paren_event(self, event): + "Handle editor 'show surrounding parens' event (menu or shortcut)." indices = (HyperParser(self.editwin, "insert") .get_surrounding_brackets()) if indices is None: @@ -97,11 +96,12 @@ def flash_paren_event(self, event): return "break" self.activate_restore() self.create_tag(indices) - self.set_timeout_last() + self.set_timeout() return "break" def paren_closed_event(self, event): - # If it was a shortcut and not really a closing paren, quit. + "Handle user input of closer." + # If user bound non-closer to <>, quit. closer = self.text.get("insert-1c") if closer not in _openers: return "break" @@ -118,6 +118,7 @@ def paren_closed_event(self, event): return "break" def restore_event(self, event=None): + "Remove effect of doing match." self.text.tag_delete("paren") self.deactivate_restore() self.counter += 1 # disable the last timer, if there is one. @@ -129,11 +130,20 @@ def handle_restore_timer(self, timer_count): # any one of the create_tag_XXX methods can be used depending on # the style - def create_tag_default(self, indices): + def create_tag_opener(self, indices): """Highlight the single paren that matches""" self.text.tag_add("paren", indices[0]) self.text.tag_config("paren", self.HILITE_CONFIG) + def create_tag_parens(self, indices): + """Highlight the left and right parens""" + if self.text.get(indices[1]) in (')', ']', '}'): + rightindex = indices[1]+"+1c" + else: + rightindex = indices[1] + self.text.tag_add("paren", indices[0], indices[0]+"+1c", rightindex+"-1c", rightindex) + self.text.tag_config("paren", self.HILITE_CONFIG) + def create_tag_expression(self, indices): """Highlight the entire expression""" if self.text.get(indices[1]) in (')', ']', '}'): @@ -162,7 +172,7 @@ def callme(callme, self=self, c=self.counter, self.editwin.text_frame.after(CHECK_DELAY, callme, callme) def set_timeout_last(self): - """The last highlight created will be removed after .5 sec""" + """The last highlight created will be removed after FLASH_DELAY millisecs""" # associate a counter with an event; only disable the "paren" # tag if the event is for the most recent timer. self.counter += 1 diff --git a/Misc/ACKS b/Misc/ACKS index b8ae14be6914fc..c4af869c74cf6b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1690,6 +1690,7 @@ John Wiseman Chris Withers Stefan Witzel Irek Wlizlo +Charles Wohlganger David Wolever Klaus-Juergen Wolf Dan Wolfe diff --git a/Misc/NEWS.d/next/IDLE/2017-06-27-19-05-40.bpo-30723.rQh06y.rst b/Misc/NEWS.d/next/IDLE/2017-06-27-19-05-40.bpo-30723.rQh06y.rst new file mode 100644 index 00000000000000..ceb42a2bbcaf2f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-27-19-05-40.bpo-30723.rQh06y.rst @@ -0,0 +1,5 @@ +IDLE: Make several improvements to parenmatch. Add 'parens' style to +highlight both opener and closer. Make 'default' style, which is not +default, a synonym for 'opener'. Make time-delay work the same with all +styles. Add help for config dialog extensions tab, including help for +parenmatch. Add new tests. Original patch by Charles Wohlganger. From 413c0a92bcc92efe92849fe5e711163da453410b Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 27 Jun 2017 23:02:38 -0400 Subject: [PATCH 0501/1345] [3.6] bpo-24813: IDLE tagline is Integrated Development and Learning Environment (GH-2451) (#2461) Patch by Mark Roseman (cherry picked from commit 592eda1) --- Lib/idlelib/help_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 967f0342a101c2..679ac78d32db95 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -81,7 +81,7 @@ def create_widgets(self): logo = Label(frame_background, image=self.icon_image, bg=self.bg) logo.grid(row=0, column=0, sticky=W, rowspan=2, padx=10, pady=10) - byline_text = "Python's Integrated DeveLopment Environment" + 5*'\n' + byline_text = "Python's Integrated Development\nand Learning Environment" + 5*'\n' byline = Label(frame_background, text=byline_text, justify=LEFT, fg=self.fg, bg=self.bg) byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) From 0834905d9b61291b1fc5e05a1ffbc69de9c9379f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 28 Jun 2017 09:27:35 +0300 Subject: [PATCH 0502/1345] [3.6] bpo-13617: Reject embedded null characters in wchar* strings. (GH-2302) (#2462) Based on patch by Victor Stinner. Add private C API function _PyUnicode_AsUnicode() which is similar to PyUnicode_AsUnicode(), but checks for null characters.. (cherry picked from commit f7eae0adfcd4c50034281b2c69f461b43b68db84) --- Include/unicodeobject.h | 10 +++++++--- Lib/ctypes/test/test_loading.py | 2 ++ Lib/test/test_builtin.py | 6 ++++++ Lib/test/test_curses.py | 11 ++++++++++- Lib/test/test_grp.py | 2 ++ Lib/test/test_imp.py | 4 ++++ Lib/test/test_locale.py | 5 +++++ Lib/test/test_time.py | 4 ++++ Lib/test/test_winsound.py | 2 ++ Modules/_ctypes/callproc.c | 7 ++++--- Modules/_cursesmodule.c | 9 +++++++++ Modules/_io/fileio.c | 3 +-- Modules/_localemodule.c | 5 +++++ Modules/grpmodule.c | 1 + Modules/nismodule.c | 1 + Modules/posixmodule.c | 18 ++++++++++++------ Modules/pwdmodule.c | 1 + Modules/spwdmodule.c | 1 + Objects/unicodeobject.c | 14 ++++++++++++++ PC/_msi.c | 6 +++++- Python/dynload_win.c | 4 ++-- Python/fileutils.c | 23 +++++++++++++++++------ 22 files changed, 115 insertions(+), 24 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index cec2b7f4b07e71..f498873870082f 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -752,23 +752,27 @@ PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4( PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode); #endif +#ifndef Py_LIMITED_API /* Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer. If the wchar_t/Py_UNICODE representation is not yet available, this function will calculate it. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( PyObject *unicode /* Unicode object */ ); -#endif + +/* Similar to PyUnicode_AsUnicode(), but raises a ValueError if the string + contains null characters. */ +PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode( + PyObject *unicode /* Unicode object */ + ); /* Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer and save the length at size. If the wchar_t/Py_UNICODE representation is not yet available, this function will calculate it. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( PyObject *unicode, /* Unicode object */ Py_ssize_t *size /* location where to save the length */ diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py index 45571f3d3d5209..f3b65b9d6e7e2e 100644 --- a/Lib/ctypes/test/test_loading.py +++ b/Lib/ctypes/test/test_loading.py @@ -62,6 +62,8 @@ def test_load_library(self): windll["kernel32"].GetModuleHandleW windll.LoadLibrary("kernel32").GetModuleHandleW WinDLL("kernel32").GetModuleHandleW + # embedded null character + self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 416316c02852c8..7a4b7eb9176ab9 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -151,6 +151,8 @@ def test_import(self): self.assertRaises(TypeError, __import__, 1, 2, 3, 4) self.assertRaises(ValueError, __import__, '') self.assertRaises(TypeError, __import__, 'sys', name='sys') + # embedded null character + self.assertRaises(ModuleNotFoundError, __import__, 'string\x00') def test_abs(self): # int @@ -1002,6 +1004,10 @@ def test_open(self): self.assertEqual(fp.read(300), 'XXX'*100) self.assertEqual(fp.read(1000), 'YYY'*100) + # embedded null bytes and characters + self.assertRaises(ValueError, open, 'a\x00b') + self.assertRaises(ValueError, open, b'a\x00b') + def test_open_default_encoding(self): old_environ = dict(os.environ) try: diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 3d8c50bcb6c6ae..0d0b160d7645a5 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -81,7 +81,7 @@ def test_window_funcs(self): win2 = curses.newwin(15,15, 5,5) for meth in [stdscr.addch, stdscr.addstr]: - for args in [('a'), ('a', curses.A_BOLD), + for args in [('a',), ('a', curses.A_BOLD), (4,4, 'a'), (5,5, 'a', curses.A_BOLD)]: with self.subTest(meth=meth.__qualname__, args=args): meth(*args) @@ -194,6 +194,15 @@ def test_window_funcs(self): self.assertRaises(ValueError, stdscr.instr, -2) self.assertRaises(ValueError, stdscr.instr, 2, 3, -2) + def test_embedded_null_chars(self): + # reject embedded null bytes and characters + stdscr = self.stdscr + for arg in ['a', b'a']: + with self.subTest(arg=arg): + self.assertRaises(ValueError, stdscr.addstr, 'a\0') + self.assertRaises(ValueError, stdscr.addnstr, 'a\0', 1) + self.assertRaises(ValueError, stdscr.insstr, 'a\0') + self.assertRaises(ValueError, stdscr.insnstr, 'a\0', 1) def test_module_funcs(self): "Test module-level functions" diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 69095a3fb98d74..e511947858c0a3 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -50,6 +50,8 @@ def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrall, 42) + # embedded null character + self.assertRaises(ValueError, grp.getgrnam, 'a\x00b') # try to get some errors bynames = {} diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 4ece3654108e49..6f35f49487ce7f 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -314,6 +314,10 @@ def test_multiple_calls_to_get_data(self): loader.get_data(imp.__file__) # File should be closed loader.get_data(imp.__file__) # Will need to create a newly opened file + def test_load_source(self): + with self.assertRaisesRegex(ValueError, 'embedded null'): + imp.load_source(__name__, __file__ + "\0") + class ReloadTests(unittest.TestCase): diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 99fab58cb9326d..650d7373d8e1df 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -339,9 +339,14 @@ def test_strcoll(self): self.assertLess(locale.strcoll('a', 'b'), 0) self.assertEqual(locale.strcoll('a', 'a'), 0) self.assertGreater(locale.strcoll('b', 'a'), 0) + # embedded null character + self.assertRaises(ValueError, locale.strcoll, 'a\0', 'a') + self.assertRaises(ValueError, locale.strcoll, 'a', 'a\0') def test_strxfrm(self): self.assertLess(locale.strxfrm('a'), locale.strxfrm('b')) + # embedded null character + self.assertRaises(ValueError, locale.strxfrm, 'a\0') class TestEnUSCollation(BaseLocalizedTest, TestCollation): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index f2242126d1826d..810ec37b1113f9 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -126,6 +126,10 @@ def test_strftime(self): except ValueError: self.fail('conversion specifier: %r failed.' % format) + self.assertRaises(TypeError, time.strftime, b'%S', tt) + # embedded null character + self.assertRaises(ValueError, time.strftime, '%S\0', tt) + def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts # of the time tuple (0 is valid for *all* values). diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index 179e069a1ccc1d..21437ef5118cb0 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -98,6 +98,8 @@ def test_errors(self): self.assertRaises(TypeError, winsound.PlaySound, "bad", winsound.SND_MEMORY) self.assertRaises(TypeError, winsound.PlaySound, 1, 0) + # embedded null character + self.assertRaises(ValueError, winsound.PlaySound, 'bad\0', 0) def test_keyword_args(self): safe_PlaySound(flags=winsound.SND_ALIAS, sound="SystemExit") diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 358f287bb7cd45..ff95dffb82c244 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1231,14 +1231,15 @@ The handle may be used to locate exported functions in this\n\ module.\n"; static PyObject *load_library(PyObject *self, PyObject *args) { - WCHAR *name; + const WCHAR *name; PyObject *nameobj; PyObject *ignored; HMODULE hMod; - if (!PyArg_ParseTuple(args, "O|O:LoadLibrary", &nameobj, &ignored)) + + if (!PyArg_ParseTuple(args, "U|O:LoadLibrary", &nameobj, &ignored)) return NULL; - name = PyUnicode_AsUnicode(nameobj); + name = _PyUnicode_AsUnicode(nameobj); if (!name) return NULL; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 41b831e58f35a6..7a7095118a0baf 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -342,6 +342,7 @@ static int PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, PyObject **bytes, wchar_t **wstr) { + char *str; if (PyUnicode_Check(obj)) { #ifdef HAVE_NCURSESW assert (wstr != NULL); @@ -354,12 +355,20 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, *bytes = PyUnicode_AsEncodedString(obj, win->encoding, NULL); if (*bytes == NULL) return 0; + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(*bytes, &str, NULL) < 0) { + return 0; + } return 1; #endif } else if (PyBytes_Check(obj)) { Py_INCREF(obj); *bytes = obj; + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(*bytes, &str, NULL) < 0) { + return 0; + } return 1; } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 833ea8e7b50894..918fa577758d28 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -280,11 +280,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (fd < 0) { #ifdef MS_WINDOWS - Py_ssize_t length; if (!PyUnicode_FSDecoder(nameobj, &stringobj)) { return -1; } - widename = PyUnicode_AsUnicodeAndSize(stringobj, &length); + widename = PyUnicode_AsUnicode(stringobj); if (widename == NULL) return -1; #else diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 0c7c3cd507fc68..71c9146ccb8f47 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -252,6 +252,11 @@ PyLocale_strxfrm(PyObject* self, PyObject* args) s = PyUnicode_AsWideCharString(str, &n1); if (s == NULL) goto exit; + if (wcslen(s) != (size_t)n1) { + PyErr_SetString(PyExc_ValueError, + "embedded null character"); + goto exit; + } /* assume no change in size, first */ n1 = n1 + 1; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 9437ae7a5b1a14..f577fd3ab4ec3b 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -151,6 +151,7 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) if ((bytes = PyUnicode_EncodeFSDefault(name)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name_chars, NULL) == -1) goto out; diff --git a/Modules/nismodule.c b/Modules/nismodule.c index b6a855c5775c74..a9028bbfe54c6c 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -169,6 +169,7 @@ nis_match (PyObject *self, PyObject *args, PyObject *kwdict) return NULL; if ((bkey = PyUnicode_EncodeFSDefault(ukey)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bkey, &key, &keylen) == -1) { Py_DECREF(bkey); return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bf8254311e02dd..4607b181c6cea6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3680,7 +3680,7 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) PyObject *result; const wchar_t *path_wchar; - path_wchar = PyUnicode_AsUnicode(path); + path_wchar = _PyUnicode_AsUnicode(path); if (path_wchar == NULL) return NULL; @@ -7088,7 +7088,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) )) return NULL; - path = PyUnicode_AsUnicode(po); + path = _PyUnicode_AsUnicode(po); if (path == NULL) return NULL; @@ -8881,6 +8881,7 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic end generated code: output=d29a567d6b2327d2 input=ba586581c2e6105f]*/ { const wchar_t *env; + Py_ssize_t size; /* Search from index 1 because on Windows starting '=' is allowed for defining hidden environment variables. */ @@ -8894,16 +8895,21 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) if (unicode == NULL) { return NULL; } - if (_MAX_ENV < PyUnicode_GET_LENGTH(unicode)) { + + env = PyUnicode_AsUnicodeAndSize(unicode, &size); + if (env == NULL) + goto error; + if (size > _MAX_ENV) { PyErr_Format(PyExc_ValueError, "the environment variable is longer than %u characters", _MAX_ENV); goto error; } - - env = PyUnicode_AsUnicode(unicode); - if (env == NULL) + if (wcslen(env) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); goto error; + } + if (_wputenv(env)) { posix_error(); goto error; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index 784e9d0349fa79..bbef2de9c52253 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -158,6 +158,7 @@ pwd_getpwnam_impl(PyObject *module, PyObject *arg) if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getpwnam(name)) == NULL) { diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 556a715a242482..1601ec0f2fc4e7 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -134,6 +134,7 @@ spwd_getspnam_impl(PyObject *module, PyObject *arg) if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getspnam(name)) == NULL) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3767064e504981..494cdbd42e033b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4164,6 +4164,20 @@ PyUnicode_AsUnicode(PyObject *unicode) return PyUnicode_AsUnicodeAndSize(unicode, NULL); } +const Py_UNICODE * +_PyUnicode_AsUnicode(PyObject *unicode) +{ + Py_ssize_t size; + const Py_UNICODE *wstr; + + wstr = PyUnicode_AsUnicodeAndSize(unicode, &size); + if (wstr && wcslen(wstr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + return wstr; +} + Py_ssize_t PyUnicode_GetSize(PyObject *unicode) diff --git a/PC/_msi.c b/PC/_msi.c index 789b04f5a25177..15aa7d8fecf324 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -600,8 +600,12 @@ summary_setproperty(msiobj* si, PyObject *args) return NULL; if (PyUnicode_Check(data)) { + const WCHAR *value = _PyUnicode_AsUnicode(data); + if (value == NULL) { + return NULL; + } status = MsiSummaryInfoSetPropertyW(si->h, field, VT_LPSTR, - 0, NULL, PyUnicode_AsUnicode(data)); + 0, NULL, value); } else if (PyLong_CheckExact(data)) { long value = PyLong_AsLong(data); if (value == -1 && PyErr_Occurred()) { diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 05050cf41d2110..0fdf77f55280ca 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -190,13 +190,13 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, { dl_funcptr p; char funcname[258], *import_python; - wchar_t *wpathname; + const wchar_t *wpathname; #ifndef _DEBUG _Py_CheckPython3(); #endif - wpathname = PyUnicode_AsUnicode(pathname); + wpathname = _PyUnicode_AsUnicode(pathname); if (wpathname == NULL) return NULL; diff --git a/Python/fileutils.c b/Python/fileutils.c index f3764e4b3c5a2d..97505e5bc6d3ce 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -711,21 +711,32 @@ _Py_stat(PyObject *path, struct stat *statbuf) #ifdef MS_WINDOWS int err; struct _stat wstatbuf; - wchar_t *wpath; + const wchar_t *wpath; - wpath = PyUnicode_AsUnicode(path); + wpath = _PyUnicode_AsUnicode(path); if (wpath == NULL) return -2; + err = _wstat(wpath, &wstatbuf); if (!err) statbuf->st_mode = wstatbuf.st_mode; return err; #else int ret; - PyObject *bytes = PyUnicode_EncodeFSDefault(path); + PyObject *bytes; + char *cpath; + + bytes = PyUnicode_EncodeFSDefault(path); if (bytes == NULL) return -2; - ret = stat(PyBytes_AS_STRING(bytes), statbuf); + + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(bytes, &cpath, NULL) == -1) { + Py_DECREF(bytes); + return -2; + } + + ret = stat(cpath, statbuf); Py_DECREF(bytes); return ret; #endif @@ -1080,7 +1091,7 @@ _Py_fopen_obj(PyObject *path, const char *mode) FILE *f; int async_err = 0; #ifdef MS_WINDOWS - wchar_t *wpath; + const wchar_t *wpath; wchar_t wmode[10]; int usize; @@ -1094,7 +1105,7 @@ _Py_fopen_obj(PyObject *path, const char *mode) Py_TYPE(path)); return NULL; } - wpath = PyUnicode_AsUnicode(path); + wpath = _PyUnicode_AsUnicode(path); if (wpath == NULL) return NULL; From e022aad73a4151b5628e2476a8465ce6c0d18b8c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 28 Jun 2017 11:49:38 +0200 Subject: [PATCH 0503/1345] [3.6] bpo-30775: Fix refleaks in test_multiprocessing (GH-2467) (#2468) Forgetting to call Process.join() can keep some resources alive. (cherry picked from commit a79f8faccf5e26f55e8b9496ad49d2071b5e299c) --- Lib/test/_test_multiprocessing.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cd2c8dbc487fad..e76c45702205e9 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1181,10 +1181,19 @@ def __init__(self, namespace, f, args, n, wait_before_exit=False): self._can_exit = namespace.Event() if not wait_before_exit: self._can_exit.set() + + threads = [] for i in range(n): p = namespace.Process(target=self.task) p.daemon = True p.start() + threads.append(p) + + def finalize(threads): + for p in threads: + p.join() + + self._finalizer = weakref.finalize(self, finalize, threads) def task(self): pid = os.getpid() @@ -1207,6 +1216,9 @@ def wait_for_finished(self): def do_finish(self): self._can_exit.set() + def close(self): + self._finalizer() + class AppendTrue(object): def __init__(self, obj): @@ -1239,8 +1251,11 @@ def DummyList(self): def run_threads(self, f, args): b = Bunch(self, f, args, self.N-1) - f(*args) - b.wait_for_finished() + try: + f(*args) + b.wait_for_finished() + finally: + b.close() @classmethod def multipass(cls, barrier, results, n): From 38d6a40898940d2c62c637fcc8acce05e75de5c2 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 28 Jun 2017 13:08:20 +0200 Subject: [PATCH 0504/1345] [3.6] Clear potential ref cycle between Process and Process target (GH-2470) (#2471) * Clear potential ref cycle between Process and Process target Besides Process.join() not being called, this was an indirect cause of bpo-30775. The threading module already does this. * Add issue reference. (cherry picked from commit 79d37ae979a65ada0b2ac820279ccc3b1cd41ba6) --- Lib/multiprocessing/process.py | 3 +++ Lib/test/_test_multiprocessing.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index bca8b7a0047070..f9c22703df29cb 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -104,6 +104,9 @@ def start(self): _cleanup() self._popen = self._Popen(self) self._sentinel = self._popen.sentinel + # Avoid a refcycle if the target function holds an indirect + # reference to the process object (see bpo-30775) + del self._target, self._args, self._kwargs _children.add(self) def terminate(self): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e76c45702205e9..871a34e4b26f12 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -191,6 +191,12 @@ def get_value(self): # Testcases # +class DummyCallable: + def __call__(self, q, c): + assert isinstance(c, DummyCallable) + q.put(5) + + class _TestProcess(BaseTestCase): ALLOWED_TYPES = ('processes', 'threads') @@ -398,6 +404,18 @@ def test_sentinel(self): p.join() self.assertTrue(wait_for_handle(sentinel, timeout=1)) + def test_lose_target_ref(self): + c = DummyCallable() + wr = weakref.ref(c) + q = self.Queue() + p = self.Process(target=c, args=(q, c)) + del c + p.start() + p.join() + self.assertIs(wr(), None) + self.assertEqual(q.get(), 5) + + # # # From b78fbaaeab9df8cfbbdae3d5faf2d1537d73e43b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Jun 2017 10:53:22 +0200 Subject: [PATCH 0505/1345] bpo-30602: Fix refleak in os.spawnv() (#2212) (#2486) When os.spawnv() fails while handling arguments, free correctly argvlist: pass lastarg+1 rather than lastarg to free_string_array() to also free the first item. (cherry picked from commit 8acb4cf2b3436652568d7a70228b166316181466) --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4607b181c6cea6..c205dd9b054afd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5106,7 +5106,7 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) return NULL; } if (i == 0 && !argvlist[0][0]) { - free_string_array(argvlist, i); + free_string_array(argvlist, i + 1); PyErr_SetString( PyExc_ValueError, "spawnv() arg 2 first element cannot be empty"); From 1d56ed5210babb68b5798cd943bb21f417e781ee Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Jun 2017 10:53:34 +0200 Subject: [PATCH 0506/1345] _winconsoleio: Fix memory leak (#2485) Fix memory leak when _winconsoleio tries to open a non-console file: free the name buffer. --- Modules/_io/winconsoleio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 346c38637f9e1b..5cf3f070d82526 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -317,6 +317,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, if (name == NULL) return -1; if (console_type == '\0') { + PyMem_Free(name); PyErr_SetString(PyExc_ValueError, "Cannot open non-console file"); return -1; @@ -400,7 +401,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, PyErr_SetString(PyExc_ValueError, "Cannot open non-console file"); goto error; - } + } if (self->writable && console_type != 'w') { PyErr_SetString(PyExc_ValueError, "Cannot open console input buffer for writing"); @@ -428,8 +429,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, internal_close(self); done: - if (name) - PyMem_Free(name); + PyMem_Free(name); return ret; } From 6f31717c47e325460e2a661bf44b45d342d65bcb Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 29 Jun 2017 19:15:18 -0400 Subject: [PATCH 0507/1345] [3.6] bpo-30495: IDLE: improve textview with docstrings, PEP8 names, more tests. (GH-2283) (#2496) Split TextViewer class into ViewWindow, ViewFrame, and TextFrame classes so that instances of the latter two can be placed with other widgets within a multiframe window. Patch by Cheryl Sabella. (cherry picked from commit 42bc8be) --- Lib/idlelib/idle_test/htest.py | 20 ++-- Lib/idlelib/idle_test/test_help_about.py | 23 ++-- Lib/idlelib/idle_test/test_textview.py | 86 +++++++++----- Lib/idlelib/pyshell.py | 2 +- Lib/idlelib/textview.py | 108 +++++++++++------- .../2017-06-29-18-23-06.bpo-30495.qIWgc4.rst | 2 + 6 files changed, 141 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-06-29-18-23-06.bpo-30495.qIWgc4.rst diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 6f676ae865ccc6..e483bbc93a2b05 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -8,7 +8,7 @@ end the test. In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attrubute is also X (the usual situation). +or function) whose .__name__ attribute is also X (the usual situation). The first parameter of X must be 'parent'. When called, the parent argument will be the root window. X must create a child Toplevel window (or subclass thereof). The Toplevel may be a test widget or @@ -306,15 +306,6 @@ def _wrapper(parent): # htest # " is an invalid add page and remove page name.\n" } -TextViewer_spec = { - 'file': 'textview', - 'kwds': {'title': 'Test textview', - 'text':'The quick brown fox jumps over the lazy dog.\n'*35, - '_htest': True}, - 'msg': "Test for read-only property of text.\n" - "Text is selectable. Window is scrollable.", - } - _tooltip_spec = { 'file': 'tooltip', 'kwds': {}, @@ -338,6 +329,15 @@ def _wrapper(parent): # htest # "by printing to the console or the IDLE shell.\n" } +ViewWindow_spec = { + 'file': 'textview', + 'kwds': {'title': 'Test textview', + 'text': 'The quick brown fox jumps over the lazy dog.\n'*35, + '_htest': True}, + 'msg': "Test for read-only property of text.\n" + "Select text, scroll window, close" + } + _widget_redirector_spec = { 'file': 'redirector', 'kwds': {}, diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 08c38a515da10d..1f67aaddb3b411 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -50,19 +50,17 @@ def test_dialog_logo(self): def test_printer_buttons(self): """Test buttons whose commands use printer function.""" dialog = self.dialog - button_sources = [(self.dialog.py_license, license), - (self.dialog.py_copyright, copyright), - (self.dialog.py_credits, credits)] + button_sources = [(dialog.py_license, license), + (dialog.py_copyright, copyright), + (dialog.py_credits, credits)] for button, printer in button_sources: printer._Printer__setup() button.invoke() + get = dialog._current_textview.viewframe.textframe.text.get + self.assertEqual(printer._Printer__lines[0], get('1.0', '1.end')) self.assertEqual( - printer._Printer__lines[0], - dialog._current_textview.text.get('1.0', '1.end')) - self.assertEqual( - printer._Printer__lines[1], - dialog._current_textview.text.get('2.0', '2.end')) + printer._Printer__lines[1], get('2.0', '2.end')) dialog._current_textview.destroy() def test_file_buttons(self): @@ -75,14 +73,11 @@ def test_file_buttons(self): for button, filename in button_sources: button.invoke() fn = findfile(filename, subdir='idlelib') + get = dialog._current_textview.viewframe.textframe.text.get with open(fn) as f: - self.assertEqual( - f.readline().strip(), - dialog._current_textview.text.get('1.0', '1.end')) + self.assertEqual(f.readline().strip(), get('1.0', '1.end')) f.readline() - self.assertEqual( - f.readline().strip(), - dialog._current_textview.text.get('3.0', '3.end')) + self.assertEqual(f.readline().strip(), get('3.0', '3.end')) dialog._current_textview.destroy() diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index 7a2f7e460f2c7e..c129c2f0819a2f 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -1,7 +1,7 @@ '''Test idlelib.textview. -Since all methods and functions create (or destroy) a TextViewer, which -is a widget containing multiple widgets, all tests must be gui tests. +Since all methods and functions create (or destroy) a ViewWindow, which +is a widget containing a widget, etcetera, all tests must be gui tests. Using mock Text would not change this. Other mocks are used to retrieve information about calls. @@ -13,7 +13,8 @@ import unittest import os -from tkinter import Tk, Button +from tkinter import Tk +from tkinter.ttk import Button from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func @@ -25,44 +26,44 @@ def setUpModule(): def tearDownModule(): global root root.update_idletasks() - root.destroy() # Pyflakes falsely sees root as undefined. + root.destroy() del root -# If we call TextViewer or wrapper functions with defaults +# If we call ViewWindow or wrapper functions with defaults # modal=True, _utest=False, test hangs on call to wait_window. # Have also gotten tk error 'can't invoke "event" command'. -class TV(tv.TextViewer): # Used in TextViewTest. +class VW(tv.ViewWindow): # Used in ViewWindowTest. transient = Func() grab_set = Func() wait_window = Func() -# Call wrapper class with mock wait_window. -class TextViewTest(unittest.TestCase): +# Call wrapper class VW with mock wait_window. +class ViewWindowTest(unittest.TestCase): def setUp(self): - TV.transient.__init__() - TV.grab_set.__init__() - TV.wait_window.__init__() + VW.transient.__init__() + VW.grab_set.__init__() + VW.wait_window.__init__() def test_init_modal(self): - view = TV(root, 'Title', 'test text') - self.assertTrue(TV.transient.called) - self.assertTrue(TV.grab_set.called) - self.assertTrue(TV.wait_window.called) + view = VW(root, 'Title', 'test text') + self.assertTrue(VW.transient.called) + self.assertTrue(VW.grab_set.called) + self.assertTrue(VW.wait_window.called) view.ok() def test_init_nonmodal(self): - view = TV(root, 'Title', 'test text', modal=False) - self.assertFalse(TV.transient.called) - self.assertFalse(TV.grab_set.called) - self.assertFalse(TV.wait_window.called) + view = VW(root, 'Title', 'test text', modal=False) + self.assertFalse(VW.transient.called) + self.assertFalse(VW.grab_set.called) + self.assertFalse(VW.wait_window.called) view.ok() def test_ok(self): - view = TV(root, 'Title', 'test text', modal=False) + view = VW(root, 'Title', 'test text', modal=False) view.destroy = Func() view.ok() self.assertTrue(view.destroy.called) @@ -70,7 +71,28 @@ def test_ok(self): view.destroy() -# Call TextViewer with modal=False. +class TextFrameTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + "By itself, this tests that file parsed without exception." + cls.root = root = Tk() + root.withdraw() + cls.frame = tv.TextFrame(root, 'test text') + + @classmethod + def tearDownClass(cls): + del cls.frame + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_line1(self): + get = self.frame.text.get + self.assertEqual(get('1.0', '1.end'), 'test text') + + +# Call ViewWindow with modal=False. class ViewFunctionTest(unittest.TestCase): @classmethod @@ -85,13 +107,16 @@ def tearDownClass(cls): def test_view_text(self): view = tv.view_text(root, 'Title', 'test text', modal=False) - self.assertIsInstance(view, tv.TextViewer) + self.assertIsInstance(view, tv.ViewWindow) + self.assertIsInstance(view.viewframe, tv.ViewFrame) view.ok() def test_view_file(self): view = tv.view_file(root, 'Title', __file__, modal=False) - self.assertIsInstance(view, tv.TextViewer) - self.assertIn('Test', view.text.get('1.0', '1.end')) + self.assertIsInstance(view, tv.ViewWindow) + self.assertIsInstance(view.viewframe, tv.ViewFrame) + get = view.viewframe.textframe.text.get + self.assertIn('Test', get('1.0', '1.end')) view.ok() def test_bad_file(self): @@ -109,8 +134,7 @@ def test_bad_encoding(self): self.assertEqual(tv.showerror.title, 'Unicode Decode Error') - -# Call TextViewer with _utest=True. +# Call ViewWindow with _utest=True. class ButtonClickTest(unittest.TestCase): def setUp(self): @@ -131,7 +155,8 @@ def _command(): self.assertEqual(self.called, True) self.assertEqual(self.view.title(), 'TITLE_TEXT') - self.assertEqual(self.view.text.get('1.0', '1.end'), 'COMMAND') + self.assertEqual(self.view.viewframe.textframe.text.get('1.0', '1.end'), + 'COMMAND') def test_view_file_bind_with_button(self): def _command(): @@ -143,12 +168,11 @@ def _command(): self.assertEqual(self.called, True) self.assertEqual(self.view.title(), 'TITLE_FILE') + get = self.view.viewframe.textframe.text.get with open(__file__) as f: - self.assertEqual(self.view.text.get('1.0', '1.end'), - f.readline().strip()) + self.assertEqual(get('1.0', '1.end'), f.readline().strip()) f.readline() - self.assertEqual(self.view.text.get('3.0', '3.end'), - f.readline().strip()) + self.assertEqual(get('3.0', '3.end'), f.readline().strip()) if __name__ == '__main__': diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 5b0e5b267642ab..be7bd3dfcfb30a 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -892,7 +892,7 @@ def __init__(self, flist=None): try: # page help() text to shell. import pydoc # import must be done here to capture i/o rebinding. - # XXX KBK 27Dec07 use TextViewer someday, but must work w/o subproc + # XXX KBK 27Dec07 use text viewer someday, but must work w/o subproc pydoc.pager = pydoc.plainpager except: sys.stderr = sys.__stderr__ diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index ab653a91413252..de4b190ff2e4de 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -1,14 +1,59 @@ """Simple text browser for IDLE """ -from tkinter import Toplevel, Frame, Button, Text -from tkinter import DISABLED, SUNKEN, VERTICAL, WORD -from tkinter import RIGHT, LEFT, TOP, BOTTOM, BOTH, X, Y -from tkinter.ttk import Scrollbar +from tkinter import Toplevel, Text +from tkinter.ttk import Frame, Scrollbar, Button from tkinter.messagebox import showerror -class TextViewer(Toplevel): +class TextFrame(Frame): + "Display text with scrollbar." + + def __init__(self, parent, rawtext): + """Create a frame for Textview. + + parent - parent widget for this frame + rawtext - text to display + """ + super().__init__(parent) + self['relief'] = 'sunken' + self['height'] = 700 + # TODO: get fg/bg from theme. + self.bg = '#ffffff' + self.fg = '#000000' + + self.text = text = Text(self, wrap='word', highlightthickness=0, + fg=self.fg, bg=self.bg) + self.scroll = scroll = Scrollbar(self, orient='vertical', + takefocus=False, command=text.yview) + text['yscrollcommand'] = scroll.set + text.insert(0.0, rawtext) + text['state'] = 'disabled' + text.focus_set() + + scroll.pack(side='right', fill='y') + text.pack(side='left', expand=True, fill='both') + + +class ViewFrame(Frame): + "Display TextFrame and Close button." + def __init__(self, parent, text): + super().__init__(parent) + self.parent = parent + self.bind('', self.ok) + self.bind('', self.ok) + self.textframe = TextFrame(self, text) + self.button_ok = button_ok = Button( + self, text='Close', command=self.ok, takefocus=False) + self.textframe.pack(side='top', expand=True, fill='both') + button_ok.pack(side='bottom') + + def ok(self, event=None): + """Dismiss text viewer dialog.""" + self.parent.destroy() + + +class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." def __init__(self, parent, title, text, modal=True, @@ -24,26 +69,19 @@ def __init__(self, parent, title, text, modal=True, _htest - bool; change box location when running htest. _utest - bool; don't wait_window when running unittest. """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) + super().__init__(parent) + self['borderwidth'] = 5 # Place dialog below parent if running htest. - self.geometry("=%dx%d+%d+%d" % (750, 500, - parent.winfo_rootx() + 10, - parent.winfo_rooty() + (10 if not _htest else 100))) - # TODO: get fg/bg from theme. - self.bg = '#ffffff' - self.fg = '#000000' + x = parent.winfo_rootx() + 10 + y = parent.winfo_rooty() + (10 if not _htest else 100) + self.geometry(f'=750x500+{x}+{y}') - self.create_widgets() self.title(title) + self.viewframe = ViewFrame(self, text) self.protocol("WM_DELETE_WINDOW", self.ok) - self.parent = parent - self.text.focus_set() - # Bind keys for closing this dialog. - self.bind('', self.ok) - self.bind('', self.ok) - self.text.insert(0.0, text) - self.text.config(state=DISABLED) + self.button_ok = button_ok = Button(self, text='Close', + command=self.ok, takefocus=False) + self.viewframe.pack(side='top', expand=True, fill='both') if modal: self.transient(parent) @@ -51,31 +89,13 @@ def __init__(self, parent, title, text, modal=True, if not _utest: self.wait_window() - def create_widgets(self): - "Create Frame with Text (with vertical Scrollbar) and Button." - frame = Frame(self, relief=SUNKEN, height=700) - frame_buttons = Frame(self) - self.button_ok = Button(frame_buttons, text='Close', - command=self.ok, takefocus=False) - self.scrollbar = Scrollbar(frame, orient=VERTICAL, takefocus=False) - self.text = Text(frame, wrap=WORD, highlightthickness=0, - fg=self.fg, bg=self.bg) - self.scrollbar.config(command=self.text.yview) - self.text.config(yscrollcommand=self.scrollbar.set) - - self.button_ok.pack() - self.scrollbar.pack(side=RIGHT, fill=Y) - self.text.pack(side=LEFT, expand=True, fill=BOTH) - frame_buttons.pack(side=BOTTOM, fill=X) - frame.pack(side=TOP, expand=True, fill=BOTH) - def ok(self, event=None): """Dismiss text viewer dialog.""" self.destroy() def view_text(parent, title, text, modal=True, _utest=False): - """Create TextViewer for given text. + """Create text viewer for given text. parent - parent of this dialog title - string which is the title of popup dialog @@ -84,11 +104,11 @@ def view_text(parent, title, text, modal=True, _utest=False): dialog is displayed _utest - bool; controls wait_window on unittest """ - return TextViewer(parent, title, text, modal, _utest=_utest) + return ViewWindow(parent, title, text, modal, _utest=_utest) def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): - """Create TextViewer for text in filename. + """Create text viewer for text in filename. Return error message if file cannot be read. Otherwise calls view_text with contents of the file. @@ -98,7 +118,7 @@ def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): contents = file.read() except OSError: showerror(title='File Load Error', - message='Unable to load file %r .' % filename, + message=f'Unable to load file {filename!r} .', parent=parent) except UnicodeDecodeError as err: showerror(title='Unicode Decode Error', @@ -113,4 +133,4 @@ def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): import unittest unittest.main('idlelib.idle_test.test_textview', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(TextViewer) + run(ViewWindow) diff --git a/Misc/NEWS.d/next/IDLE/2017-06-29-18-23-06.bpo-30495.qIWgc4.rst b/Misc/NEWS.d/next/IDLE/2017-06-29-18-23-06.bpo-30495.qIWgc4.rst new file mode 100644 index 00000000000000..5e046e0069ecd1 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-06-29-18-23-06.bpo-30495.qIWgc4.rst @@ -0,0 +1,2 @@ +IDLE: Improve textview with docstrings, PEP8 names, and more tests. Patch by +Cheryl Sabella. From 6f3cb059fd1f3a57ffd70c9d54a394a85a6ea13d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 30 Jun 2017 10:54:24 +0200 Subject: [PATCH 0508/1345] [3.6] bpo-30807: signal.setitimer() may disable the timer by mistake (GH-2493) (#2497) * bpo-30807: signal.setitimer() may disable the timer by mistake * Add NEWS blurb (cherry picked from commit 729780a810bbcb12b245a1b652302a601fc9f6fd) --- Lib/test/test_signal.py | 9 +++++++++ .../Library/2017-06-29-22-04-44.bpo-30807.sLtjY-.rst | 6 ++++++ Modules/signalmodule.c | 4 ++++ 3 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-06-29-22-04-44.bpo-30807.sLtjY-.rst diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index ab42ed70cc91ba..aa2b8bb48cd328 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -611,6 +611,15 @@ def test_itimer_prof(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) + def test_setitimer_tiny(self): + # bpo-30807: C setitimer() takes a microsecond-resolution interval. + # Check that float -> timeval conversion doesn't round + # the interval down to zero, which would disable the timer. + self.itimer = signal.ITIMER_REAL + signal.setitimer(self.itimer, 1e-6) + time.sleep(1) + self.assertEqual(self.hndl_called, True) + class PendingSignalsTests(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Library/2017-06-29-22-04-44.bpo-30807.sLtjY-.rst b/Misc/NEWS.d/next/Library/2017-06-29-22-04-44.bpo-30807.sLtjY-.rst new file mode 100644 index 00000000000000..ce6f48a61f6554 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-06-29-22-04-44.bpo-30807.sLtjY-.rst @@ -0,0 +1,6 @@ +signal.setitimer() may disable the timer when passed a tiny value. + +Tiny values (such as 1e-6) are valid non-zero values for setitimer(), which +is specified as taking microsecond-resolution intervals. However, on some +platform, our conversion routine could convert 1e-6 into a zero interval, +therefore disabling the timer instead of (re-)scheduling it. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index ed2f4e8cf0a2dc..55e4c2a00e9ef3 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -139,6 +139,10 @@ timeval_from_double(double d, struct timeval *tv) { tv->tv_sec = floor(d); tv->tv_usec = fmod(d, 1.0) * 1000000.0; + /* Don't disable the timer if the computation above rounds down to zero. */ + if (d > 0.0 && tv->tv_sec == 0 && tv->tv_usec == 0) { + tv->tv_usec = 1; + } } Py_LOCAL_INLINE(double) From d0aac5da59b1bbd113e6081d7c807ad5bced8a05 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 30 Jun 2017 13:12:12 +0200 Subject: [PATCH 0509/1345] bpo-30812: Fix test_warnings, restore _showwarnmsg (#2504) (#2507) bpo-26568, bpo-30812: Fix test_showwarnmsg_missing(): restore the attribute after removing it. (cherry picked from commit 7eebeb8fb84e2a9cb73903a08c59cf1d3b32cee0) --- Lib/test/test_warnings/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 0cddf4a2f4b10a..755ee65438cadd 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -727,10 +727,15 @@ def test_showwarnmsg_missing(self): text = 'del _showwarnmsg test' with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) - del self.module._showwarnmsg - with support.captured_output('stderr') as stream: - self.module.warn(text) - result = stream.getvalue() + + show = self.module._showwarnmsg + try: + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + finally: + self.module._showwarnmsg = show self.assertIn(text, result) def test_showwarning_not_callable(self): From 22d4e8fb99b16657eabfe7f9fee2d40a5ef882f6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 30 Jun 2017 13:12:20 +0200 Subject: [PATCH 0510/1345] bpo-30813: Fix unittest when hunting refleaks (#2502) (#2505) bpo-11798, bpo-16662, bpo-16935, bpo-30813: Skip test_discover_with_module_that_raises_SkipTest_on_import() and test_discover_with_init_module_that_raises_SkipTest_on_import() of test_unittest when hunting reference leaks using regrtest. (cherry picked from commit e4f9a2d2be42d5a2cdd624f8ed7cdf5028c5fbc3) --- Lib/unittest/test/test_discovery.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/unittest/test/test_discovery.py b/Lib/unittest/test/test_discovery.py index 1996a8ee5d3680..48d8fe95bce261 100644 --- a/Lib/unittest/test/test_discovery.py +++ b/Lib/unittest/test/test_discovery.py @@ -528,6 +528,9 @@ def _get_module_from_name(name): pickle.loads(pickle.dumps(test, proto)) def test_discover_with_module_that_raises_SkipTest_on_import(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + loader = unittest.TestLoader() def _get_module_from_name(name): @@ -548,6 +551,9 @@ def _get_module_from_name(name): pickle.loads(pickle.dumps(suite, proto)) def test_discover_with_init_module_that_raises_SkipTest_on_import(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + vfs = {abspath('/foo'): ['my_package'], abspath('/foo/my_package'): ['__init__.py', 'test_module.py']} self.setup_import_issue_package_tests(vfs) From 23caf8cfc6c52bdedd8e8db688d807530355fd6d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 30 Jun 2017 17:20:33 +0200 Subject: [PATCH 0511/1345] bpo-30280: Cleanup threads in ayncio tests (#2501) (#2511) * bpo-30280: asyncio now cleans up threads asyncio base TestCase now uses threading_setup() and threading_cleanup() of test.support to cleanup threads. * asyncio: Fix TestBaseSelectorEventLoop cleanup bpo-30280: TestBaseSelectorEventLoop of test.test_asyncio.test_selector_events now correctly closes the event loop: cleanup its executor to not leak threads. Don't override the close() method of the event loop, only override the_close_self_pipe() method. (cherry picked from commit b9030674624c181d6e9047cdb14ad65bb6c84c66) --- Lib/asyncio/test_utils.py | 6 ++++++ Lib/test/test_asyncio/test_selector_events.py | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index b12d5db2a9755d..94d48e1361a398 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -33,6 +33,7 @@ from . import tasks from .coroutines import coroutine from .log import logger +from test import support if sys.platform == 'win32': # pragma: no cover @@ -455,6 +456,7 @@ def unpatch_get_running_loop(self): def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None + self._thread_cleanup = support.threading_setup() def tearDown(self): self.unpatch_get_running_loop() @@ -465,6 +467,10 @@ def tearDown(self): # in an except block of a generator self.assertEqual(sys.exc_info(), (None, None, None)) + self.doCleanups() + support.threading_cleanup(*self._thread_cleanup) + support.reap_children() + if not compat.PY34: # Python 3.3 compatibility def subTest(self, *args, **kwargs): diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 6bf7862ecf9c0a..c50b3e49565c92 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,16 +24,14 @@ class TestBaseSelectorEventLoop(BaseSelectorEventLoop): - def close(self): - # Don't call the close() method of the parent class, because the - # selector is mocked - self._closed = True - def _make_self_pipe(self): self._ssock = mock.Mock() self._csock = mock.Mock() self._internal_fds += 1 + def _close_self_pipe(self): + pass + def list_to_buffer(l=()): return bytearray().join(l) From a3ca94d0504157a112a1f89bfe8be1307116fc73 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 30 Jun 2017 17:31:16 +0200 Subject: [PATCH 0512/1345] [3.6] bpo-29512, bpo-30776: Backport regrtest enhancements from master to 3.6 (#2513) * bpo-29512: Add test.bisect, bisect failing tests (#2452) Add a new "python3 -m test.bisect" tool to bisect failing tests. It can be used to find which test method(s) leak references, leak files, etc. (cherry picked from commit 84d9d14a1fa395fbd21262ba195490be25a7b3bc) * bpo-30776: regrtest: reduce memleak false positive (#2484) Only report a leak if each run leaks at least one memory block. (cherry picked from commit beeca6e1e5fd01531b1db7059498b13d07dca525) --- Lib/test/bisect.py | 167 ++++++++++++++++++++++++++++++++ Lib/test/libregrtest/refleak.py | 13 +-- 2 files changed, 170 insertions(+), 10 deletions(-) create mode 100755 Lib/test/bisect.py diff --git a/Lib/test/bisect.py b/Lib/test/bisect.py new file mode 100755 index 00000000000000..0fd577d4e5619f --- /dev/null +++ b/Lib/test/bisect.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Command line tool to bisect failing CPython tests. + +Find the test_os test method which alters the environment: + + ./python -m test.bisect --fail-env-changed test_os + +Find a reference leak in "test_os", write the list of failing tests into the +"bisect" file: + + ./python -m test.bisect -o bisect -R 3:3 test_os + +Load an existing list of tests from a file using -i option: + + ./python -m test --list-cases -m FileTests test_os > tests + ./python -m test.bisect -i tests test_os +""" + +import argparse +import datetime +import os.path +import math +import random +import subprocess +import sys +import tempfile +import time + + +def write_tests(filename, tests): + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + fp.flush() + + +def write_output(filename, tests): + if not filename: + return + print("Write %s tests into %s" % (len(tests), filename)) + write_tests(filename, tests) + return filename + + +def format_shell_args(args): + return ' '.join(args) + + +def list_cases(args): + cmd = [sys.executable, '-m', 'test', '--list-cases'] + cmd.extend(args.test_args) + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + universal_newlines=True) + exitcode = proc.returncode + if exitcode: + cmd = format_shell_args(cmd) + print("Failed to list tests: %s failed with exit code %s" + % (cmd, exitcode)) + sys.exit(exitcode) + tests = proc.stdout.splitlines() + return tests + + +def run_tests(args, tests, huntrleaks=None): + tmp = tempfile.mktemp() + try: + write_tests(tmp, tests) + + cmd = [sys.executable, '-m', 'test', '--matchfile', tmp] + cmd.extend(args.test_args) + print("+ %s" % format_shell_args(cmd)) + proc = subprocess.run(cmd) + return proc.returncode + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', + help='Test names produced by --list-tests written ' + 'into a file. If not set, run --list-tests') + parser.add_argument('-o', '--output', + help='Result of the bisection') + parser.add_argument('-n', '--max-tests', type=int, default=1, + help='Maximum number of tests to stop the bisection ' + '(default: 1)') + parser.add_argument('-N', '--max-iter', type=int, default=100, + help='Maximum number of bisection iterations ' + '(default: 100)') + # FIXME: document that following arguments are test arguments + + args, test_args = parser.parse_known_args() + args.test_args = test_args + return args + + +def main(): + args = parse_args() + + if args.input: + with open(args.input) as fp: + tests = [line.strip() for line in fp] + else: + tests = list_cases(args) + + print("Start bisection with %s tests" % len(tests)) + print("Test arguments: %s" % format_shell_args(args.test_args)) + print("Bisection will stop when getting %s or less tests " + "(-n/--max-tests option), or after %s iterations " + "(-N/--max-iter option)" + % (args.max_tests, args.max_iter)) + output = write_output(args.output, tests) + print() + + start_time = time.monotonic() + iteration = 1 + try: + while len(tests) > args.max_tests and iteration <= args.max_iter: + ntest = len(tests) + ntest = max(ntest // 2, 1) + subtests = random.sample(tests, ntest) + + print("[+] Iteration %s: run %s tests/%s" + % (iteration, len(subtests), len(tests))) + print() + + exitcode = run_tests(args, subtests) + + print("ran %s tests/%s" % (ntest, len(tests))) + print("exit", exitcode) + if exitcode: + print("Tests failed: use this new subtest") + tests = subtests + output = write_output(args.output, tests) + else: + print("Tests succeeded: skip this subtest, try a new subbset") + print() + iteration += 1 + except KeyboardInterrupt: + print() + print("Bisection interrupted!") + print() + + print("Tests (%s):" % len(tests)) + for test in tests: + print("* %s" % test) + print() + + if output: + print("Output written into %s" % output) + + dt = math.ceil(time.monotonic() - start_time) + if len(tests) <= args.max_tests: + print("Bisection completed in %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) + else: + print("Bisection failed after %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 8e18d75b217e6e..8e93816d965877 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -96,6 +96,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): # These checkers return False on success, True on failure def check_rc_deltas(deltas): + # Checker for reference counters and memomry blocks. + # # bpo-30776: Try to ignore false positives: # # [3, 0, 0] @@ -108,22 +110,13 @@ def check_rc_deltas(deltas): # [10, 1, 1] return all(delta >= 1 for delta in deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False - def check_fd_deltas(deltas): return any(deltas) failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas), + (alloc_deltas, 'memory blocks', check_rc_deltas), (fd_deltas, 'file descriptors', check_fd_deltas) ]: # ignore warmup runs From 48290c1c3023b2386b229f133b8629ffe5e7dd47 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 30 Jun 2017 18:15:18 -0700 Subject: [PATCH 0513/1345] [3.6] Fix trivial typo in json module docstring (GH-2274) (#2430) (cherry picked from commit 76c567ee27342d76f631a35c8291b715b2a61f3e) --- Lib/json/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index b8d5e6cff8cb21..6d0511ebfe90cf 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,7 @@ >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") + ... raise TypeError(repr(obj) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' From 3024c0529077f5cff0b32dc84b5923c8fba99a87 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 1 Jul 2017 19:12:05 +0200 Subject: [PATCH 0514/1345] [3.6] bpo-30703: Improve signal delivery (GH-2415) (#2527) * [3.6] bpo-30703: Improve signal delivery (GH-2415) * Improve signal delivery Avoid using Py_AddPendingCall from signal handler, to avoid calling signal-unsafe functions. * Remove unused function * Improve comments * Add stress test * Adapt for --without-threads * Add second stress test * Add NEWS blurb * Address comments @haypo. (cherry picked from commit c08177a1ccad2ed0d50898c2731b518c631aed14) * bpo-30796: Fix failures in signal delivery stress test (#2488) * bpo-30796: Fix failures in signal delivery stress test setitimer() can have a poor minimum resolution on some machines, this would make the test reach its deadline (and a stray signal could then kill a subsequent test). * Make sure to clear the itimer after the test --- Include/ceval.h | 1 + Lib/test/test_signal.py | 133 +++++++++++++++++- .../2017-06-28-21-07-32.bpo-30703.ULCdFp.rst | 6 + Modules/signalmodule.c | 30 ++-- Python/ceval.c | 75 +++++++--- 5 files changed, 207 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-06-28-21-07-32.bpo-30703.ULCdFp.rst diff --git a/Include/ceval.h b/Include/ceval.h index 1e482729a1cc98..38d470999f7350 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -46,6 +46,7 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); #endif PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); +PyAPI_FUNC(void) _PyEval_SignalReceived(void); PyAPI_FUNC(int) Py_MakePendingCalls(void); /* Protection against deeply nested recursive calls diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index aa2b8bb48cd328..8d6386793d4a57 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -3,11 +3,13 @@ from contextlib import closing import enum import gc +import os import pickle +import random import select import signal import socket -import struct +import statistics import subprocess import traceback import sys, os, time, errno @@ -955,6 +957,135 @@ def handler(signum, frame): (exitcode, stdout)) +class StressTest(unittest.TestCase): + """ + Stress signal delivery, especially when a signal arrives in + the middle of recomputing the signal state or executing + previously tripped signal handlers. + """ + + def setsig(self, signum, handler): + old_handler = signal.signal(signum, handler) + self.addCleanup(signal.signal, signum, old_handler) + + def measure_itimer_resolution(self): + N = 20 + times = [] + + def handler(signum=None, frame=None): + if len(times) < N: + times.append(time.perf_counter()) + # 1 µs is the smallest possible timer interval, + # we want to measure what the concrete duration + # will be on this platform + signal.setitimer(signal.ITIMER_REAL, 1e-6) + + self.addCleanup(signal.setitimer, signal.ITIMER_REAL, 0) + self.setsig(signal.SIGALRM, handler) + handler() + while len(times) < N: + time.sleep(1e-3) + + durations = [times[i+1] - times[i] for i in range(len(times) - 1)] + med = statistics.median(durations) + if support.verbose: + print("detected median itimer() resolution: %.6f s." % (med,)) + return med + + def decide_itimer_count(self): + # Some systems have poor setitimer() resolution (for example + # measured around 20 ms. on FreeBSD 9), so decide on a reasonable + # number of sequential timers based on that. + reso = self.measure_itimer_resolution() + if reso <= 1e-4: + return 10000 + elif reso <= 1e-2: + return 100 + else: + self.skipTest("detected itimer resolution (%.3f s.) too high " + "(> 10 ms.) on this platform (or system too busy)" + % (reso,)) + + @unittest.skipUnless(hasattr(signal, "setitimer"), + "test needs setitimer()") + def test_stress_delivery_dependent(self): + """ + This test uses dependent signal handlers. + """ + N = self.decide_itimer_count() + sigs = [] + + def first_handler(signum, frame): + # 1e-6 is the minimum non-zero value for `setitimer()`. + # Choose a random delay so as to improve chances of + # triggering a race condition. Ideally the signal is received + # when inside critical signal-handling routines such as + # Py_MakePendingCalls(). + signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) + + def second_handler(signum=None, frame=None): + sigs.append(signum) + + # Here on Linux, SIGPROF > SIGALRM > SIGUSR1. By using both + # ascending and descending sequences (SIGUSR1 then SIGALRM, + # SIGPROF then SIGALRM), we maximize chances of hitting a bug. + self.setsig(signal.SIGPROF, first_handler) + self.setsig(signal.SIGUSR1, first_handler) + self.setsig(signal.SIGALRM, second_handler) # for ITIMER_REAL + + expected_sigs = 0 + deadline = time.time() + 15.0 + + while expected_sigs < N: + os.kill(os.getpid(), signal.SIGPROF) + expected_sigs += 1 + # Wait for handlers to run to avoid signal coalescing + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + os.kill(os.getpid(), signal.SIGUSR1) + expected_sigs += 1 + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + # All ITIMER_REAL signals should have been delivered to the + # Python handler + self.assertEqual(len(sigs), N, "Some signals were lost") + + @unittest.skipUnless(hasattr(signal, "setitimer"), + "test needs setitimer()") + def test_stress_delivery_simultaneous(self): + """ + This test uses simultaneous signal handlers. + """ + N = self.decide_itimer_count() + sigs = [] + + def handler(signum, frame): + sigs.append(signum) + + self.setsig(signal.SIGUSR1, handler) + self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL + + expected_sigs = 0 + deadline = time.time() + 15.0 + + while expected_sigs < N: + # Hopefully the SIGALRM will be received somewhere during + # initial processing of SIGUSR1. + signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) + os.kill(os.getpid(), signal.SIGUSR1) + + expected_sigs += 2 + # Wait for handlers to run to avoid signal coalescing + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + # All ITIMER_REAL signals should have been delivered to the + # Python handler + self.assertEqual(len(sigs), N, "Some signals were lost") + + def tearDownModule(): support.reap_children() diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-06-28-21-07-32.bpo-30703.ULCdFp.rst b/Misc/NEWS.d/next/Core and Builtins/2017-06-28-21-07-32.bpo-30703.ULCdFp.rst new file mode 100644 index 00000000000000..9adeb450aeeb72 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-06-28-21-07-32.bpo-30703.ULCdFp.rst @@ -0,0 +1,6 @@ +Improve signal delivery. + +Avoid using Py_AddPendingCall from signal handler, to avoid calling signal- +unsafe functions. The tests I'm adding here fail without the rest of the +patch, on Linux and OS X. This means our signal delivery logic had defects +(some signals could be lost). diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 55e4c2a00e9ef3..30990f89c3eeaf 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -192,12 +192,6 @@ The default handler for SIGINT installed by Python.\n\ It raises KeyboardInterrupt."); -static int -checksignals_witharg(void * unused) -{ - return PyErr_CheckSignals(); -} - static int report_wakeup_write_error(void *data) { @@ -248,17 +242,15 @@ trip_signal(int sig_num) Handlers[sig_num].tripped = 1; - if (!is_tripped) { - /* Set is_tripped after setting .tripped, as it gets - cleared in PyErr_CheckSignals() before .tripped. */ - is_tripped = 1; - Py_AddPendingCall(checksignals_witharg, NULL); - } + /* Set is_tripped after setting .tripped, as it gets + cleared in PyErr_CheckSignals() before .tripped. */ + is_tripped = 1; + _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and - doing the Py_AddPendingCall. We used to write to the wakeup fd and then - set the flag, but this allowed the following sequence of events - (especially on windows, where trip_signal runs in a new thread): + doing the _PyEval_SignalReceived. We used to write to the wakeup fd + and then set the flag, but this allowed the following sequence of events + (especially on windows, where trip_signal may run in a new thread): - main thread blocks on select([wakeup_fd], ...) - signal arrives @@ -293,6 +285,8 @@ trip_signal(int sig_num) wakeup.send_err_set = 1; wakeup.send_errno = errno; wakeup.send_win_error = GetLastError(); + /* Py_AddPendingCall() isn't signal-safe, but we + still use it for this exceptional case. */ Py_AddPendingCall(report_wakeup_send_error, NULL); } } @@ -306,6 +300,8 @@ trip_signal(int sig_num) rc = _Py_write_noraise(fd, &byte, 1); if (rc < 0) { + /* Py_AddPendingCall() isn't signal-safe, but we + still use it for this exceptional case. */ Py_AddPendingCall(report_wakeup_write_error, (void *)(intptr_t)errno); } @@ -1555,8 +1551,10 @@ PyErr_CheckSignals(void) arglist); Py_DECREF(arglist); } - if (!result) + if (!result) { + is_tripped = 1; return -1; + } Py_DECREF(result); } diff --git a/Python/ceval.c b/Python/ceval.c index ea79f5f2b57f8f..bf9103dd9aa4e3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -195,6 +195,15 @@ PyEval_GetCallStats(PyObject *self) do { pending_async_exc = 0; COMPUTE_EVAL_BREAKER(); } while (0) +/* This single variable consolidates all requests to break out of the fast path + in the eval loop. */ +static _Py_atomic_int eval_breaker = {0}; +/* Request for running pending calls. */ +static _Py_atomic_int pendingcalls_to_do = {0}; +/* Request for looking at the `async_exc` field of the current thread state. + Guarded by the GIL. */ +static int pending_async_exc = 0; + #ifdef WITH_THREAD #ifdef HAVE_ERRNO_H @@ -204,16 +213,8 @@ PyEval_GetCallStats(PyObject *self) static PyThread_type_lock pending_lock = 0; /* for pending calls */ static long main_thread = 0; -/* This single variable consolidates all requests to break out of the fast path - in the eval loop. */ -static _Py_atomic_int eval_breaker = {0}; /* Request for dropping the GIL */ static _Py_atomic_int gil_drop_request = {0}; -/* Request for running pending calls. */ -static _Py_atomic_int pendingcalls_to_do = {0}; -/* Request for looking at the `async_exc` field of the current thread state. - Guarded by the GIL. */ -static int pending_async_exc = 0; #include "ceval_gil.h" @@ -326,9 +327,6 @@ PyEval_ReInitThreads(void) _PyThreadState_DeleteExcept(current_tstate); } -#else -static _Py_atomic_int eval_breaker = {0}; -static int pending_async_exc = 0; #endif /* WITH_THREAD */ /* This function is used to signal that async exceptions are waiting to be @@ -403,6 +401,15 @@ PyEval_RestoreThread(PyThreadState *tstate) #endif */ +void +_PyEval_SignalReceived(void) +{ + /* bpo-30703: Function called when the C signal handler of Python gets a + signal. We cannot queue a callback using Py_AddPendingCall() since + that function is not async-signal-safe. */ + SIGNAL_PENDING_CALLS(); +} + #ifdef WITH_THREAD /* The WITH_THREAD implementation is thread-safe. It allows @@ -467,6 +474,8 @@ Py_MakePendingCalls(void) int i; int r = 0; + assert(PyGILState_Check()); + if (!pending_lock) { /* initial allocation of the lock */ pending_lock = PyThread_allocate_lock(); @@ -481,6 +490,16 @@ Py_MakePendingCalls(void) if (busy) return 0; busy = 1; + /* unsignal before starting to call callbacks, so that any callback + added in-between re-signals */ + UNSIGNAL_PENDING_CALLS(); + + /* Python signal handler doesn't really queue a callback: it only signals + that a signal was received, see _PyEval_SignalReceived(). */ + if (PyErr_CheckSignals() < 0) { + goto error; + } + /* perform a bounded number of calls, in case of recursion */ for (i=0; i Date: Sun, 2 Jul 2017 11:48:07 +0900 Subject: [PATCH 0515/1345] [3.6] bpo-29933: Improve set_write_buffer_limits description (GH-2262) (GH-2532) Improve the description of the high and low parameters for set_write_buffer_limits. Also fixed a small grammatical issue. (cherry picked from commit 5200a7c7f9ea65a96330c5f276f4acc6ec70854e) --- Doc/library/asyncio-protocol.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3f55506c669a29..dfe37c239d6b39 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -163,11 +163,17 @@ WriteTransport Set the *high*- and *low*-water limits for write flow control. - These two values control when call the protocol's + These two values (measured in number of + bytes) control when the protocol's :meth:`pause_writing` and :meth:`resume_writing` methods are called. If specified, the low-water limit must be less than or equal to the high-water limit. Neither *high* nor *low* can be negative. + :meth:`pause_writing` is called when the buffer size becomes greater + than or equal to the *high* value. If writing has been paused, + :meth:`resume_writing` is called when the buffer size becomes less + than or equal to the *low* value. + The defaults are implementation-specific. If only the high-water limit is given, the low-water limit defaults to an implementation-specific value less than or equal to the From bfdc6fdc0e76204caf2261f9cfa9d232fc23906a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 3 Jul 2017 17:49:50 +1000 Subject: [PATCH 0516/1345] [3.6] bpo-30597: Show expected input in custom 'print' error message. (GH-2531) (cherry picked from commit 3a7f03584ab75afbf5507970711c87042e423bb4) --- Lib/test/test_print.py | 28 ++++++++++++++++++++++++ Misc/NEWS | 3 +++ Objects/exceptions.c | 48 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 7eea349115a947..03f13b4edfc598 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -128,5 +128,33 @@ def flush(self): raise RuntimeError self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True) + +class TestPy2MigrationHint(unittest.TestCase): + """Test that correct hint is produced analogous to Python3 syntax, + if print statement is executed as in Python 2. + """ + + def test_normal_string(self): + python2_print_str = 'print "Hello World"' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World")', str(context.exception)) + + def test_string_with_soft_space(self): + python2_print_str = 'print "Hello World",' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World", end=" ")', str(context.exception)) + + def test_string_with_excessive_whitespace(self): + python2_print_str = 'print "Hello World", ' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World", end=" ")', str(context.exception)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index a3a32ae6c60810..92ccf52149aac1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.3 release candidate 1? Core and Builtins ----------------- +- bpo-30597: ``print`` now shows expected input in custom error message when + used as a Python 2 statement. Patch by Sanyam Khurana. + Library ------- diff --git a/Objects/exceptions.c b/Objects/exceptions.c index d158b9768d45f5..f286ec076f8286 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2881,6 +2881,49 @@ _PyErr_TrySetFromCause(const char *format, ...) * or minus, using the stream redirection syntax). */ + +// Static helper for setting legacy print error message +static int +_set_legacy_print_statement_msg(PySyntaxErrorObject *self, Py_ssize_t start) +{ + PyObject *strip_sep_obj = PyUnicode_FromString(" \t\r\n"); + if (strip_sep_obj == NULL) + return -1; + + // PRINT_OFFSET is to remove `print ` word from the data. + const int PRINT_OFFSET = 6; + Py_ssize_t text_len = PyUnicode_GET_LENGTH(self->text); + PyObject *data = PyUnicode_Substring(self->text, PRINT_OFFSET, text_len); + + if (data == NULL) { + Py_DECREF(strip_sep_obj); + return -1; + } + PyObject *new_data = _PyUnicode_XStrip(data, 2, strip_sep_obj); + Py_DECREF(data); + Py_DECREF(strip_sep_obj); + + if (new_data == NULL) { + return -1; + } + // gets the modified text_len after stripping `print ` + text_len = PyUnicode_GET_LENGTH(new_data); + const char *maybe_end_arg = ""; + if (text_len > 0 && PyUnicode_READ_CHAR(new_data, text_len-1) == ',') { + maybe_end_arg = " end=\" \""; + } + PyObject *error_msg = PyUnicode_FromFormat( + "Missing parentheses in call to 'print'. Did you mean print(%U%s)?", + new_data, maybe_end_arg + ); + Py_DECREF(new_data); + if (error_msg == NULL) + return -1; + + Py_XSETREF(self->msg, error_msg); + return 1; +} + static int _check_for_legacy_statements(PySyntaxErrorObject *self, Py_ssize_t start) { @@ -2916,9 +2959,8 @@ _check_for_legacy_statements(PySyntaxErrorObject *self, Py_ssize_t start) } if (PyUnicode_Tailmatch(self->text, print_prefix, start, text_len, -1)) { - Py_XSETREF(self->msg, - PyUnicode_FromString("Missing parentheses in call to 'print'")); - return 1; + + return _set_legacy_print_statement_msg(self, start); } /* Check for legacy exec statements */ From 4132adb4b8f90893d8f19ff46edff56f189faca0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jul 2017 12:50:36 +0200 Subject: [PATCH 0517/1345] bpo-30764: Fix regrtest --fail-env-changed --forever (#2536) (#2539) --forever now stops if a fail changes the environment. (cherry picked from commit 5e87592fd12e0b7c41edc11d4885ed7298d5063b) --- Lib/test/libregrtest/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 571eb6112520eb..bc8155bbff2b47 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -415,6 +415,8 @@ def _test_forever(self, tests): yield test if self.bad: return + if self.ns.fail_env_changed and self.environment_changed: + return def display_header(self): # Print basic platform information @@ -478,7 +480,7 @@ def finalize(self): result = "FAILURE" elif self.interrupted: result = "INTERRUPTED" - elif self.environment_changed and self.ns.fail_env_changed: + elif self.ns.fail_env_changed and self.environment_changed: result = "ENV CHANGED" else: result = "SUCCESS" From bebd2cfa5f21811dd0ee4f3b1a1b85d379b83436 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Jul 2017 07:55:32 +0300 Subject: [PATCH 0518/1345] [3.6] bpo-30441: Fix bug when modifying os.environ while iterating over it (GH-2409). (#2556) (cherry picked from commit 8a8d28501fc8ce25926d168f1c657656c809fd4c) --- Lib/os.py | 4 +++- Lib/test/test_os.py | 24 +++++++++++++++++++ Misc/ACKS | 1 + .../2017-06-29-14-25-14.bpo-30441.3Wh9kc.rst | 1 + 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-06-29-14-25-14.bpo-30441.3Wh9kc.rst diff --git a/Lib/os.py b/Lib/os.py index fa06f3937ba505..9fa8acb4b131ef 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -685,7 +685,9 @@ def __delitem__(self, key): raise KeyError(key) from None def __iter__(self): - for key in self._data: + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: yield self.decodekey(key) def __len__(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index fdeb9a9647b819..f74c199c84146a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -836,6 +836,30 @@ def test_key_type(self): self.assertIs(cm.exception.args[0], missing) self.assertTrue(cm.exception.__suppress_context__) + def _test_environ_iteration(self, collection): + iterator = iter(collection) + new_key = "__new_key__" + + next(iterator) # start iteration over os.environ.items + + # add a new key in os.environ mapping + os.environ[new_key] = "test_environ_iteration" + + try: + next(iterator) # force iteration over modified mapping + self.assertEqual(os.environ[new_key], "test_environ_iteration") + finally: + del os.environ[new_key] + + def test_iter_error_when_changing_os_environ(self): + self._test_environ_iteration(os.environ) + + def test_iter_error_when_changing_os_environ_items(self): + self._test_environ_iteration(os.environ.items()) + + def test_iter_error_when_changing_os_environ_values(self): + self._test_environ_iteration(os.environ.values()) + class WalkTests(unittest.TestCase): """Tests for os.walk().""" diff --git a/Misc/ACKS b/Misc/ACKS index c4af869c74cf6b..3420d4b6894f48 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1077,6 +1077,7 @@ Vilmos Nebehaj Fredrik Nehr Tony Nelson Trent Nelson +Osvaldo Santana Neto Chad Netzer Max Neunhöffer Anthon van der Neut diff --git a/Misc/NEWS.d/next/Library/2017-06-29-14-25-14.bpo-30441.3Wh9kc.rst b/Misc/NEWS.d/next/Library/2017-06-29-14-25-14.bpo-30441.3Wh9kc.rst new file mode 100644 index 00000000000000..55dd6136c8d708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-06-29-14-25-14.bpo-30441.3Wh9kc.rst @@ -0,0 +1 @@ +Fix bug when modifying os.environ while iterating over it From 23e2c3d193eafa8d16d895a2e59b6f6571e50cd3 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Tue, 4 Jul 2017 09:27:35 +0300 Subject: [PATCH 0519/1345] Remove outdated FOX from GUI FAQ (GH-2538) FXpy doesn't have a Python 3 port and it only supports Python 2.2 and older versions. Reported by Alex Walters on docs@p.o. (cherry picked from commit d3ed2877a798d07df75422afe136b4727e500c99) --- Doc/faq/gui.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 477d8c633438d1..38e1796267ff3a 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -94,15 +94,6 @@ Python bindings for `the FLTK toolkit `_, a simple yet powerful and mature cross-platform windowing system, are available from `the PyFLTK project `_. - -FOX ----- - -A wrapper for `the FOX toolkit `_ called `FXpy -`_ is available. FOX supports both Unix variants -and Windows. - - OpenGL ------ From 580cd5cd3603317d294a881628880a92ea221334 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Jul 2017 16:46:10 +0200 Subject: [PATCH 0520/1345] bpo-30319: socket.close() now ignores ECONNRESET (#2565) (#2566) socket.close() was modified in Python 3.6 to raise OSError on failure: see bpo-26685. (cherry picked from commit 67e1478dba6efe60b8e1890192014b8b06dd6bd9) --- .../next/Library/2017-07-04-13-48-21.bpo-30319.hg_3TX.rst | 1 + Modules/socketmodule.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-04-13-48-21.bpo-30319.hg_3TX.rst diff --git a/Misc/NEWS.d/next/Library/2017-07-04-13-48-21.bpo-30319.hg_3TX.rst b/Misc/NEWS.d/next/Library/2017-07-04-13-48-21.bpo-30319.hg_3TX.rst new file mode 100644 index 00000000000000..1112d2f373753c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-04-13-48-21.bpo-30319.hg_3TX.rst @@ -0,0 +1 @@ +socket.close() now ignores ECONNRESET error. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 42aec59ca7f6e5..d006bd4def3ce4 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2704,7 +2704,9 @@ sock_close(PySocketSockObject *s) Py_BEGIN_ALLOW_THREADS res = SOCKETCLOSE(fd); Py_END_ALLOW_THREADS - if (res < 0) { + /* bpo-30319: The peer can already have closed the connection. + Python ignores ECONNRESET on close(). */ + if (res < 0 && errno != ECONNRESET) { return s->errorhandler(); } } From d02c8416fe1b29b3322004b73133bf6c8a2e353a Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Tue, 4 Jul 2017 21:38:47 +0300 Subject: [PATCH 0521/1345] [3.6] bpo-30726: Also fix pyexpat.vcxproj (GH-2375) (#2570) (cherry picked from commit 7526cadd64566725ffc56071a7208828a46ddbd8) --- PCbuild/pyexpat.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 99c7286b2bdfde..a2763f02063cd5 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -59,7 +59,7 @@ $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) - PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions) From e5bb112ba2048cb3ea19481a7b38fe1433c5a7ef Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 5 Jul 2017 00:54:55 -0400 Subject: [PATCH 0522/1345] [3.6] bpo-30777: IDLE: configdialog - add docstrings and improve comments (GH-2440) (#2577) Patch by Cheryl Sabella. (cherry picked from commit 7eb5883) --- Lib/idlelib/configdialog.py | 425 +++++++++++++++++++++++++----------- 1 file changed, 297 insertions(+), 128 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 4a25846238f6e1..84092275183685 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -28,10 +28,16 @@ from idlelib.tabbedpages import TabbedPageSet from idlelib.textview import view_text + class ConfigDialog(Toplevel): + """Config dialog for IDLE. + """ def __init__(self, parent, title='', _htest=False, _utest=False): - """ + """Show the tabbed dialog for user configuration. + + parent - parent of this dialog + title - string which is the title of this popup dialog _htest - bool, change box location when running htest _utest - bool, don't wait_window when running unittest """ @@ -46,9 +52,9 @@ def __init__(self, parent, title='', _htest=False, _utest=False): x = parent.winfo_rootx() + 20 y = parent.winfo_rooty() + (30 if not _htest else 150) self.geometry(f'+{x}+{y}') - #Theme Elements. Each theme element key is its display name. - #The first value of the tuple is the sample area tag name. - #The second value is the display name list sort index. + # Each theme element key is its display name. + # The first value of the tuple is the sample area tag name. + # The second value is the display name list sort index. self.theme_elements={ 'Normal Text': ('normal', '00'), 'Python Keywords': ('keyword', '01'), @@ -65,25 +71,27 @@ def __init__(self, parent, title='', _htest=False, _utest=False): 'Shell Stdout Text': ('stdout', '12'), 'Shell Stderr Text': ('stderr', '13'), } - self.reset_changed_items() #load initial values in changed items dict + self.reset_changed_items() # Initialize changed_items dict. self.create_widgets() self.resizable(height=FALSE, width=FALSE) self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.cancel) self.tab_pages.focus_set() - #key bindings for this dialog - #self.bind('', self.Cancel) #dismiss dialog, no save - #self.bind('', self.Apply) #apply changes, save - #self.bind('', self.Help) #context help + # XXX Decide whether to keep or delete these key bindings. + # Key bindings for this dialog. + # self.bind('', self.Cancel) #dismiss dialog, no save + # self.bind('', self.Apply) #apply changes, save + # self.bind('', self.Help) #context help self.load_configs() - self.attach_var_callbacks() #avoid callbacks during load_configs + self.attach_var_callbacks() # Avoid callbacks during load_configs. if not _utest: self.wm_deiconify() self.wait_window() def create_widgets(self): + "Create and place widgets for tabbed dialog." self.tab_pages = TabbedPageSet(self, page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', 'Extensions']) @@ -96,9 +104,10 @@ def create_widgets(self): self.create_action_buttons().pack(side=BOTTOM) def create_action_buttons(self): + "Return frame of action buttons for dialog." if macosx.isAquaTk(): # Changing the default padding on OSX results in unreadable - # text in the buttons + # text in the buttons. padding_args = {} else: padding_args = {'padx':6, 'pady':3} @@ -111,12 +120,21 @@ def create_action_buttons(self): ('Help', self.help)): Button(buttons, text=txt, command=cmd, takefocus=FALSE, **padding_args).pack(side=LEFT, padx=5) - # add space above buttons + # Add space above buttons. Frame(outer, height=2, borderwidth=0).pack(side=TOP) buttons.pack(side=BOTTOM) return outer def create_page_font_tab(self): + """Return frame of widgets for Font/Tabs tab. + + Configuration attributes: + font_size: Font size. + font_bold: Select font bold or not. + font_name: Font face. + space_num: Indentation width. + edit_font: Font widget with default font name, size, and weight. + """ parent = self.parent self.font_size = StringVar(parent) self.font_bold = BooleanVar(parent) @@ -185,11 +203,22 @@ def create_page_font_tab(self): return frame def create_page_highlight(self): + """Return frame of widgets for Highlighting tab. + + Configuration attributes: + builtin_theme: Menu variable for built-in theme. + custom_theme: Menu variable for custom theme. + fg_bg_toggle: Toggle for foreground/background color. + colour: Color of selected target. + is_builtin_theme: Selector for built-in or custom theme. + highlight_target: Menu variable for the highlight tag target. + """ parent = self.parent self.builtin_theme = StringVar(parent) self.custom_theme = StringVar(parent) self.fg_bg_toggle = BooleanVar(parent) self.colour = StringVar(parent) + # XXX - font_name is defined in create_page_font_tab. Needed here too? self.font_name = StringVar(parent) self.is_builtin_theme = BooleanVar(parent) self.highlight_target = StringVar(parent) @@ -296,7 +325,16 @@ def tem(event, elem=element): return frame def create_page_keys(self): + """Return frame of widgets for Keys tab. + + Configuration attributes: + builtin_keys: Menu variable for built-in keybindings. + custom_keys: Menu variable for custom keybindings. + are_keys_builtin: Selector for built-in or custom keybindings. + keybinding: Action/key bindings. + """ parent = self.parent + # XXX - binding_target isn't used. self.binding_target = StringVar(parent) self.builtin_keys = StringVar(parent) self.custom_keys = StringVar(parent) @@ -375,12 +413,23 @@ def create_page_keys(self): return frame def create_page_general(self): + """Return frame of widgets for General tab. + + Configuration attributes: + win_width: Initial window width in characters. + win_height: Initial window height in characters. + startup_edit: Selector for opening in editor or shell mode. + autosave: Selector for save prompt popup when using Run. + encoding: ? + """ parent = self.parent self.win_width = StringVar(parent) self.win_height = StringVar(parent) self.startup_edit = IntVar(parent) self.autosave = IntVar(parent) + # XXX - encoding isn't on the screen to be set, but is saved to config. self.encoding = StringVar(parent) + # XXX - user_help_browser and help_browser aren't used. self.user_help_browser = BooleanVar(parent) self.help_browser = StringVar(parent) @@ -471,6 +520,7 @@ def create_page_general(self): return frame def attach_var_callbacks(self): + "Attach callbacks to variables that can be changed." self.font_size.trace_add('write', self.var_changed_font) self.font_name.trace_add('write', self.var_changed_font) self.font_bold.trace_add('write', self.var_changed_font) @@ -502,10 +552,12 @@ def remove_var_callbacks(self): var.trace_remove('write', var.trace_info()[0][1]) def var_changed_font(self, *params): - '''When one font attribute changes, save them all, as they are + """Store changes to font attributes. + + When one font attribute changes, save them all, as they are not independent from each other. In particular, when we are overriding the default font, we need to write out everything. - ''' + """ value = self.font_name.get() self.add_changed_item('main', 'EditorWindow', 'font', value) value = self.font_size.get() @@ -514,13 +566,20 @@ def var_changed_font(self, *params): self.add_changed_item('main', 'EditorWindow', 'font-bold', value) def var_changed_space_num(self, *params): + "Store change to indentation size." value = self.space_num.get() self.add_changed_item('main', 'Indent', 'num-spaces', value) def var_changed_colour(self, *params): + "Process change to color choice." self.on_new_colour_set() def var_changed_builtin_theme(self, *params): + """Process new builtin theme selection. + + Add the changed theme's name to the changed_items and recreate + the sample with the values from the selected theme. + """ old_themes = ('IDLE Classic', 'IDLE New') value = self.builtin_theme.get() if value not in old_themes: @@ -536,12 +595,22 @@ def var_changed_builtin_theme(self, *params): self.paint_theme_sample() def var_changed_custom_theme(self, *params): + """Process new custom theme selection. + + If a new custom theme is selected, add the name to the + changed_items and apply the theme to the sample. + """ value = self.custom_theme.get() if value != '- no custom themes -': self.add_changed_item('main', 'Theme', 'name', value) self.paint_theme_sample() def var_changed_is_builtin_theme(self, *params): + """Process toggle between builtin and custom theme. + + Update the default toggle value and apply the newly + selected theme type. + """ value = self.is_builtin_theme.get() self.add_changed_item('main', 'Theme', 'default', value) if value: @@ -550,21 +619,23 @@ def var_changed_is_builtin_theme(self, *params): self.var_changed_custom_theme() def var_changed_highlight_target(self, *params): + "Process selection of new target tag for highlighting." self.set_highlight_target() def var_changed_keybinding(self, *params): + "Store change to a keybinding." value = self.keybinding.get() key_set = self.custom_keys.get() event = self.list_bindings.get(ANCHOR).split()[0] if idleConf.IsCoreBinding(event): - #this is a core keybinding self.add_changed_item('keys', key_set, event, value) - else: #this is an extension key binding + else: # Event is an extension binding. ext_name = idleConf.GetExtnNameForEvent(event) ext_keybind_section = ext_name + '_cfgBindings' self.add_changed_item('extensions', ext_keybind_section, event, value) def var_changed_builtin_keys(self, *params): + "Process selection of builtin key set." old_keys = ( 'IDLE Classic Windows', 'IDLE Classic Unix', @@ -585,12 +656,14 @@ def var_changed_builtin_keys(self, *params): self.load_keys_list(value) def var_changed_custom_keys(self, *params): + "Process selection of custom key set." value = self.custom_keys.get() if value != '- no custom keys -': self.add_changed_item('main', 'Keys', 'name', value) self.load_keys_list(value) def var_changed_are_keys_builtin(self, *params): + "Process toggle between builtin key set and custom key set." value = self.are_keys_builtin.get() self.add_changed_item('main', 'Keys', 'default', value) if value: @@ -599,41 +672,51 @@ def var_changed_are_keys_builtin(self, *params): self.var_changed_custom_keys() def var_changed_win_width(self, *params): + "Store change to window width." value = self.win_width.get() self.add_changed_item('main', 'EditorWindow', 'width', value) def var_changed_win_height(self, *params): + "Store change to window height." value = self.win_height.get() self.add_changed_item('main', 'EditorWindow', 'height', value) def var_changed_startup_edit(self, *params): + "Store change to toggle for starting IDLE in the editor or shell." value = self.startup_edit.get() self.add_changed_item('main', 'General', 'editor-on-startup', value) def var_changed_autosave(self, *params): + "Store change to autosave." value = self.autosave.get() self.add_changed_item('main', 'General', 'autosave', value) def var_changed_encoding(self, *params): + "Store change to encoding." value = self.encoding.get() self.add_changed_item('main', 'EditorWindow', 'encoding', value) def reset_changed_items(self): - #When any config item is changed in this dialog, an entry - #should be made in the relevant section (config type) of this - #dictionary. The key should be the config file section name and the - #value a dictionary, whose key:value pairs are item=value pairs for - #that config file section. + """Reset dictionary containing the items changed on each tab. + + When any config item is changed in this dialog, an entry + should be made in the relevant section (config type) of this + dictionary. The key should be the config file section name and the + value a dictionary, whose key:value pairs are item=value pairs for + that config file section. + """ self.changed_items = {'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} def add_changed_item(self, typ, section, item, value): - value = str(value) #make sure we use a string + "Add item/value pair to changed items dictionary for typ and section." + value = str(value) # Make sure we use a string. if section not in self.changed_items[typ]: self.changed_items[typ][section] = {} self.changed_items[typ][section][item] = value def GetDefaultItems(self): + "Return dictionary of default configuration settings." d_items={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} for config_type in d_items: sections = idleConf.GetSectionList('default', config_type) @@ -646,6 +729,7 @@ def GetDefaultItems(self): return d_items def set_theme_type(self): + "Set available screen options based on builtin or custom theme." if self.is_builtin_theme.get(): self.opt_menu_theme_builtin.config(state=NORMAL) self.opt_menu_theme_custom.config(state=DISABLED) @@ -657,6 +741,7 @@ def set_theme_type(self): self.button_delete_custom_theme.config(state=NORMAL) def set_keys_type(self): + "Set available screen options based on builtin or custom key set." if self.are_keys_builtin.get(): self.opt_menu_keys_builtin.config(state=NORMAL) self.opt_menu_keys_custom.config(state=DISABLED) @@ -668,31 +753,39 @@ def set_keys_type(self): self.button_delete_custom_keys.config(state=NORMAL) def get_new_keys(self): + """Handle event to change key binding for selected line. + + A selection of a key/binding in the list of current + bindings pops up a dialog to enter a new binding. If + the current key set is builtin and a binding has + changed, then a name for a custom key set needs to be + entered for the change to be applied. + """ list_index = self.list_bindings.index(ANCHOR) binding = self.list_bindings.get(list_index) - bind_name = binding.split()[0] #first part, up to first space + bind_name = binding.split()[0] if self.are_keys_builtin.get(): current_key_set_name = self.builtin_keys.get() else: current_key_set_name = self.custom_keys.get() current_bindings = idleConf.GetCurrentKeySet() - if current_key_set_name in self.changed_items['keys']: #unsaved changes + if current_key_set_name in self.changed_items['keys']: # unsaved changes key_set_changes = self.changed_items['keys'][current_key_set_name] for event in key_set_changes: current_bindings[event] = key_set_changes[event].split() current_key_sequences = list(current_bindings.values()) new_keys = GetKeysDialog(self, 'Get New Keys', bind_name, current_key_sequences).result - if new_keys: #new keys were specified - if self.are_keys_builtin.get(): #current key set is a built-in + if new_keys: + if self.are_keys_builtin.get(): # Current key set is a built-in. message = ('Your changes will be saved as a new Custom Key Set.' ' Enter a name for your new Custom Key Set below.') new_keyset = self.get_new_keys_name(message) - if not new_keyset: #user cancelled custom key set creation + if not new_keyset: # User cancelled custom key set creation. self.list_bindings.select_set(list_index) self.list_bindings.select_anchor(list_index) return - else: #create new custom key set based on previously active key set + else: # Create new custom key set based on previously active key set. self.create_new_key_set(new_keyset) self.list_bindings.delete(list_index) self.list_bindings.insert(list_index, bind_name+' - '+new_keys) @@ -704,6 +797,7 @@ def get_new_keys(self): self.list_bindings.select_anchor(list_index) def get_new_keys_name(self, message): + "Return new key set name from query popup." used_names = (idleConf.GetSectionList('user', 'keys') + idleConf.GetSectionList('default', 'keys')) new_keyset = SectionName( @@ -711,34 +805,40 @@ def get_new_keys_name(self, message): return new_keyset def save_as_new_key_set(self): + "Prompt for name of new key set and save changes using that name." new_keys_name = self.get_new_keys_name('New Key Set Name:') if new_keys_name: self.create_new_key_set(new_keys_name) def keybinding_selected(self, event): + "Activate button to assign new keys to selected action." self.button_new_keys.config(state=NORMAL) def create_new_key_set(self, new_key_set_name): - #creates new custom key set based on the previously active key set, - #and makes the new key set active + """Create a new custom key set with the given name. + + Create the new key set based on the previously active set + with the current changes applied. Once it is saved, then + activate the new key set. + """ if self.are_keys_builtin.get(): prev_key_set_name = self.builtin_keys.get() else: prev_key_set_name = self.custom_keys.get() prev_keys = idleConf.GetCoreKeys(prev_key_set_name) new_keys = {} - for event in prev_keys: #add key set to changed items - event_name = event[2:-2] #trim off the angle brackets + for event in prev_keys: # Add key set to changed items. + event_name = event[2:-2] # Trim off the angle brackets. binding = ' '.join(prev_keys[event]) new_keys[event_name] = binding - #handle any unsaved changes to prev key set + # Handle any unsaved changes to prev key set. if prev_key_set_name in self.changed_items['keys']: key_set_changes = self.changed_items['keys'][prev_key_set_name] for event in key_set_changes: new_keys[event] = key_set_changes[event] - #save the new theme + # Save the new key set. self.save_new_key_set(new_key_set_name, new_keys) - #change gui over to the new key set + # Change GUI over to the new key set. custom_key_list = idleConf.GetSectionList('user', 'keys') custom_key_list.sort() self.opt_menu_keys_custom.SetMenu(custom_key_list, new_key_set_name) @@ -746,7 +846,12 @@ def create_new_key_set(self, new_key_set_name): self.set_keys_type() def load_keys_list(self, keyset_name): + """Reload the list of action/key binding pairs for the active key set. + + An action/key binding can be selected to change the key binding. + """ reselect = 0 + # XXX - new_keyset isn't used in this function. new_keyset = 0 if self.list_bindings.curselection(): reselect = 1 @@ -756,10 +861,10 @@ def load_keys_list(self, keyset_name): bind_names.sort() self.list_bindings.delete(0, END) for bind_name in bind_names: - key = ' '.join(keyset[bind_name]) #make key(s) into a string - bind_name = bind_name[2:-2] #trim off the angle brackets + key = ' '.join(keyset[bind_name]) + bind_name = bind_name[2:-2] # Trim off the angle brackets. if keyset_name in self.changed_items['keys']: - #handle any unsaved changes to this key set + # Handle any unsaved changes to this key set. if bind_name in self.changed_items['keys'][keyset_name]: key = self.changed_items['keys'][keyset_name][bind_name] self.list_bindings.insert(END, bind_name+' - '+key) @@ -769,19 +874,25 @@ def load_keys_list(self, keyset_name): self.list_bindings.select_anchor(list_index) def delete_custom_keys(self): + """Handle event to delete a custom key set. + + Applying the delete deactivates the current configuration and + reverts to the default. The custom key set is permanently + deleted from the config file. + """ keyset_name=self.custom_keys.get() delmsg = 'Are you sure you wish to delete the key set %r ?' if not tkMessageBox.askyesno( 'Delete Key Set', delmsg % keyset_name, parent=self): return self.deactivate_current_config() - #remove key set from config + # Remove key set from config. idleConf.userCfg['keys'].remove_section(keyset_name) if keyset_name in self.changed_items['keys']: del(self.changed_items['keys'][keyset_name]) - #write changes + # Write changes. idleConf.userCfg['keys'].Save() - #reload user key set list + # Reload user key set list. item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() if not item_list: @@ -789,30 +900,36 @@ def delete_custom_keys(self): self.opt_menu_keys_custom.SetMenu(item_list, '- no custom keys -') else: self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) - #revert to default key set + # Revert to default key set. self.are_keys_builtin.set(idleConf.defaultCfg['main'] .Get('Keys', 'default')) self.builtin_keys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') or idleConf.default_keys()) - #user can't back out of these changes, they must be applied now + # User can't back out of these changes, they must be applied now. self.save_all_changed_configs() self.activate_config_changes() self.set_keys_type() def delete_custom_theme(self): + """Handle event to delete custom theme. + + The current theme is deactivated and the default theme is + activated. The custom theme is permanently removed from + the config file. + """ theme_name = self.custom_theme.get() delmsg = 'Are you sure you wish to delete the theme %r ?' if not tkMessageBox.askyesno( 'Delete Theme', delmsg % theme_name, parent=self): return self.deactivate_current_config() - #remove theme from config + # Remove theme from config. idleConf.userCfg['highlight'].remove_section(theme_name) if theme_name in self.changed_items['highlight']: del(self.changed_items['highlight'][theme_name]) - #write changes + # Write changes. idleConf.userCfg['highlight'].Save() - #reload user theme list + # Reload user theme list. item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() if not item_list: @@ -820,37 +937,43 @@ def delete_custom_theme(self): self.opt_menu_theme_custom.SetMenu(item_list, '- no custom themes -') else: self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) - #revert to default theme + # Revert to default theme. self.is_builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) self.builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) - #user can't back out of these changes, they must be applied now + # User can't back out of these changes, they must be applied now. self.save_all_changed_configs() self.activate_config_changes() self.set_theme_type() def get_colour(self): + """Handle button to select a new color for the target tag. + + If a new color is selected while using a builtin theme, a + name must be supplied to create a custom theme. + """ target = self.highlight_target.get() prev_colour = self.frame_colour_set.cget('bg') rgbTuplet, colour_string = tkColorChooser.askcolor( parent=self, title='Pick new colour for : '+target, initialcolor=prev_colour) if colour_string and (colour_string != prev_colour): - #user didn't cancel, and they chose a new colour - if self.is_builtin_theme.get(): #current theme is a built-in + # User didn't cancel and they chose a new colour. + if self.is_builtin_theme.get(): # Current theme is a built-in. message = ('Your changes will be saved as a new Custom Theme. ' 'Enter a name for your new Custom Theme below.') new_theme = self.get_new_theme_name(message) - if not new_theme: #user cancelled custom theme creation + if not new_theme: # User cancelled custom theme creation. return - else: #create new custom theme based on previously active theme + else: # Create new custom theme based on previously active theme. self.create_new_theme(new_theme) self.colour.set(colour_string) - else: #current theme is user defined + else: # Current theme is user defined. self.colour.set(colour_string) def on_new_colour_set(self): + "Display sample of new color selection on the dialog." new_colour=self.colour.get() - self.frame_colour_set.config(bg=new_colour) #set sample + self.frame_colour_set.config(bg=new_colour) # Set sample. plane ='foreground' if self.fg_bg_toggle.get() else 'background' sample_element = self.theme_elements[self.highlight_target.get()][0] self.text_highlight_sample.tag_config(sample_element, **{plane:new_colour}) @@ -859,6 +982,7 @@ def on_new_colour_set(self): self.add_changed_item('highlight', theme, theme_element, new_colour) def get_new_theme_name(self, message): + "Return name of new theme from query popup." used_names = (idleConf.GetSectionList('user', 'highlight') + idleConf.GetSectionList('default', 'highlight')) new_theme = SectionName( @@ -866,13 +990,18 @@ def get_new_theme_name(self, message): return new_theme def save_as_new_theme(self): + "Prompt for new theme name and create the theme." new_theme_name = self.get_new_theme_name('New Theme Name:') if new_theme_name: self.create_new_theme(new_theme_name) def create_new_theme(self, new_theme_name): - #creates new custom theme based on the previously active theme, - #and makes the new theme active + """Create a new custom theme with the given name. + + Create the new theme based on the previously active theme + with the current changes applied. Once it is saved, then + activate the new theme. + """ if self.is_builtin_theme.get(): theme_type = 'default' theme_name = self.builtin_theme.get() @@ -880,14 +1009,14 @@ def create_new_theme(self, new_theme_name): theme_type = 'user' theme_name = self.custom_theme.get() new_theme = idleConf.GetThemeDict(theme_type, theme_name) - #apply any of the old theme's unsaved changes to the new theme + # Apply any of the old theme's unsaved changes to the new theme. if theme_name in self.changed_items['highlight']: theme_changes = self.changed_items['highlight'][theme_name] for element in theme_changes: new_theme[element] = theme_changes[element] - #save the new theme + # Save the new theme. self.save_new_theme(new_theme_name, new_theme) - #change gui over to the new theme + # Change GUI over to the new theme. custom_theme_list = idleConf.GetSectionList('user', 'highlight') custom_theme_list.sort() self.opt_menu_theme_custom.SetMenu(custom_theme_list, new_theme_name) @@ -895,11 +1024,17 @@ def create_new_theme(self, new_theme_name): self.set_theme_type() def on_list_fonts_button_release(self, event): + """Handle event of selecting a font from the list. + + Change the font name to the font selected from the list + and update sample text to show that font. + """ font = self.list_fonts.get(ANCHOR) self.font_name.set(font.lower()) self.set_font_sample() def set_font_sample(self, event=None): + "Update the screen samples with the font settings from the dialog." font_name = self.font_name.get() font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) @@ -907,38 +1042,42 @@ def set_font_sample(self, event=None): self.text_highlight_sample.configure(font=new_font) def set_highlight_target(self): - if self.highlight_target.get() == 'Cursor': #bg not possible + "Set fg/bg toggle and color based on highlight tag target." + if self.highlight_target.get() == 'Cursor': # bg not possible self.radio_fg.config(state=DISABLED) self.radio_bg.config(state=DISABLED) self.fg_bg_toggle.set(1) - else: #both fg and bg can be set + else: # Both fg and bg can be set. self.radio_fg.config(state=NORMAL) self.radio_bg.config(state=NORMAL) self.fg_bg_toggle.set(1) self.set_colour_sample() def set_colour_sample_binding(self, *args): + "Change color sample based on foreground/background toggle." self.set_colour_sample() def set_colour_sample(self): - #set the colour smaple area + "Set the color of the frame background to reflect the selected target." + # Set the colour sample area. tag = self.theme_elements[self.highlight_target.get()][0] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' colour = self.text_highlight_sample.tag_cget(tag, plane) self.frame_colour_set.config(bg=colour) def paint_theme_sample(self): - if self.is_builtin_theme.get(): #a default theme + "Apply the theme colors to each element tag in the sample text." + if self.is_builtin_theme.get(): # Default theme theme = self.builtin_theme.get() - else: #a user theme + else: # User theme theme = self.custom_theme.get() for element_title in self.theme_elements: element = self.theme_elements[element_title][0] colours = idleConf.GetHighlight(theme, element) - if element == 'cursor': #cursor sample needs special painting + if element == 'cursor': # Cursor sample needs special painting. colours['background'] = idleConf.GetHighlight( theme, 'normal', fgBg='bg') - #handle any unsaved changes to this theme + # Handle any unsaved changes to this theme. if theme in self.changed_items['highlight']: theme_dict = self.changed_items['highlight'][theme] if element + '-foreground' in theme_dict: @@ -949,21 +1088,28 @@ def paint_theme_sample(self): self.set_colour_sample() def help_source_selected(self, event): + "Handle event for selecting additional help." self.set_helplist_button_states() def set_helplist_button_states(self): - if self.list_help.size() < 1: #no entries in list + "Toggle the state for the help list buttons based on list entries." + if self.list_help.size() < 1: # No entries in list. self.button_helplist_edit.config(state=DISABLED) self.button_helplist_remove.config(state=DISABLED) - else: #there are some entries - if self.list_help.curselection(): #there currently is a selection + else: # Some entries. + if self.list_help.curselection(): # There currently is a selection. self.button_helplist_edit.config(state=NORMAL) self.button_helplist_remove.config(state=NORMAL) - else: #there currently is not a selection + else: # There currently is not a selection. self.button_helplist_edit.config(state=DISABLED) self.button_helplist_remove.config(state=DISABLED) def helplist_item_add(self): + """Handle add button for the help list. + + Query for name and location of new help sources and add + them to the list. + """ help_source = HelpSource(self, 'New Help Source', ).result if help_source: @@ -973,6 +1119,11 @@ def helplist_item_add(self): self.set_helplist_button_states() def helplist_item_edit(self): + """Handle edit button for the help list. + + Query with existing help source information and update + config if the values are changed. + """ item_index = self.list_help.index(ANCHOR) help_source = self.user_helplist[item_index] new_help_source = HelpSource( @@ -988,6 +1139,10 @@ def helplist_item_edit(self): self.set_helplist_button_states() def helplist_item_remove(self): + """Handle remove button for the help list. + + Delete the help list item from config. + """ item_index = self.list_help.index(ANCHOR) del(self.user_helplist[item_index]) self.list_help.delete(item_index) @@ -1003,7 +1158,8 @@ def update_user_help_changed_items(self): ';'.join(self.user_helplist[num-1][:2])) def load_font_cfg(self): - ##base editor font selection list + "Load current configuration settings for the font options." + # Set base editor font selection list. fonts = list(tkFont.families(self)) fonts.sort() for font in fonts: @@ -1021,29 +1177,31 @@ def load_font_cfg(self): self.list_fonts.select_anchor(current_font_index) except ValueError: pass - ##font size dropdown + # Set font size dropdown. self.opt_menu_font_size.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', '16', '18', '20', '22', '25', '29', '34', '40'), font_size ) - ##font_weight + # Set font weight. self.font_bold.set(font_bold) - ##font sample + # Set font sample. self.set_font_sample() def load_tab_cfg(self): - ##indent sizes + "Load current configuration settings for the tab options." + # Set indent sizes. space_num = idleConf.GetOption( 'main', 'Indent', 'num-spaces', default=4, type='int') self.space_num.set(space_num) def load_theme_cfg(self): - ##current theme type radiobutton + "Load current configuration settings for the theme options." + # Set current theme type radiobutton. self.is_builtin_theme.set(idleConf.GetOption( 'main', 'Theme', 'default', type='bool', default=1)) - ##currently set theme + # Set current theme. current_option = idleConf.CurrentTheme() - ##load available theme option menus - if self.is_builtin_theme.get(): #default theme selected + # Load available theme option menus. + if self.is_builtin_theme.get(): # Default theme selected. item_list = idleConf.GetSectionList('default', 'highlight') item_list.sort() self.opt_menu_theme_builtin.SetMenu(item_list, current_option) @@ -1054,7 +1212,7 @@ def load_theme_cfg(self): self.custom_theme.set('- no custom themes -') else: self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) - else: #user theme selected + else: # User theme selected. item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() self.opt_menu_theme_custom.SetMenu(item_list, current_option) @@ -1062,7 +1220,7 @@ def load_theme_cfg(self): item_list.sort() self.opt_menu_theme_builtin.SetMenu(item_list, item_list[0]) self.set_theme_type() - ##load theme element option menu + # Load theme element option menu. theme_names = list(self.theme_elements.keys()) theme_names.sort(key=lambda x: self.theme_elements[x][1]) self.opt_menu_highlight_target.SetMenu(theme_names, theme_names[0]) @@ -1070,13 +1228,14 @@ def load_theme_cfg(self): self.set_highlight_target() def load_key_cfg(self): - ##current keys type radiobutton + "Load current configuration settings for the keybinding options." + # Set current keys type radiobutton. self.are_keys_builtin.set(idleConf.GetOption( 'main', 'Keys', 'default', type='bool', default=1)) - ##currently set keys + # Set current keys. current_option = idleConf.CurrentKeys() - ##load available keyset option menus - if self.are_keys_builtin.get(): #default theme selected + # Load available keyset option menus. + if self.are_keys_builtin.get(): # Default theme selected. item_list = idleConf.GetSectionList('default', 'keys') item_list.sort() self.opt_menu_keys_builtin.SetMenu(item_list, current_option) @@ -1087,7 +1246,7 @@ def load_key_cfg(self): self.custom_keys.set('- no custom keys -') else: self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) - else: #user key set selected + else: # User key set selected. item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() self.opt_menu_keys_custom.SetMenu(item_list, current_option) @@ -1095,50 +1254,48 @@ def load_key_cfg(self): item_list.sort() self.opt_menu_keys_builtin.SetMenu(item_list, idleConf.default_keys()) self.set_keys_type() - ##load keyset element list + # Load keyset element list. keyset_name = idleConf.CurrentKeys() self.load_keys_list(keyset_name) def load_general_cfg(self): - #startup state + "Load current configuration settings for the general options." + # Set startup state. self.startup_edit.set(idleConf.GetOption( 'main', 'General', 'editor-on-startup', default=1, type='bool')) - #autosave state + # Set autosave state. self.autosave.set(idleConf.GetOption( 'main', 'General', 'autosave', default=0, type='bool')) - #initial window size + # Set initial window size. self.win_width.set(idleConf.GetOption( 'main', 'EditorWindow', 'width', type='int')) self.win_height.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) - # default source encoding + # Set default source encoding. self.encoding.set(idleConf.GetOption( 'main', 'EditorWindow', 'encoding', default='none')) - # additional help sources + # Set additional help sources. self.user_helplist = idleConf.GetAllExtraHelpSourcesList() for help_item in self.user_helplist: self.list_help.insert(END, help_item[0]) self.set_helplist_button_states() def load_configs(self): - """ - load configuration from default and user config files and populate + """Load configuration for each page. + + Load configuration from default and user config files and populate the widgets on the config dialog pages. """ - ### fonts / tabs page self.load_font_cfg() self.load_tab_cfg() - ### highlighting page self.load_theme_cfg() - ### keys page self.load_key_cfg() - ### general page self.load_general_cfg() # note: extension page handled separately def save_new_key_set(self, keyset_name, keyset): - """ - save a newly created core key set. + """Save a newly created core key set. + keyset_name - string, the name of the new key set keyset - dictionary containing the new key set """ @@ -1149,8 +1306,8 @@ def save_new_key_set(self, keyset_name, keyset): idleConf.userCfg['keys'].SetOption(keyset_name, event, value) def save_new_theme(self, theme_name, theme): - """ - save a newly created theme. + """Save a newly created theme. + theme_name - string, the name of the new theme theme - dictionary containing the new theme """ @@ -1161,21 +1318,22 @@ def save_new_theme(self, theme_name, theme): idleConf.userCfg['highlight'].SetOption(theme_name, element, value) def set_user_value(self, config_type, section, item, value): + "Return True if the configuration value was added or changed." if idleConf.defaultCfg[config_type].has_option(section, item): if idleConf.defaultCfg[config_type].Get(section, item) == value: - #the setting equals a default setting, remove it from user cfg + # The setting equals a default setting, remove it from user cfg. return idleConf.userCfg[config_type].RemoveOption(section, item) - #if we got here set the option + # If we got here, set the option. return idleConf.userCfg[config_type].SetOption(section, item, value) def save_all_changed_configs(self): - "Save configuration changes to the user config file." + "Save all configuration changes to the user config file." idleConf.userCfg['main'].Save() for config_type in self.changed_items: cfg_type_changed = False for section in self.changed_items[config_type]: if section == 'HelpFiles': - #this section gets completely replaced + # This section gets completely replaced. idleConf.userCfg['main'].remove_section('HelpFiles') cfg_type_changed = True for item in self.changed_items[config_type][section]: @@ -1185,14 +1343,15 @@ def save_all_changed_configs(self): if cfg_type_changed: idleConf.userCfg[config_type].Save() for config_type in ['keys', 'highlight']: - # save these even if unchanged! + # Save these even if unchanged! idleConf.userCfg[config_type].Save() - self.reset_changed_items() #clear the changed items dict - self.save_all_changed_extensions() # uses a different mechanism + self.reset_changed_items() # Clear the changed items dict. + self.save_all_changed_extensions() # Uses a different mechanism. def deactivate_current_config(self): - #Before a config is saved, some cleanup of current - #config must be done - remove the previous keybindings + "Remove current key bindings." + # Before a config is saved, some cleanup of current + # config must be done - remove the previous keybindings. win_instances = self.parent.instance_dict.keys() for instance in win_instances: instance.RemoveKeybindings() @@ -1208,18 +1367,22 @@ def activate_config_changes(self): instance.reset_help_menu_entries() def cancel(self): + "Dismiss config dialog." self.destroy() def ok(self): + "Apply config changes, then dismiss dialog." self.apply() self.destroy() def apply(self): + "Apply config changes and leave dialog open." self.deactivate_current_config() self.save_all_changed_configs() self.activate_config_changes() def help(self): + "Create textview for config dialog help." page = self.tab_pages._current_page view_text(self, title='Help for IDLE preferences', text=help_common+help_pages.get(page, '')) @@ -1247,8 +1410,8 @@ def create_page_extensions(self): self.ext_userCfg = idleConf.userCfg['extensions'] self.is_int = self.register(is_int) self.load_extensions() - # create widgets - a listbox shows all available extensions, with the - # controls for the extension selected in the listbox to the right + # Create widgets - a listbox shows all available extensions, with the + # controls for the extension selected in the listbox to the right. self.extension_names = StringVar(self) frame.rowconfigure(0, weight=1) frame.columnconfigure(2, weight=1) @@ -1268,7 +1431,7 @@ def create_page_extensions(self): self.outerframe = self # TEMPORARY self.tabbed_page_set = self.extension_list # TEMPORARY - # create the frame holding controls for each extension + # Create the frame holding controls for each extension. ext_names = '' for ext_name in sorted(self.extensions): self.create_extension_frame(ext_name) @@ -1286,7 +1449,7 @@ def load_extensions(self): for ext_name in self.extensions: opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) - # bring 'enable' options to the beginning of the list + # Bring 'enable' options to the beginning of the list. enables = [opt_name for opt_name in opt_list if opt_name.startswith('enable')] for opt_name in enables: @@ -1310,8 +1473,8 @@ def load_extensions(self): value = self.ext_userCfg.Get( ext_name, opt_name, type=opt_type, raw=True, default=def_obj) - except ValueError: # Need this until .Get fixed - value = def_obj # bad values overwritten by entry + except ValueError: # Need this until .Get fixed. + value = def_obj # Bad values overwritten by entry. var = StringVar(self) var.set(str(value)) @@ -1323,6 +1486,7 @@ def load_extensions(self): }) def extension_selected(self, event): + "Handle selection of an extension from the list." newsel = self.extension_list.curselection() if newsel: newsel = self.extension_list.get(newsel) @@ -1341,9 +1505,9 @@ def create_extension_frame(self, ext_name): f = VerticalScrolledFrame(self.details_frame, height=250, width=250) self.config_frame[ext_name] = f entry_area = f.interior - # create an entry for each configuration option + # Create an entry for each configuration option. for row, opt in enumerate(self.extensions[ext_name]): - # create a row with a label and entry/checkbutton + # Create a row with a label and entry/checkbutton. label = Label(entry_area, text=opt['name']) label.grid(row=row, column=0, sticky=NW) var = opt['var'] @@ -1363,15 +1527,20 @@ def create_extension_frame(self, ext_name): return def set_extension_value(self, section, opt): + """Return True if the configuration was added or changed. + + If the value is the same as the default, then remove it + from user config file. + """ name = opt['name'] default = opt['default'] value = opt['var'].get().strip() or default opt['var'].set(value) # if self.defaultCfg.has_section(section): - # Currently, always true; if not, indent to return + # Currently, always true; if not, indent to return. if (value == default): return self.ext_userCfg.RemoveOption(section, name) - # set the option + # Set the option. return self.ext_userCfg.SetOption(section, name, value) def save_all_changed_extensions(self): @@ -1446,7 +1615,7 @@ class VerticalScrolledFrame(Frame): def __init__(self, parent, *args, **kw): Frame.__init__(self, parent, *args, **kw) - # create a canvas object and a vertical scrollbar for scrolling it + # Create a canvas object and a vertical scrollbar for scrolling it. vscrollbar = Scrollbar(self, orient=VERTICAL) vscrollbar.pack(fill=Y, side=RIGHT, expand=FALSE) canvas = Canvas(self, bd=0, highlightthickness=0, @@ -1454,25 +1623,25 @@ def __init__(self, parent, *args, **kw): canvas.pack(side=LEFT, fill=BOTH, expand=TRUE) vscrollbar.config(command=canvas.yview) - # reset the view + # Reset the view. canvas.xview_moveto(0) canvas.yview_moveto(0) - # create a frame inside the canvas which will be scrolled with it + # Create a frame inside the canvas which will be scrolled with it. self.interior = interior = Frame(canvas) interior_id = canvas.create_window(0, 0, window=interior, anchor=NW) - # track changes to the canvas and frame width and sync them, - # also updating the scrollbar + # Track changes to the canvas and frame width and sync them, + # also updating the scrollbar. def _configure_interior(event): - # update the scrollbars to match the size of the inner frame + # Update the scrollbars to match the size of the inner frame. size = (interior.winfo_reqwidth(), interior.winfo_reqheight()) canvas.config(scrollregion="0 0 %s %s" % size) interior.bind('', _configure_interior) def _configure_canvas(event): if interior.winfo_reqwidth() != canvas.winfo_width(): - # update the inner frame's width to fill the canvas + # Update the inner frame's width to fill the canvas. canvas.itemconfigure(interior_id, width=canvas.winfo_width()) canvas.bind('', _configure_canvas) From 1278d29000794085f1d9e25f2dadbf70b9076e30 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 5 Jul 2017 00:55:29 -0400 Subject: [PATCH 0523/1345] [3.6] bpo-21624: IDLE -- minor htest fixes (GH-2575) (#2578) (cherry picked from commit 2000150) --- Lib/idlelib/browser.py | 2 +- Lib/idlelib/config_key.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index ea05638df1a189..73c251e76d3cce 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -215,7 +215,7 @@ def OnDoubleClick(self): edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) -def _class_browser(parent): #Wrapper for htest +def _class_browser(parent): # htest # try: file = __file__ except NameError: diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index fc6e341acad0ee..5556b767964739 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -293,5 +293,6 @@ def bind_ok(self, keys): if __name__ == '__main__': import unittest unittest.main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(GetKeysDialog) From 0884c40603be59cc40dd2af510a4c008dfac85f2 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 5 Jul 2017 00:55:59 -0400 Subject: [PATCH 0524/1345] [3.6] NEWS for 30777 (GH-2576) (#2579) (cherry picked from commit aaa917f) --- Misc/NEWS.d/next/IDLE/2017-07-04-22-45-46.bpo-30777.uxzlMB.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-04-22-45-46.bpo-30777.uxzlMB.rst diff --git a/Misc/NEWS.d/next/IDLE/2017-07-04-22-45-46.bpo-30777.uxzlMB.rst b/Misc/NEWS.d/next/IDLE/2017-07-04-22-45-46.bpo-30777.uxzlMB.rst new file mode 100644 index 00000000000000..b7118bf2c25b9f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-04-22-45-46.bpo-30777.uxzlMB.rst @@ -0,0 +1,2 @@ +IDLE: configdialog - Add docstrings and fix comments. Patch by Cheryl +Sabella. From e3a0ff0d76b9e7a994afa7f2c54b19a63f1bb57d Mon Sep 17 00:00:00 2001 From: Masayuki Yamamoto Date: Wed, 5 Jul 2017 18:24:46 +0900 Subject: [PATCH 0525/1345] [3.6] bpo-30854: Fix compile error when --without-threads (GH-2581) (#2583) * bpo-30854: Fix compile error when --without-threads * bpo-30854: fix news (cherry picked from commit 0c3116309307ad2c7f8e2d2096612f4ab33cbb62) --- Misc/NEWS.d/next/Build/2017-07-05-16-54-59.bpo-30854.sPADRI.rst | 2 ++ Python/ceval.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2017-07-05-16-54-59.bpo-30854.sPADRI.rst diff --git a/Misc/NEWS.d/next/Build/2017-07-05-16-54-59.bpo-30854.sPADRI.rst b/Misc/NEWS.d/next/Build/2017-07-05-16-54-59.bpo-30854.sPADRI.rst new file mode 100644 index 00000000000000..f75aea420ad5dd --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-07-05-16-54-59.bpo-30854.sPADRI.rst @@ -0,0 +1,2 @@ +Fix compile error when compiling --without-threads. +Patch by Masayuki Yamamoto. diff --git a/Python/ceval.c b/Python/ceval.c index bf9103dd9aa4e3..770dfcba89e513 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -621,7 +621,7 @@ Py_MakePendingCalls(void) arg = pendingcalls[i].arg; pendingfirst = (i + 1) % NPENDINGCALLS; if (func(arg) < 0) { - goto error: + goto error; } } busy = 0; From aaa4f991518611d101fba1ef3ecb18d7b385ad5b Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 5 Jul 2017 14:03:10 -0400 Subject: [PATCH 0526/1345] [3.6] bpo-30828: Fix out of bounds write in `asyncio.CFuture.remove_done_callback() (GH-2569) (#2590) (cherry picked from commit 833a3b0d3707200daeaccdd218e8f18a190284aa) --- Lib/test/test_asyncio/test_futures.py | 24 ++++++++++++++++++- .../2017-07-04-13-10-52.bpo-30828.CLvEvV.rst | 1 + Modules/_asynciomodule.c | 13 +++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-04-13-10-52.bpo-30828.CLvEvV.rst diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 5d4b2d2aa0efc0..ce657fc1b6a058 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -593,7 +593,7 @@ def __eq__(self, other): fut.remove_done_callback(evil()) - def test_schedule_callbacks_list_mutation(self): + def test_schedule_callbacks_list_mutation_1(self): # see http://bugs.python.org/issue28963 for details def mut(f): @@ -606,6 +606,28 @@ def mut(f): fut.set_result(1) test_utils.run_briefly(self.loop) + def test_schedule_callbacks_list_mutation_2(self): + # see http://bugs.python.org/issue30828 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + max_extra_cbs = 100 + extra_cbs = 0 + + class evil: + def __eq__(self, other): + nonlocal extra_cbs + extra_cbs += 1 + if extra_cbs < max_extra_cbs: + fut.add_done_callback(id) + return False + + fut.remove_done_callback(evil()) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS.d/next/Library/2017-07-04-13-10-52.bpo-30828.CLvEvV.rst b/Misc/NEWS.d/next/Library/2017-07-04-13-10-52.bpo-30828.CLvEvV.rst new file mode 100644 index 00000000000000..8924962d07139a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-04-13-10-52.bpo-30828.CLvEvV.rst @@ -0,0 +1 @@ +Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 492b98359713df..8fbd565dd9c26c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -531,9 +531,16 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) goto fail; } if (ret == 0) { - Py_INCREF(item); - PyList_SET_ITEM(newlist, j, item); - j++; + if (j < len) { + Py_INCREF(item); + PyList_SET_ITEM(newlist, j, item); + j++; + } + else { + if (PyList_Append(newlist, item)) { + goto fail; + } + } } } From 03b0e8374b2ea93adf8fb6a48db2916f3b0388cc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Jul 2017 08:38:24 +0300 Subject: [PATCH 0527/1345] [3.6] bpo-30814: Fixed a race condition when import a submodule from a package. (GH-2580). (#2598) (cherry picked from commit b4baacee1adc06edbe30ac7574d17a8cd168e2e0) --- Lib/importlib/_bootstrap.py | 28 +- Lib/test/test_import/__init__.py | 28 + Lib/test/test_import/data/package/__init__.py | 2 + .../test_import/data/package/submodule.py | 0 Misc/NEWS | 2 + Python/import.c | 17 +- Python/importlib.h | 588 +++++++++--------- 7 files changed, 340 insertions(+), 325 deletions(-) create mode 100644 Lib/test/test_import/data/package/__init__.py create mode 100644 Lib/test/test_import/data/package/submodule.py diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index a531a0351dfb06..f812a18cde3247 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -956,9 +956,19 @@ def _find_and_load_unlocked(name, import_): def _find_and_load(name, import_): - """Find and load the module, and release the import lock.""" - with _ModuleLockManager(name): - return _find_and_load_unlocked(name, import_) + """Find and load the module.""" + _imp.acquire_lock() + if name not in sys.modules: + with _ModuleLockManager(name): + return _find_and_load_unlocked(name, import_) + module = sys.modules[name] + if module is None: + _imp.release_lock() + message = ('import of {} halted; ' + 'None in sys.modules'.format(name)) + raise ModuleNotFoundError(message, name=name) + _lock_unlock_module(name) + return module def _gcd_import(name, package=None, level=0): @@ -973,17 +983,7 @@ def _gcd_import(name, package=None, level=0): _sanity_check(name, package, level) if level > 0: name = _resolve_name(name, package, level) - _imp.acquire_lock() - if name not in sys.modules: - return _find_and_load(name, _gcd_import) - module = sys.modules[name] - if module is None: - _imp.release_lock() - message = ('import of {} halted; ' - 'None in sys.modules'.format(name)) - raise ModuleNotFoundError(message, name=name) - _lock_unlock_module(name) - return module + return _find_and_load(name, _gcd_import) def _handle_fromlist(module, fromlist, import_): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4aca9e597cf9f9..3291d0896cf912 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -10,6 +10,8 @@ import random import stat import sys +import threading +import time import unittest import unittest.mock as mock import textwrap @@ -350,6 +352,32 @@ def __getattr__(self, _): with self.assertRaises(ImportError): from test_from_import_AttributeError import does_not_exist + def test_concurrency(self): + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'data')) + try: + exc = None + def run(): + event.wait() + try: + import package + except BaseException as e: + nonlocal exc + exc = e + + for i in range(10): + event = threading.Event() + threads = [threading.Thread(target=run) for x in range(2)] + try: + with test.support.start_threads(threads, event.set): + time.sleep(0) + finally: + sys.modules.pop('package', None) + sys.modules.pop('package.submodule', None) + if exc is not None: + raise exc + finally: + del sys.path[0] + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): diff --git a/Lib/test/test_import/data/package/__init__.py b/Lib/test/test_import/data/package/__init__.py new file mode 100644 index 00000000000000..a4f2bc3400ce5e --- /dev/null +++ b/Lib/test/test_import/data/package/__init__.py @@ -0,0 +1,2 @@ +import package.submodule +package.submodule diff --git a/Lib/test/test_import/data/package/submodule.py b/Lib/test/test_import/data/package/submodule.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Misc/NEWS b/Misc/NEWS index 92ccf52149aac1..89876f8fa24543 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.3 release candidate 1? Core and Builtins ----------------- +- bpo-30814: Fixed a race condition when import a submodule from a package. + - bpo-30597: ``print`` now shows expected input in custom error message when used as a Python 2 statement. Patch by Sanyam Khurana. diff --git a/Python/import.c b/Python/import.c index a23a1026e11169..a9aa5b3c25bd2e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1533,18 +1533,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } mod = PyDict_GetItem(interp->modules, abs_name); - if (mod == Py_None) { - PyObject *msg = PyUnicode_FromFormat("import of %R halted; " - "None in sys.modules", abs_name); - if (msg != NULL) { - PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg, - abs_name, NULL); - Py_DECREF(msg); - } - mod = NULL; - goto error; - } - else if (mod != NULL) { + if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); _Py_IDENTIFIER(_lock_unlock_module); @@ -1585,10 +1574,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } } else { -#ifdef WITH_THREAD - _PyImport_AcquireLock(); -#endif - /* _bootstrap._find_and_load() releases the import lock */ mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__find_and_load, abs_name, interp->import_func, NULL); diff --git a/Python/importlib.h b/Python/importlib.h index 4607c40de60e7c..0584477f190397 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1521,302 +1521,300 @@ const unsigned char _Py_M__importlib[] = { 14,1,4,1,10,1,10,2,10,1,10,1,10,1,2,1, 10,1,14,1,16,1,20,1,10,1,8,1,20,2,8,1, 4,2,10,1,22,1,114,180,0,0,0,99,2,0,0,0, - 0,0,0,0,2,0,0,0,10,0,0,0,67,0,0,0, - 115,30,0,0,0,116,0,124,0,131,1,143,12,1,0,116, - 1,124,0,124,1,131,2,83,0,81,0,82,0,88,0,100, - 1,83,0,41,2,122,54,70,105,110,100,32,97,110,100,32, - 108,111,97,100,32,116,104,101,32,109,111,100,117,108,101,44, - 32,97,110,100,32,114,101,108,101,97,115,101,32,116,104,101, - 32,105,109,112,111,114,116,32,108,111,99,107,46,78,41,2, - 114,42,0,0,0,114,180,0,0,0,41,2,114,15,0,0, - 0,114,179,0,0,0,114,10,0,0,0,114,10,0,0,0, + 0,0,0,0,4,0,0,0,11,0,0,0,67,0,0,0, + 115,104,0,0,0,116,0,106,1,131,0,1,0,124,0,116, + 2,106,3,107,7,114,44,116,4,124,0,131,1,143,12,1, + 0,116,5,124,0,124,1,131,2,83,0,81,0,82,0,88, + 0,116,2,106,3,124,0,25,0,125,2,124,2,100,1,107, + 8,114,92,116,0,106,6,131,0,1,0,100,2,106,7,124, + 0,131,1,125,3,116,8,124,3,124,0,100,3,141,2,130, + 1,116,9,124,0,131,1,1,0,124,2,83,0,41,4,122, + 25,70,105,110,100,32,97,110,100,32,108,111,97,100,32,116, + 104,101,32,109,111,100,117,108,101,46,78,122,40,105,109,112, + 111,114,116,32,111,102,32,123,125,32,104,97,108,116,101,100, + 59,32,78,111,110,101,32,105,110,32,115,121,115,46,109,111, + 100,117,108,101,115,41,1,114,15,0,0,0,41,10,114,46, + 0,0,0,114,137,0,0,0,114,14,0,0,0,114,79,0, + 0,0,114,42,0,0,0,114,180,0,0,0,114,47,0,0, + 0,114,38,0,0,0,114,178,0,0,0,114,56,0,0,0, + 41,4,114,15,0,0,0,114,179,0,0,0,114,83,0,0, + 0,114,67,0,0,0,114,10,0,0,0,114,10,0,0,0, 114,11,0,0,0,218,14,95,102,105,110,100,95,97,110,100, - 95,108,111,97,100,190,3,0,0,115,4,0,0,0,0,2, - 10,1,114,181,0,0,0,114,19,0,0,0,99,3,0,0, - 0,0,0,0,0,5,0,0,0,4,0,0,0,67,0,0, - 0,115,120,0,0,0,116,0,124,0,124,1,124,2,131,3, - 1,0,124,2,100,1,107,4,114,32,116,1,124,0,124,1, - 124,2,131,3,125,0,116,2,106,3,131,0,1,0,124,0, - 116,4,106,5,107,7,114,60,116,6,124,0,116,7,131,2, - 83,0,116,4,106,5,124,0,25,0,125,3,124,3,100,2, - 107,8,114,108,116,2,106,8,131,0,1,0,100,3,106,9, - 124,0,131,1,125,4,116,10,124,4,124,0,100,4,141,2, - 130,1,116,11,124,0,131,1,1,0,124,3,83,0,41,5, - 97,50,1,0,0,73,109,112,111,114,116,32,97,110,100,32, - 114,101,116,117,114,110,32,116,104,101,32,109,111,100,117,108, - 101,32,98,97,115,101,100,32,111,110,32,105,116,115,32,110, - 97,109,101,44,32,116,104,101,32,112,97,99,107,97,103,101, - 32,116,104,101,32,99,97,108,108,32,105,115,10,32,32,32, - 32,98,101,105,110,103,32,109,97,100,101,32,102,114,111,109, - 44,32,97,110,100,32,116,104,101,32,108,101,118,101,108,32, - 97,100,106,117,115,116,109,101,110,116,46,10,10,32,32,32, - 32,84,104,105,115,32,102,117,110,99,116,105,111,110,32,114, - 101,112,114,101,115,101,110,116,115,32,116,104,101,32,103,114, - 101,97,116,101,115,116,32,99,111,109,109,111,110,32,100,101, - 110,111,109,105,110,97,116,111,114,32,111,102,32,102,117,110, - 99,116,105,111,110,97,108,105,116,121,10,32,32,32,32,98, - 101,116,119,101,101,110,32,105,109,112,111,114,116,95,109,111, - 100,117,108,101,32,97,110,100,32,95,95,105,109,112,111,114, - 116,95,95,46,32,84,104,105,115,32,105,110,99,108,117,100, - 101,115,32,115,101,116,116,105,110,103,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,102,10,32,32,32,32,116,104, - 101,32,108,111,97,100,101,114,32,100,105,100,32,110,111,116, - 46,10,10,32,32,32,32,114,19,0,0,0,78,122,40,105, - 109,112,111,114,116,32,111,102,32,123,125,32,104,97,108,116, - 101,100,59,32,78,111,110,101,32,105,110,32,115,121,115,46, - 109,111,100,117,108,101,115,41,1,114,15,0,0,0,41,12, - 114,175,0,0,0,114,163,0,0,0,114,46,0,0,0,114, - 137,0,0,0,114,14,0,0,0,114,79,0,0,0,114,181, + 95,108,111,97,100,190,3,0,0,115,24,0,0,0,0,2, + 8,1,10,1,10,1,16,1,10,1,8,1,8,1,4,1, + 6,1,12,1,8,1,114,181,0,0,0,114,19,0,0,0, + 99,3,0,0,0,0,0,0,0,3,0,0,0,4,0,0, + 0,67,0,0,0,115,42,0,0,0,116,0,124,0,124,1, + 124,2,131,3,1,0,124,2,100,1,107,4,114,32,116,1, + 124,0,124,1,124,2,131,3,125,0,116,2,124,0,116,3, + 131,2,83,0,41,2,97,50,1,0,0,73,109,112,111,114, + 116,32,97,110,100,32,114,101,116,117,114,110,32,116,104,101, + 32,109,111,100,117,108,101,32,98,97,115,101,100,32,111,110, + 32,105,116,115,32,110,97,109,101,44,32,116,104,101,32,112, + 97,99,107,97,103,101,32,116,104,101,32,99,97,108,108,32, + 105,115,10,32,32,32,32,98,101,105,110,103,32,109,97,100, + 101,32,102,114,111,109,44,32,97,110,100,32,116,104,101,32, + 108,101,118,101,108,32,97,100,106,117,115,116,109,101,110,116, + 46,10,10,32,32,32,32,84,104,105,115,32,102,117,110,99, + 116,105,111,110,32,114,101,112,114,101,115,101,110,116,115,32, + 116,104,101,32,103,114,101,97,116,101,115,116,32,99,111,109, + 109,111,110,32,100,101,110,111,109,105,110,97,116,111,114,32, + 111,102,32,102,117,110,99,116,105,111,110,97,108,105,116,121, + 10,32,32,32,32,98,101,116,119,101,101,110,32,105,109,112, + 111,114,116,95,109,111,100,117,108,101,32,97,110,100,32,95, + 95,105,109,112,111,114,116,95,95,46,32,84,104,105,115,32, + 105,110,99,108,117,100,101,115,32,115,101,116,116,105,110,103, + 32,95,95,112,97,99,107,97,103,101,95,95,32,105,102,10, + 32,32,32,32,116,104,101,32,108,111,97,100,101,114,32,100, + 105,100,32,110,111,116,46,10,10,32,32,32,32,114,19,0, + 0,0,41,4,114,175,0,0,0,114,163,0,0,0,114,181, 0,0,0,218,11,95,103,99,100,95,105,109,112,111,114,116, - 114,47,0,0,0,114,38,0,0,0,114,178,0,0,0,114, - 56,0,0,0,41,5,114,15,0,0,0,114,161,0,0,0, - 114,162,0,0,0,114,83,0,0,0,114,67,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,182, - 0,0,0,196,3,0,0,115,28,0,0,0,0,9,12,1, - 8,1,12,1,8,1,10,1,10,1,10,1,8,1,8,1, - 4,1,6,1,12,1,8,1,114,182,0,0,0,99,3,0, - 0,0,0,0,0,0,6,0,0,0,17,0,0,0,67,0, - 0,0,115,164,0,0,0,116,0,124,0,100,1,131,2,114, - 160,100,2,124,1,107,6,114,58,116,1,124,1,131,1,125, - 1,124,1,106,2,100,2,131,1,1,0,116,0,124,0,100, - 3,131,2,114,58,124,1,106,3,124,0,106,4,131,1,1, - 0,120,100,124,1,68,0,93,92,125,3,116,0,124,0,124, - 3,131,2,115,64,100,4,106,5,124,0,106,6,124,3,131, - 2,125,4,121,14,116,7,124,2,124,4,131,2,1,0,87, - 0,113,64,4,0,116,8,107,10,114,154,1,0,125,5,1, - 0,122,20,124,5,106,9,124,4,107,2,114,136,119,64,130, - 0,87,0,89,0,100,5,100,5,125,5,126,5,88,0,113, - 64,88,0,113,64,87,0,124,0,83,0,41,6,122,238,70, - 105,103,117,114,101,32,111,117,116,32,119,104,97,116,32,95, - 95,105,109,112,111,114,116,95,95,32,115,104,111,117,108,100, - 32,114,101,116,117,114,110,46,10,10,32,32,32,32,84,104, - 101,32,105,109,112,111,114,116,95,32,112,97,114,97,109,101, - 116,101,114,32,105,115,32,97,32,99,97,108,108,97,98,108, - 101,32,119,104,105,99,104,32,116,97,107,101,115,32,116,104, - 101,32,110,97,109,101,32,111,102,32,109,111,100,117,108,101, - 32,116,111,10,32,32,32,32,105,109,112,111,114,116,46,32, - 73,116,32,105,115,32,114,101,113,117,105,114,101,100,32,116, - 111,32,100,101,99,111,117,112,108,101,32,116,104,101,32,102, - 117,110,99,116,105,111,110,32,102,114,111,109,32,97,115,115, - 117,109,105,110,103,32,105,109,112,111,114,116,108,105,98,39, - 115,10,32,32,32,32,105,109,112,111,114,116,32,105,109,112, - 108,101,109,101,110,116,97,116,105,111,110,32,105,115,32,100, - 101,115,105,114,101,100,46,10,10,32,32,32,32,114,127,0, - 0,0,250,1,42,218,7,95,95,97,108,108,95,95,122,5, - 123,125,46,123,125,78,41,10,114,4,0,0,0,114,126,0, - 0,0,218,6,114,101,109,111,118,101,218,6,101,120,116,101, - 110,100,114,184,0,0,0,114,38,0,0,0,114,1,0,0, - 0,114,58,0,0,0,114,178,0,0,0,114,15,0,0,0, - 41,6,114,83,0,0,0,218,8,102,114,111,109,108,105,115, - 116,114,179,0,0,0,218,1,120,90,9,102,114,111,109,95, - 110,97,109,101,90,3,101,120,99,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,218,16,95,104,97,110,100,108, - 101,95,102,114,111,109,108,105,115,116,221,3,0,0,115,32, - 0,0,0,0,10,10,1,8,1,8,1,10,1,10,1,12, - 1,10,1,10,1,14,1,2,1,14,1,16,4,10,1,2, - 1,24,1,114,189,0,0,0,99,1,0,0,0,0,0,0, - 0,3,0,0,0,6,0,0,0,67,0,0,0,115,146,0, - 0,0,124,0,106,0,100,1,131,1,125,1,124,0,106,0, - 100,2,131,1,125,2,124,1,100,3,107,9,114,82,124,2, - 100,3,107,9,114,78,124,1,124,2,106,1,107,3,114,78, - 116,2,106,3,100,4,124,1,155,2,100,5,124,2,106,1, - 155,2,100,6,157,5,116,4,100,7,100,8,141,3,1,0, - 124,1,83,0,124,2,100,3,107,9,114,96,124,2,106,1, - 83,0,116,2,106,3,100,9,116,4,100,7,100,8,141,3, - 1,0,124,0,100,10,25,0,125,1,100,11,124,0,107,7, - 114,142,124,1,106,5,100,12,131,1,100,13,25,0,125,1, - 124,1,83,0,41,14,122,167,67,97,108,99,117,108,97,116, - 101,32,119,104,97,116,32,95,95,112,97,99,107,97,103,101, - 95,95,32,115,104,111,117,108,100,32,98,101,46,10,10,32, - 32,32,32,95,95,112,97,99,107,97,103,101,95,95,32,105, - 115,32,110,111,116,32,103,117,97,114,97,110,116,101,101,100, - 32,116,111,32,98,101,32,100,101,102,105,110,101,100,32,111, - 114,32,99,111,117,108,100,32,98,101,32,115,101,116,32,116, - 111,32,78,111,110,101,10,32,32,32,32,116,111,32,114,101, - 112,114,101,115,101,110,116,32,116,104,97,116,32,105,116,115, - 32,112,114,111,112,101,114,32,118,97,108,117,101,32,105,115, - 32,117,110,107,110,111,119,110,46,10,10,32,32,32,32,114, - 130,0,0,0,114,89,0,0,0,78,122,32,95,95,112,97, - 99,107,97,103,101,95,95,32,33,61,32,95,95,115,112,101, - 99,95,95,46,112,97,114,101,110,116,32,40,122,4,32,33, - 61,32,250,1,41,233,3,0,0,0,41,1,90,10,115,116, - 97,99,107,108,101,118,101,108,122,89,99,97,110,39,116,32, - 114,101,115,111,108,118,101,32,112,97,99,107,97,103,101,32, - 102,114,111,109,32,95,95,115,112,101,99,95,95,32,111,114, - 32,95,95,112,97,99,107,97,103,101,95,95,44,32,102,97, - 108,108,105,110,103,32,98,97,99,107,32,111,110,32,95,95, - 110,97,109,101,95,95,32,97,110,100,32,95,95,112,97,116, - 104,95,95,114,1,0,0,0,114,127,0,0,0,114,117,0, - 0,0,114,19,0,0,0,41,6,114,30,0,0,0,114,119, - 0,0,0,114,167,0,0,0,114,168,0,0,0,114,169,0, - 0,0,114,118,0,0,0,41,3,218,7,103,108,111,98,97, - 108,115,114,161,0,0,0,114,82,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,17,95,99,97, - 108,99,95,95,95,112,97,99,107,97,103,101,95,95,252,3, - 0,0,115,30,0,0,0,0,7,10,1,10,1,8,1,18, - 1,22,2,10,1,4,1,8,1,6,2,6,2,10,1,8, - 1,8,1,14,1,114,193,0,0,0,99,5,0,0,0,0, - 0,0,0,9,0,0,0,5,0,0,0,67,0,0,0,115, - 166,0,0,0,124,4,100,1,107,2,114,18,116,0,124,0, - 131,1,125,5,110,36,124,1,100,2,107,9,114,30,124,1, - 110,2,105,0,125,6,116,1,124,6,131,1,125,7,116,0, - 124,0,124,7,124,4,131,3,125,5,124,3,115,150,124,4, - 100,1,107,2,114,84,116,0,124,0,106,2,100,3,131,1, - 100,1,25,0,131,1,83,0,124,0,115,92,124,5,83,0, - 116,3,124,0,131,1,116,3,124,0,106,2,100,3,131,1, - 100,1,25,0,131,1,24,0,125,8,116,4,106,5,124,5, - 106,6,100,2,116,3,124,5,106,6,131,1,124,8,24,0, - 133,2,25,0,25,0,83,0,110,12,116,7,124,5,124,3, - 116,0,131,3,83,0,100,2,83,0,41,4,97,215,1,0, - 0,73,109,112,111,114,116,32,97,32,109,111,100,117,108,101, - 46,10,10,32,32,32,32,84,104,101,32,39,103,108,111,98, - 97,108,115,39,32,97,114,103,117,109,101,110,116,32,105,115, - 32,117,115,101,100,32,116,111,32,105,110,102,101,114,32,119, - 104,101,114,101,32,116,104,101,32,105,109,112,111,114,116,32, - 105,115,32,111,99,99,117,114,114,105,110,103,32,102,114,111, - 109,10,32,32,32,32,116,111,32,104,97,110,100,108,101,32, - 114,101,108,97,116,105,118,101,32,105,109,112,111,114,116,115, - 46,32,84,104,101,32,39,108,111,99,97,108,115,39,32,97, - 114,103,117,109,101,110,116,32,105,115,32,105,103,110,111,114, - 101,100,46,32,84,104,101,10,32,32,32,32,39,102,114,111, - 109,108,105,115,116,39,32,97,114,103,117,109,101,110,116,32, - 115,112,101,99,105,102,105,101,115,32,119,104,97,116,32,115, - 104,111,117,108,100,32,101,120,105,115,116,32,97,115,32,97, - 116,116,114,105,98,117,116,101,115,32,111,110,32,116,104,101, - 32,109,111,100,117,108,101,10,32,32,32,32,98,101,105,110, - 103,32,105,109,112,111,114,116,101,100,32,40,101,46,103,46, - 32,96,96,102,114,111,109,32,109,111,100,117,108,101,32,105, - 109,112,111,114,116,32,60,102,114,111,109,108,105,115,116,62, - 96,96,41,46,32,32,84,104,101,32,39,108,101,118,101,108, - 39,10,32,32,32,32,97,114,103,117,109,101,110,116,32,114, - 101,112,114,101,115,101,110,116,115,32,116,104,101,32,112,97, - 99,107,97,103,101,32,108,111,99,97,116,105,111,110,32,116, - 111,32,105,109,112,111,114,116,32,102,114,111,109,32,105,110, - 32,97,32,114,101,108,97,116,105,118,101,10,32,32,32,32, - 105,109,112,111,114,116,32,40,101,46,103,46,32,96,96,102, - 114,111,109,32,46,46,112,107,103,32,105,109,112,111,114,116, - 32,109,111,100,96,96,32,119,111,117,108,100,32,104,97,118, - 101,32,97,32,39,108,101,118,101,108,39,32,111,102,32,50, - 41,46,10,10,32,32,32,32,114,19,0,0,0,78,114,117, - 0,0,0,41,8,114,182,0,0,0,114,193,0,0,0,218, - 9,112,97,114,116,105,116,105,111,110,114,159,0,0,0,114, - 14,0,0,0,114,79,0,0,0,114,1,0,0,0,114,189, - 0,0,0,41,9,114,15,0,0,0,114,192,0,0,0,218, - 6,108,111,99,97,108,115,114,187,0,0,0,114,162,0,0, - 0,114,83,0,0,0,90,8,103,108,111,98,97,108,115,95, - 114,161,0,0,0,90,7,99,117,116,95,111,102,102,114,10, - 0,0,0,114,10,0,0,0,114,11,0,0,0,218,10,95, - 95,105,109,112,111,114,116,95,95,23,4,0,0,115,26,0, - 0,0,0,11,8,1,10,2,16,1,8,1,12,1,4,3, - 8,1,18,1,4,1,4,4,26,3,32,2,114,196,0,0, - 0,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, - 0,0,67,0,0,0,115,38,0,0,0,116,0,106,1,124, - 0,131,1,125,1,124,1,100,0,107,8,114,30,116,2,100, - 1,124,0,23,0,131,1,130,1,116,3,124,1,131,1,83, - 0,41,2,78,122,25,110,111,32,98,117,105,108,116,45,105, - 110,32,109,111,100,117,108,101,32,110,97,109,101,100,32,41, - 4,114,142,0,0,0,114,146,0,0,0,114,70,0,0,0, - 114,141,0,0,0,41,2,114,15,0,0,0,114,82,0,0, + 41,3,114,15,0,0,0,114,161,0,0,0,114,162,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,18,95,98,117,105,108,116,105,110,95,102,114,111,109,95, - 110,97,109,101,58,4,0,0,115,8,0,0,0,0,1,10, - 1,8,1,12,1,114,197,0,0,0,99,2,0,0,0,0, - 0,0,0,12,0,0,0,12,0,0,0,67,0,0,0,115, - 244,0,0,0,124,1,97,0,124,0,97,1,116,2,116,1, - 131,1,125,2,120,86,116,1,106,3,106,4,131,0,68,0, - 93,72,92,2,125,3,125,4,116,5,124,4,124,2,131,2, - 114,28,124,3,116,1,106,6,107,6,114,62,116,7,125,5, - 110,18,116,0,106,8,124,3,131,1,114,28,116,9,125,5, - 110,2,113,28,116,10,124,4,124,5,131,2,125,6,116,11, - 124,6,124,4,131,2,1,0,113,28,87,0,116,1,106,3, - 116,12,25,0,125,7,120,54,100,5,68,0,93,46,125,8, - 124,8,116,1,106,3,107,7,114,144,116,13,124,8,131,1, - 125,9,110,10,116,1,106,3,124,8,25,0,125,9,116,14, - 124,7,124,8,124,9,131,3,1,0,113,120,87,0,121,12, - 116,13,100,2,131,1,125,10,87,0,110,24,4,0,116,15, - 107,10,114,206,1,0,1,0,1,0,100,3,125,10,89,0, - 110,2,88,0,116,14,124,7,100,2,124,10,131,3,1,0, - 116,13,100,4,131,1,125,11,116,14,124,7,100,4,124,11, - 131,3,1,0,100,3,83,0,41,6,122,250,83,101,116,117, - 112,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105, - 109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,32, - 98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,115, - 32,97,110,100,32,105,110,106,101,99,116,105,110,103,32,116, - 104,101,109,10,32,32,32,32,105,110,116,111,32,116,104,101, - 32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99, - 101,46,10,10,32,32,32,32,65,115,32,115,121,115,32,105, - 115,32,110,101,101,100,101,100,32,102,111,114,32,115,121,115, - 46,109,111,100,117,108,101,115,32,97,99,99,101,115,115,32, - 97,110,100,32,95,105,109,112,32,105,115,32,110,101,101,100, - 101,100,32,116,111,32,108,111,97,100,32,98,117,105,108,116, - 45,105,110,10,32,32,32,32,109,111,100,117,108,101,115,44, - 32,116,104,111,115,101,32,116,119,111,32,109,111,100,117,108, - 101,115,32,109,117,115,116,32,98,101,32,101,120,112,108,105, - 99,105,116,108,121,32,112,97,115,115,101,100,32,105,110,46, - 10,10,32,32,32,32,114,167,0,0,0,114,20,0,0,0, - 78,114,55,0,0,0,41,1,114,167,0,0,0,41,16,114, - 46,0,0,0,114,14,0,0,0,114,13,0,0,0,114,79, - 0,0,0,218,5,105,116,101,109,115,114,171,0,0,0,114, - 69,0,0,0,114,142,0,0,0,114,75,0,0,0,114,152, - 0,0,0,114,128,0,0,0,114,133,0,0,0,114,1,0, - 0,0,114,197,0,0,0,114,5,0,0,0,114,70,0,0, - 0,41,12,218,10,115,121,115,95,109,111,100,117,108,101,218, - 11,95,105,109,112,95,109,111,100,117,108,101,90,11,109,111, - 100,117,108,101,95,116,121,112,101,114,15,0,0,0,114,83, - 0,0,0,114,93,0,0,0,114,82,0,0,0,90,11,115, - 101,108,102,95,109,111,100,117,108,101,90,12,98,117,105,108, - 116,105,110,95,110,97,109,101,90,14,98,117,105,108,116,105, - 110,95,109,111,100,117,108,101,90,13,116,104,114,101,97,100, - 95,109,111,100,117,108,101,90,14,119,101,97,107,114,101,102, - 95,109,111,100,117,108,101,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,218,6,95,115,101,116,117,112,65,4, - 0,0,115,50,0,0,0,0,9,4,1,4,3,8,1,20, - 1,10,1,10,1,6,1,10,1,6,2,2,1,10,1,14, - 3,10,1,10,1,10,1,10,2,10,1,16,3,2,1,12, - 1,14,2,10,1,12,3,8,1,114,201,0,0,0,99,2, - 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67, - 0,0,0,115,66,0,0,0,116,0,124,0,124,1,131,2, - 1,0,116,1,106,2,106,3,116,4,131,1,1,0,116,1, - 106,2,106,3,116,5,131,1,1,0,100,1,100,2,108,6, - 125,2,124,2,97,7,124,2,106,8,116,1,106,9,116,10, - 25,0,131,1,1,0,100,2,83,0,41,3,122,50,73,110, - 115,116,97,108,108,32,105,109,112,111,114,116,108,105,98,32, - 97,115,32,116,104,101,32,105,109,112,108,101,109,101,110,116, - 97,116,105,111,110,32,111,102,32,105,109,112,111,114,116,46, - 114,19,0,0,0,78,41,11,114,201,0,0,0,114,14,0, - 0,0,114,166,0,0,0,114,109,0,0,0,114,142,0,0, - 0,114,152,0,0,0,218,26,95,102,114,111,122,101,110,95, - 105,109,112,111,114,116,108,105,98,95,101,120,116,101,114,110, - 97,108,114,115,0,0,0,218,8,95,105,110,115,116,97,108, - 108,114,79,0,0,0,114,1,0,0,0,41,3,114,199,0, - 0,0,114,200,0,0,0,114,202,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,203,0,0,0, - 112,4,0,0,115,12,0,0,0,0,2,10,2,12,1,12, - 3,8,1,4,1,114,203,0,0,0,41,2,78,78,41,1, - 78,41,2,78,114,19,0,0,0,41,50,114,3,0,0,0, - 114,115,0,0,0,114,12,0,0,0,114,16,0,0,0,114, - 51,0,0,0,114,29,0,0,0,114,36,0,0,0,114,17, - 0,0,0,114,18,0,0,0,114,41,0,0,0,114,42,0, - 0,0,114,45,0,0,0,114,56,0,0,0,114,58,0,0, - 0,114,68,0,0,0,114,74,0,0,0,114,77,0,0,0, - 114,84,0,0,0,114,95,0,0,0,114,96,0,0,0,114, - 102,0,0,0,114,78,0,0,0,218,6,111,98,106,101,99, - 116,90,9,95,80,79,80,85,76,65,84,69,114,128,0,0, - 0,114,133,0,0,0,114,136,0,0,0,114,91,0,0,0, - 114,80,0,0,0,114,140,0,0,0,114,141,0,0,0,114, - 81,0,0,0,114,142,0,0,0,114,152,0,0,0,114,157, - 0,0,0,114,163,0,0,0,114,165,0,0,0,114,170,0, - 0,0,114,175,0,0,0,90,15,95,69,82,82,95,77,83, - 71,95,80,82,69,70,73,88,114,177,0,0,0,114,180,0, - 0,0,114,181,0,0,0,114,182,0,0,0,114,189,0,0, - 0,114,193,0,0,0,114,196,0,0,0,114,197,0,0,0, - 114,201,0,0,0,114,203,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8, - 60,109,111,100,117,108,101,62,8,0,0,0,115,94,0,0, - 0,4,17,4,2,8,8,8,7,4,2,4,3,16,4,14, - 68,14,21,14,19,8,19,8,19,8,11,14,8,8,11,8, - 12,8,16,8,36,14,27,14,101,16,26,6,3,10,45,14, - 60,8,17,8,17,8,25,8,29,8,23,8,16,14,73,14, - 77,14,13,8,9,8,9,10,47,8,20,4,1,8,2,8, - 27,8,6,10,25,8,31,8,27,18,35,8,7,8,47, + 114,182,0,0,0,206,3,0,0,115,8,0,0,0,0,9, + 12,1,8,1,12,1,114,182,0,0,0,99,3,0,0,0, + 0,0,0,0,6,0,0,0,17,0,0,0,67,0,0,0, + 115,164,0,0,0,116,0,124,0,100,1,131,2,114,160,100, + 2,124,1,107,6,114,58,116,1,124,1,131,1,125,1,124, + 1,106,2,100,2,131,1,1,0,116,0,124,0,100,3,131, + 2,114,58,124,1,106,3,124,0,106,4,131,1,1,0,120, + 100,124,1,68,0,93,92,125,3,116,0,124,0,124,3,131, + 2,115,64,100,4,106,5,124,0,106,6,124,3,131,2,125, + 4,121,14,116,7,124,2,124,4,131,2,1,0,87,0,113, + 64,4,0,116,8,107,10,114,154,1,0,125,5,1,0,122, + 20,124,5,106,9,124,4,107,2,114,136,119,64,130,0,87, + 0,89,0,100,5,100,5,125,5,126,5,88,0,113,64,88, + 0,113,64,87,0,124,0,83,0,41,6,122,238,70,105,103, + 117,114,101,32,111,117,116,32,119,104,97,116,32,95,95,105, + 109,112,111,114,116,95,95,32,115,104,111,117,108,100,32,114, + 101,116,117,114,110,46,10,10,32,32,32,32,84,104,101,32, + 105,109,112,111,114,116,95,32,112,97,114,97,109,101,116,101, + 114,32,105,115,32,97,32,99,97,108,108,97,98,108,101,32, + 119,104,105,99,104,32,116,97,107,101,115,32,116,104,101,32, + 110,97,109,101,32,111,102,32,109,111,100,117,108,101,32,116, + 111,10,32,32,32,32,105,109,112,111,114,116,46,32,73,116, + 32,105,115,32,114,101,113,117,105,114,101,100,32,116,111,32, + 100,101,99,111,117,112,108,101,32,116,104,101,32,102,117,110, + 99,116,105,111,110,32,102,114,111,109,32,97,115,115,117,109, + 105,110,103,32,105,109,112,111,114,116,108,105,98,39,115,10, + 32,32,32,32,105,109,112,111,114,116,32,105,109,112,108,101, + 109,101,110,116,97,116,105,111,110,32,105,115,32,100,101,115, + 105,114,101,100,46,10,10,32,32,32,32,114,127,0,0,0, + 250,1,42,218,7,95,95,97,108,108,95,95,122,5,123,125, + 46,123,125,78,41,10,114,4,0,0,0,114,126,0,0,0, + 218,6,114,101,109,111,118,101,218,6,101,120,116,101,110,100, + 114,184,0,0,0,114,38,0,0,0,114,1,0,0,0,114, + 58,0,0,0,114,178,0,0,0,114,15,0,0,0,41,6, + 114,83,0,0,0,218,8,102,114,111,109,108,105,115,116,114, + 179,0,0,0,218,1,120,90,9,102,114,111,109,95,110,97, + 109,101,90,3,101,120,99,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,218,16,95,104,97,110,100,108,101,95, + 102,114,111,109,108,105,115,116,221,3,0,0,115,32,0,0, + 0,0,10,10,1,8,1,8,1,10,1,10,1,12,1,10, + 1,10,1,14,1,2,1,14,1,16,4,10,1,2,1,24, + 1,114,189,0,0,0,99,1,0,0,0,0,0,0,0,3, + 0,0,0,6,0,0,0,67,0,0,0,115,146,0,0,0, + 124,0,106,0,100,1,131,1,125,1,124,0,106,0,100,2, + 131,1,125,2,124,1,100,3,107,9,114,82,124,2,100,3, + 107,9,114,78,124,1,124,2,106,1,107,3,114,78,116,2, + 106,3,100,4,124,1,155,2,100,5,124,2,106,1,155,2, + 100,6,157,5,116,4,100,7,100,8,141,3,1,0,124,1, + 83,0,124,2,100,3,107,9,114,96,124,2,106,1,83,0, + 116,2,106,3,100,9,116,4,100,7,100,8,141,3,1,0, + 124,0,100,10,25,0,125,1,100,11,124,0,107,7,114,142, + 124,1,106,5,100,12,131,1,100,13,25,0,125,1,124,1, + 83,0,41,14,122,167,67,97,108,99,117,108,97,116,101,32, + 119,104,97,116,32,95,95,112,97,99,107,97,103,101,95,95, + 32,115,104,111,117,108,100,32,98,101,46,10,10,32,32,32, + 32,95,95,112,97,99,107,97,103,101,95,95,32,105,115,32, + 110,111,116,32,103,117,97,114,97,110,116,101,101,100,32,116, + 111,32,98,101,32,100,101,102,105,110,101,100,32,111,114,32, + 99,111,117,108,100,32,98,101,32,115,101,116,32,116,111,32, + 78,111,110,101,10,32,32,32,32,116,111,32,114,101,112,114, + 101,115,101,110,116,32,116,104,97,116,32,105,116,115,32,112, + 114,111,112,101,114,32,118,97,108,117,101,32,105,115,32,117, + 110,107,110,111,119,110,46,10,10,32,32,32,32,114,130,0, + 0,0,114,89,0,0,0,78,122,32,95,95,112,97,99,107, + 97,103,101,95,95,32,33,61,32,95,95,115,112,101,99,95, + 95,46,112,97,114,101,110,116,32,40,122,4,32,33,61,32, + 250,1,41,233,3,0,0,0,41,1,90,10,115,116,97,99, + 107,108,101,118,101,108,122,89,99,97,110,39,116,32,114,101, + 115,111,108,118,101,32,112,97,99,107,97,103,101,32,102,114, + 111,109,32,95,95,115,112,101,99,95,95,32,111,114,32,95, + 95,112,97,99,107,97,103,101,95,95,44,32,102,97,108,108, + 105,110,103,32,98,97,99,107,32,111,110,32,95,95,110,97, + 109,101,95,95,32,97,110,100,32,95,95,112,97,116,104,95, + 95,114,1,0,0,0,114,127,0,0,0,114,117,0,0,0, + 114,19,0,0,0,41,6,114,30,0,0,0,114,119,0,0, + 0,114,167,0,0,0,114,168,0,0,0,114,169,0,0,0, + 114,118,0,0,0,41,3,218,7,103,108,111,98,97,108,115, + 114,161,0,0,0,114,82,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,17,95,99,97,108,99, + 95,95,95,112,97,99,107,97,103,101,95,95,252,3,0,0, + 115,30,0,0,0,0,7,10,1,10,1,8,1,18,1,22, + 2,10,1,4,1,8,1,6,2,6,2,10,1,8,1,8, + 1,14,1,114,193,0,0,0,99,5,0,0,0,0,0,0, + 0,9,0,0,0,5,0,0,0,67,0,0,0,115,166,0, + 0,0,124,4,100,1,107,2,114,18,116,0,124,0,131,1, + 125,5,110,36,124,1,100,2,107,9,114,30,124,1,110,2, + 105,0,125,6,116,1,124,6,131,1,125,7,116,0,124,0, + 124,7,124,4,131,3,125,5,124,3,115,150,124,4,100,1, + 107,2,114,84,116,0,124,0,106,2,100,3,131,1,100,1, + 25,0,131,1,83,0,124,0,115,92,124,5,83,0,116,3, + 124,0,131,1,116,3,124,0,106,2,100,3,131,1,100,1, + 25,0,131,1,24,0,125,8,116,4,106,5,124,5,106,6, + 100,2,116,3,124,5,106,6,131,1,124,8,24,0,133,2, + 25,0,25,0,83,0,110,12,116,7,124,5,124,3,116,0, + 131,3,83,0,100,2,83,0,41,4,97,215,1,0,0,73, + 109,112,111,114,116,32,97,32,109,111,100,117,108,101,46,10, + 10,32,32,32,32,84,104,101,32,39,103,108,111,98,97,108, + 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,117, + 115,101,100,32,116,111,32,105,110,102,101,114,32,119,104,101, + 114,101,32,116,104,101,32,105,109,112,111,114,116,32,105,115, + 32,111,99,99,117,114,114,105,110,103,32,102,114,111,109,10, + 32,32,32,32,116,111,32,104,97,110,100,108,101,32,114,101, + 108,97,116,105,118,101,32,105,109,112,111,114,116,115,46,32, + 84,104,101,32,39,108,111,99,97,108,115,39,32,97,114,103, + 117,109,101,110,116,32,105,115,32,105,103,110,111,114,101,100, + 46,32,84,104,101,10,32,32,32,32,39,102,114,111,109,108, + 105,115,116,39,32,97,114,103,117,109,101,110,116,32,115,112, + 101,99,105,102,105,101,115,32,119,104,97,116,32,115,104,111, + 117,108,100,32,101,120,105,115,116,32,97,115,32,97,116,116, + 114,105,98,117,116,101,115,32,111,110,32,116,104,101,32,109, + 111,100,117,108,101,10,32,32,32,32,98,101,105,110,103,32, + 105,109,112,111,114,116,101,100,32,40,101,46,103,46,32,96, + 96,102,114,111,109,32,109,111,100,117,108,101,32,105,109,112, + 111,114,116,32,60,102,114,111,109,108,105,115,116,62,96,96, + 41,46,32,32,84,104,101,32,39,108,101,118,101,108,39,10, + 32,32,32,32,97,114,103,117,109,101,110,116,32,114,101,112, + 114,101,115,101,110,116,115,32,116,104,101,32,112,97,99,107, + 97,103,101,32,108,111,99,97,116,105,111,110,32,116,111,32, + 105,109,112,111,114,116,32,102,114,111,109,32,105,110,32,97, + 32,114,101,108,97,116,105,118,101,10,32,32,32,32,105,109, + 112,111,114,116,32,40,101,46,103,46,32,96,96,102,114,111, + 109,32,46,46,112,107,103,32,105,109,112,111,114,116,32,109, + 111,100,96,96,32,119,111,117,108,100,32,104,97,118,101,32, + 97,32,39,108,101,118,101,108,39,32,111,102,32,50,41,46, + 10,10,32,32,32,32,114,19,0,0,0,78,114,117,0,0, + 0,41,8,114,182,0,0,0,114,193,0,0,0,218,9,112, + 97,114,116,105,116,105,111,110,114,159,0,0,0,114,14,0, + 0,0,114,79,0,0,0,114,1,0,0,0,114,189,0,0, + 0,41,9,114,15,0,0,0,114,192,0,0,0,218,6,108, + 111,99,97,108,115,114,187,0,0,0,114,162,0,0,0,114, + 83,0,0,0,90,8,103,108,111,98,97,108,115,95,114,161, + 0,0,0,90,7,99,117,116,95,111,102,102,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,10,95,95,105, + 109,112,111,114,116,95,95,23,4,0,0,115,26,0,0,0, + 0,11,8,1,10,2,16,1,8,1,12,1,4,3,8,1, + 18,1,4,1,4,4,26,3,32,2,114,196,0,0,0,99, + 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, + 67,0,0,0,115,38,0,0,0,116,0,106,1,124,0,131, + 1,125,1,124,1,100,0,107,8,114,30,116,2,100,1,124, + 0,23,0,131,1,130,1,116,3,124,1,131,1,83,0,41, + 2,78,122,25,110,111,32,98,117,105,108,116,45,105,110,32, + 109,111,100,117,108,101,32,110,97,109,101,100,32,41,4,114, + 142,0,0,0,114,146,0,0,0,114,70,0,0,0,114,141, + 0,0,0,41,2,114,15,0,0,0,114,82,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,18, + 95,98,117,105,108,116,105,110,95,102,114,111,109,95,110,97, + 109,101,58,4,0,0,115,8,0,0,0,0,1,10,1,8, + 1,12,1,114,197,0,0,0,99,2,0,0,0,0,0,0, + 0,12,0,0,0,12,0,0,0,67,0,0,0,115,244,0, + 0,0,124,1,97,0,124,0,97,1,116,2,116,1,131,1, + 125,2,120,86,116,1,106,3,106,4,131,0,68,0,93,72, + 92,2,125,3,125,4,116,5,124,4,124,2,131,2,114,28, + 124,3,116,1,106,6,107,6,114,62,116,7,125,5,110,18, + 116,0,106,8,124,3,131,1,114,28,116,9,125,5,110,2, + 113,28,116,10,124,4,124,5,131,2,125,6,116,11,124,6, + 124,4,131,2,1,0,113,28,87,0,116,1,106,3,116,12, + 25,0,125,7,120,54,100,5,68,0,93,46,125,8,124,8, + 116,1,106,3,107,7,114,144,116,13,124,8,131,1,125,9, + 110,10,116,1,106,3,124,8,25,0,125,9,116,14,124,7, + 124,8,124,9,131,3,1,0,113,120,87,0,121,12,116,13, + 100,2,131,1,125,10,87,0,110,24,4,0,116,15,107,10, + 114,206,1,0,1,0,1,0,100,3,125,10,89,0,110,2, + 88,0,116,14,124,7,100,2,124,10,131,3,1,0,116,13, + 100,4,131,1,125,11,116,14,124,7,100,4,124,11,131,3, + 1,0,100,3,83,0,41,6,122,250,83,101,116,117,112,32, + 105,109,112,111,114,116,108,105,98,32,98,121,32,105,109,112, + 111,114,116,105,110,103,32,110,101,101,100,101,100,32,98,117, + 105,108,116,45,105,110,32,109,111,100,117,108,101,115,32,97, + 110,100,32,105,110,106,101,99,116,105,110,103,32,116,104,101, + 109,10,32,32,32,32,105,110,116,111,32,116,104,101,32,103, + 108,111,98,97,108,32,110,97,109,101,115,112,97,99,101,46, + 10,10,32,32,32,32,65,115,32,115,121,115,32,105,115,32, + 110,101,101,100,101,100,32,102,111,114,32,115,121,115,46,109, + 111,100,117,108,101,115,32,97,99,99,101,115,115,32,97,110, + 100,32,95,105,109,112,32,105,115,32,110,101,101,100,101,100, + 32,116,111,32,108,111,97,100,32,98,117,105,108,116,45,105, + 110,10,32,32,32,32,109,111,100,117,108,101,115,44,32,116, + 104,111,115,101,32,116,119,111,32,109,111,100,117,108,101,115, + 32,109,117,115,116,32,98,101,32,101,120,112,108,105,99,105, + 116,108,121,32,112,97,115,115,101,100,32,105,110,46,10,10, + 32,32,32,32,114,167,0,0,0,114,20,0,0,0,78,114, + 55,0,0,0,41,1,114,167,0,0,0,41,16,114,46,0, + 0,0,114,14,0,0,0,114,13,0,0,0,114,79,0,0, + 0,218,5,105,116,101,109,115,114,171,0,0,0,114,69,0, + 0,0,114,142,0,0,0,114,75,0,0,0,114,152,0,0, + 0,114,128,0,0,0,114,133,0,0,0,114,1,0,0,0, + 114,197,0,0,0,114,5,0,0,0,114,70,0,0,0,41, + 12,218,10,115,121,115,95,109,111,100,117,108,101,218,11,95, + 105,109,112,95,109,111,100,117,108,101,90,11,109,111,100,117, + 108,101,95,116,121,112,101,114,15,0,0,0,114,83,0,0, + 0,114,93,0,0,0,114,82,0,0,0,90,11,115,101,108, + 102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,105, + 110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,95, + 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, + 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, + 111,100,117,108,101,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,218,6,95,115,101,116,117,112,65,4,0,0, + 115,50,0,0,0,0,9,4,1,4,3,8,1,20,1,10, + 1,10,1,6,1,10,1,6,2,2,1,10,1,14,3,10, + 1,10,1,10,1,10,2,10,1,16,3,2,1,12,1,14, + 2,10,1,12,3,8,1,114,201,0,0,0,99,2,0,0, + 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, + 0,115,66,0,0,0,116,0,124,0,124,1,131,2,1,0, + 116,1,106,2,106,3,116,4,131,1,1,0,116,1,106,2, + 106,3,116,5,131,1,1,0,100,1,100,2,108,6,125,2, + 124,2,97,7,124,2,106,8,116,1,106,9,116,10,25,0, + 131,1,1,0,100,2,83,0,41,3,122,50,73,110,115,116, + 97,108,108,32,105,109,112,111,114,116,108,105,98,32,97,115, + 32,116,104,101,32,105,109,112,108,101,109,101,110,116,97,116, + 105,111,110,32,111,102,32,105,109,112,111,114,116,46,114,19, + 0,0,0,78,41,11,114,201,0,0,0,114,14,0,0,0, + 114,166,0,0,0,114,109,0,0,0,114,142,0,0,0,114, + 152,0,0,0,218,26,95,102,114,111,122,101,110,95,105,109, + 112,111,114,116,108,105,98,95,101,120,116,101,114,110,97,108, + 114,115,0,0,0,218,8,95,105,110,115,116,97,108,108,114, + 79,0,0,0,114,1,0,0,0,41,3,114,199,0,0,0, + 114,200,0,0,0,114,202,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,114,203,0,0,0,112,4, + 0,0,115,12,0,0,0,0,2,10,2,12,1,12,3,8, + 1,4,1,114,203,0,0,0,41,2,78,78,41,1,78,41, + 2,78,114,19,0,0,0,41,50,114,3,0,0,0,114,115, + 0,0,0,114,12,0,0,0,114,16,0,0,0,114,51,0, + 0,0,114,29,0,0,0,114,36,0,0,0,114,17,0,0, + 0,114,18,0,0,0,114,41,0,0,0,114,42,0,0,0, + 114,45,0,0,0,114,56,0,0,0,114,58,0,0,0,114, + 68,0,0,0,114,74,0,0,0,114,77,0,0,0,114,84, + 0,0,0,114,95,0,0,0,114,96,0,0,0,114,102,0, + 0,0,114,78,0,0,0,218,6,111,98,106,101,99,116,90, + 9,95,80,79,80,85,76,65,84,69,114,128,0,0,0,114, + 133,0,0,0,114,136,0,0,0,114,91,0,0,0,114,80, + 0,0,0,114,140,0,0,0,114,141,0,0,0,114,81,0, + 0,0,114,142,0,0,0,114,152,0,0,0,114,157,0,0, + 0,114,163,0,0,0,114,165,0,0,0,114,170,0,0,0, + 114,175,0,0,0,90,15,95,69,82,82,95,77,83,71,95, + 80,82,69,70,73,88,114,177,0,0,0,114,180,0,0,0, + 114,181,0,0,0,114,182,0,0,0,114,189,0,0,0,114, + 193,0,0,0,114,196,0,0,0,114,197,0,0,0,114,201, + 0,0,0,114,203,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,218,8,60,109, + 111,100,117,108,101,62,8,0,0,0,115,94,0,0,0,4, + 17,4,2,8,8,8,7,4,2,4,3,16,4,14,68,14, + 21,14,19,8,19,8,19,8,11,14,8,8,11,8,12,8, + 16,8,36,14,27,14,101,16,26,6,3,10,45,14,60,8, + 17,8,17,8,25,8,29,8,23,8,16,14,73,14,77,14, + 13,8,9,8,9,10,47,8,20,4,1,8,2,8,27,8, + 16,10,15,8,31,8,27,18,35,8,7,8,47, }; From d0126481292e4699a14e162e5ac7f53d5d590c8f Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 6 Jul 2017 15:29:35 -0400 Subject: [PATCH 0528/1345] [3.6] Update idlelib/NEWS.txt with merges upto 2017 Jul 5. (GH-2595) (#2596) (cherry picked from commit 1ccbad9c95cf5782a1329eeaecba6e3eb0c37cb8) --- Lib/idlelib/NEWS.txt | 49 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index f12c185bbbac81..af88048fa0ca2e 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,5 +1,50 @@ +What's New in IDLE 3.6.3 +Released on 2017-09-25? +======================== + + +bpo-30777: Configdialog - add docstrings and improve comments. +Patch by Cheryl Sabella. + +bpo-30495: Improve textview with docstrings, PEP8 names, and more tests. +Split TextViewer class into ViewWindow, ViewFrame, and TextFrame classes +so that instances of the latter two can be placed with other widgets +within a multiframe window. +Patches by Cheryl Sabella and Terry Jan Reedy. + +bp0-30723: Make several improvements to parenmatch. +* Add 'parens' style to highlight both opener and closer. +* Make 'default' style, which is not default, a synonym for 'opener'. +* Make time-delay work the same with all styles. +* Add help for config dialog extensions tab, including parenmatch. +* Add new tests. +Original patch by Charles Wohlganger. Revisions by Terry Jan Reedy + +bpo-30674: Grep -- Add docstrings. Patch by Cheryl Sabella. + +bpo-21519: IDLE's basic custom key entry dialog now detects +duplicates properly. Original patch by Saimadhav Heblikar. + +bpo-29910: IDLE no longer deletes a character after commenting out a +region by a key shortcut. Add "return 'break'" for this and other +potential conflicts between IDLE and default key bindings. +Patch by Serhiy Storchaka. + +bpo-30728: Modernize idlelib.configdialog: +* replace import * with specific imports; +* lowercase method and attribute lines. +Patch by Cheryl Sabella. + +bpo-6739: Verify user-entered key sequences by trying to bind them +with to a tk widget. Add tests for all 3 validation functions. +Original patch by G Polo. Tests added by Cheryl Sabella. +Code revised and more tests added by Terry Jan Reedy + +bpo-24813: Add icon to help_about and make other changes. + + What's New in IDLE 3.6.2 -Released on 2017-06-30? +Released on 2017-07-11 ======================== bpo-15786: Fix several problems with IDLE's autocompletion box. @@ -29,7 +74,7 @@ What's New in IDLE 3.6.1 Released on 2017-03-21 ======================== -Issue #29071: IDLE colors f-string prefixes (but not invalid ur prefixes). +Issue #29071: IDLE colors f-string prefixes but not invalid ur prefixes. Issue #28572: Add 10% to coverage of IDLE's test_configdialog. Update and augment description of the configuration system. From 5d2550cd2ef26c3f5625ab5a1afe3269e44dcb8f Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Fri, 7 Jul 2017 00:08:29 +0300 Subject: [PATCH 0529/1345] [3.6] Fix case in .gitignore (GH-2607) (GH-2608) (cherry picked from commit be5ebe58776a02d53bcf5e43ab946b555c0e025f) --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9392fe5defd6f5..bb5b13bf754d17 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,7 @@ PCbuild/*.VC.opendb PCbuild/.vs/ PCbuild/amd64/ PCbuild/obj/ -PCBuild/win32/ +PCbuild/win32/ .purify Parser/pgen __pycache__ From c60d2f5e8609b040ab58c498fde23928fe9dbef5 Mon Sep 17 00:00:00 2001 From: Joel Hillacre Date: Thu, 6 Jul 2017 15:28:22 -0600 Subject: [PATCH 0530/1345] bpo-30532: Fix whitespace folding in certain cases (#2591) Leading whitespace was incorrectly dropped during folding of certain lines in the _header_value_parser's folding algorithm. This makes the whitespace handling code consistent. --- Lib/email/_header_value_parser.py | 4 +--- Lib/test/test_email/test__header_value_parser.py | 12 ++++++++++++ Misc/ACKS | 1 + .../Library/2017-06-26-11-01-59.bpo-30532.qTeL1o.rst | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-06-26-11-01-59.bpo-30532.qTeL1o.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 57d01fbcb0f2c2..9b9697f77346a6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -341,9 +341,7 @@ def _fold(self, folded): # avoid infinite recursion. ws = part.pop_leading_fws() if ws is not None: - # Peel off the leading whitespace and make it sticky, to - # avoid infinite recursion. - folded.stickyspace = str(part.pop(0)) + folded.stickyspace = str(ws) if folded.append_if_fits(part): continue if part.has_fws: diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 26ece6911f2377..e0ec87d2080185 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2711,5 +2711,17 @@ def test_whitespace_splitting(self): self._test(parser.get_unstructured('xxx ' + 'y'*77), 'xxx \n ' + 'y'*77 + '\n') + def test_long_filename_attachment(self): + folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"') + self.assertEqual( + 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"\n', + folded + ) + folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"') + self.assertEqual( + 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"\n', + folded + ) + if __name__ == '__main__': unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 3420d4b6894f48..940e07e6d6e667 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -624,6 +624,7 @@ Wouter van Heyst Kelsey Hightower Jason Hildebrand Aaron Hill +Joel Hillacre Richie Hindle Konrad Hinsen David Hobley diff --git a/Misc/NEWS.d/next/Library/2017-06-26-11-01-59.bpo-30532.qTeL1o.rst b/Misc/NEWS.d/next/Library/2017-06-26-11-01-59.bpo-30532.qTeL1o.rst new file mode 100644 index 00000000000000..adce85fbc2b4c6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-06-26-11-01-59.bpo-30532.qTeL1o.rst @@ -0,0 +1 @@ +Fix email header value parser dropping folding white space in certain cases. From df0f99329843c10701ffaefbd3948ac698c12220 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 6 Jul 2017 23:16:31 -0400 Subject: [PATCH 0531/1345] [3.6] bpo-30780: Fix error in idlelib.test_idle.test_configdialog (GH-2606) (#2613) (cherry picked from commit 25a4206) --- Lib/idlelib/idle_test/test_configdialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 4946f02306a786..ffc8a723ca0e6a 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -44,7 +44,7 @@ def setUpModule(): def tearDownModule(): global root, configure - idleConf.userCfg = testcfg + idleConf.userCfg = usercfg configure.remove_var_callbacks() del configure root.update_idletasks() From 00134f64d982561750f57f1a0bb7bb073f9f237c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 19 Jun 2017 11:12:56 -0700 Subject: [PATCH 0532/1345] bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat (#2252) (#2280) * Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat Also fixes bdist_wininst.vcxproj to use correct version in generated name. (cherry picked from commit 06d6e3d0bb5b8a3d3105289034953a8014356a0b) --- Misc/NEWS | 3 ++ PC/bdist_wininst/bdist_wininst.vcxproj | 4 ++ PC/bdist_wininst/build.bat | 11 ++---- PCbuild/build.bat | 32 ++++++---------- PCbuild/find_msbuild.bat | 51 ++++++++++++++++++++++++++ PCbuild/python.vcxproj | 5 ++- Tools/msi/build.bat | 10 ++--- Tools/msi/buildrelease.bat | 15 ++++---- Tools/msi/uploadrelease.bat | 15 ++++---- Tools/nuget/build.bat | 7 ++-- 10 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 PCbuild/find_msbuild.bat diff --git a/Misc/NEWS b/Misc/NEWS index bccdf94e24983b..2349eaaa0fb28d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -327,6 +327,9 @@ Tests Windows ------- +- bpo-30687: Locate msbuild.exe on Windows when building rather than + vcvarsall.bat + - bpo-30450: The build process on Windows no longer depends on Subversion, instead pulling external code from GitHub via a Python script. If Python 3.6 is not found on the system (via ``py -3.6``), NuGet is used to download a diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj b/PC/bdist_wininst/bdist_wininst.vcxproj index 4dc5009b58e6ac..9bbfda28ccbe2b 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj +++ b/PC/bdist_wininst/bdist_wininst.vcxproj @@ -58,6 +58,10 @@ $(PySourcePath)lib\distutils\command\ false wininst-$(VisualStudioVersion) + wininst-14.0 + wininst-12.0 + wininst-11.0 + wininst-10.0 $(TargetName)-amd64 .exe diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index ee6856754faa8c..d5cf75206dd810 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -7,16 +7,13 @@ set PCBUILD=%~dp0..\..\PCBuild\ echo Building Lib\distutils\command\wininst-xx.0.exe -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 if errorlevel 1 goto :eof echo Building Lib\distutils\command\wininst-xx.0-amd64.exe -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8e1c168a8591f8..b63843fa95d971 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -48,7 +48,6 @@ exit /b 127 :Run setlocal set platf=Win32 -set vs_platf=x86 set conf=Release set target=Build set dir=%~dp0 @@ -57,10 +56,6 @@ set verbose=/nologo /v:m set kill= set do_pgo= set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) :CheckOpts if "%~1"=="-h" goto Usage @@ -90,18 +85,12 @@ if "%IncludeTkinter%"=="" set IncludeTkinter=true if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 ) ) @@ -110,7 +99,8 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul +call "%dir%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if "%kill%"=="true" call :Kill @@ -129,7 +119,7 @@ if "%do_pgo%"=="true" ( goto Build :Kill echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ +%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:KillPython=true @@ -141,7 +131,7 @@ rem Call on MSBuild to do the work, echo the command. rem Passing %1-9 is not the preferred option, but argument parsing in rem batch is, shall we say, "lackluster" echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ +%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ @@ -153,4 +143,4 @@ goto :eof :Version rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat new file mode 100644 index 00000000000000..1877906e00a55d --- /dev/null +++ b/PCbuild/find_msbuild.bat @@ -0,0 +1,51 @@ +@rem +@rem Searches for MSBuild.exe. This is the only tool we need to initiate +@rem a build, so we no longer search for the full VC toolset. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found + +:begin_search +@set MSBUILD= + +@rem If msbuild.exe is on the PATH, assume that the user wants that one. +@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" +@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found + +@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( + @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found + +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + + +@exit /b 1 + +:found +@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%) +@set _Py_MSBuild_Source= diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 807213f12a76f4..2786ac2ebf7b6b 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -83,7 +83,7 @@ - + ucrtbase ucrtbased @@ -94,6 +94,8 @@ set PYTHONPATH=$(PySourcePath)Lib + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64 <_Content>@rem This script invokes the most recently built Python with all arguments @rem passed through to the interpreter. This file is generated by the @rem build process and any changes *will* be thrown away by the next @@ -103,6 +105,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) +$(_PGOPath) @"$(OutDir)python$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index 59e0261c6d5236..f720ea9a86a8cb 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,8 +22,8 @@ if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%get_externals.bat" - -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined BUILDX86 ( call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% @@ -44,7 +44,7 @@ if defined BUILDDOC ( ) rem Build the launcher MSI separately -msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 +%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 set BUILD_CMD="%D%bundle\snapshot.wixproj" if defined BUILDTEST ( @@ -58,11 +58,11 @@ if defined REBUILD ( ) if defined BUILDX86 ( - msbuild %BUILD_CMD% + %MSBUILD% %BUILD_CMD% if errorlevel 1 goto :eof ) if defined BUILDX64 ( - msbuild /p:Platform=x64 %BUILD_CMD% + %MSBUILD% /p:Platform=x64 %BUILD_CMD% if errorlevel 1 goto :eof ) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index ed930c653c11ea..f60fe6af393165 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -69,6 +69,8 @@ if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) :builddoc if "%SKIPBUILD%" EQU "1" goto skipdoc @@ -165,28 +167,27 @@ if not "%SKIPBUILD%" EQU "1" ( @echo off ) -call "%PCBUILD%env.bat" if "%OUTDIR_PLAT%" EQU "win32" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true +%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true if errorlevel 1 exit /B -msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false +%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false if errorlevel 1 exit /B if defined BUILDZIP ( - msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) if defined BUILDNUGET ( - msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 0873d55589f0db..610b092b863374 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -49,16 +49,17 @@ if defined NOGPG ( echo Found gpg2.exe at %GPG% ) -call "%PCBUILD%env.bat" > nul 2> nul +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% if not defined NOTEST ( - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 ) -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd exit /B 0 diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 5068c17244456e..54498ca28a2471 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -21,7 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%..\msi\get_externals.bat" -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" @@ -30,7 +31,7 @@ if defined BUILDX86 ( ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) @@ -39,7 +40,7 @@ if defined BUILDX64 ( ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) From b0fba8874a4bd6bf4773e6efdbd8fa762e9f05bd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 20 Jun 2017 15:37:24 +0200 Subject: [PATCH 0533/1345] bpo-30500: urllib: Simplify splithost by calling into urlparse. (#1849) (#2289) The current regex based splitting produces a wrong result. For example:: http://abc#@def Web browsers parse that URL as ``http://abc/#@def``, that is, the host is ``abc``, the path is ``/``, and the fragment is ``#@def``. (cherry picked from commit 90e01e50ef8a9e6c91f30d965563c378a4ad26de) (cherry picked from commit 536c1f1246f4faa302f9f5613fc3444e7ae09b4a) --- Lib/test/test_urlparse.py | 51 ++++++++++++++++++++++++++++++--------- Lib/urllib/parse.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 17 +++++++++++++ 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 3c89ed928ff51f..e5f6130e4a075e 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -755,28 +755,35 @@ def test_default_scheme(self): def test_parse_fragments(self): # Exercise the allow_fragments parameter of urlparse() and urlsplit() tests = ( - ("http:#frag", "path"), - ("//example.net#frag", "path"), - ("index.html#frag", "path"), - (";a=b#frag", "params"), - ("?a=b#frag", "query"), - ("#frag", "path"), + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), ) - for url, attr in tests: + for url, attr, expected_frag in tests: for func in (urllib.parse.urlparse, urllib.parse.urlsplit): if attr == "params" and func is urllib.parse.urlsplit: attr = "path" with self.subTest(url=url, function=func): result = func(url, allow_fragments=False) self.assertEqual(result.fragment, "") - self.assertTrue(getattr(result, attr).endswith("#frag")) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) self.assertEqual(func(url, "", False).fragment, "") result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, "frag") - self.assertFalse(getattr(result, attr).endswith("frag")) - self.assertEqual(func(url, "", True).fragment, "frag") - self.assertEqual(func(url).fragment, "frag") + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -983,6 +990,26 @@ def test_splithost(self): self.assertEqual(splithost('/foo/bar/baz.html'), (None, '/foo/bar/baz.html')) + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + def test_splituser(self): splituser = urllib.parse.splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 888247b8e90f56..3cab2d13d5fb56 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -944,7 +944,7 @@ def splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: diff --git a/Misc/ACKS b/Misc/ACKS index 64484f3f8f26aa..03f9ea6cc97ae3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1081,6 +1081,7 @@ Max Neunhöffer Anthon van der Neut George Neville-Neil Hieu Nguyen +Nam Nguyen Johannes Nicolai Samuel Nicolary Jonathan Niehof diff --git a/Misc/NEWS b/Misc/NEWS index 2349eaaa0fb28d..3a52a3c4635771 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,23 @@ Python News +++++++++++ +What's New in Python 3.6.3 release candidate 1? +=============================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + +- [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse + fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now + correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` + as the host in an authentification (``login@host``). + + What's New in Python 3.6.2 release candidate 1? =============================================== From ea1ab803ddc14ab02ffed50ecc5089897f259623 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Jun 2017 16:05:11 +0200 Subject: [PATCH 0534/1345] bpo-30694: Upgrade Modules/expat/ to libexpat 2.2.1 (#2300) (#2313) New file: Modules/expat/siphash.h. (cherry picked from commit 5ff7132313eb651107b179d20218dfe5d4e47f13) (cherry picked from commit 4a66524006852fc982aebafa277f2c043d9ad149) --- Misc/NEWS | 9 + Modules/expat/COPYING | 2 +- Modules/expat/expat.h | 22 +- Modules/expat/expat_external.h | 5 +- Modules/expat/siphash.h | 344 +++++++++++++++++++ Modules/expat/winconfig.h | 10 + Modules/expat/xmlparse.c | 609 ++++++++++++++++++++++++++------- Modules/expat/xmlrole.c | 10 +- Modules/expat/xmltok.c | 34 +- 9 files changed, 887 insertions(+), 158 deletions(-) create mode 100644 Modules/expat/siphash.h diff --git a/Misc/NEWS b/Misc/NEWS index 3a52a3c4635771..1f6161a1f10177 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,15 @@ Core and Builtins Library ------- +- [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes + of multiple security vulnerabilities including: CVE-2017-9233 (External + entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), + CVE-2016-0718 (Fix regression bugs from 2.2.0's fix to CVE-2016-0718) + and CVE-2012-0876 (Counter hash flooding with SipHash). + Note: the CVE-2016-5300 (Use os-specific entropy sources like getrandom) + doesn't impact Python, since Python already gets entropy from the OS to set + the expat secret using ``XML_SetHashSalt()``. + - [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 092c83baee13ae..8d288f0f28fddd 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2016 Expat maintainers +Copyright (c) 2001-2017 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 086e24b39c51ae..28b0f954d41963 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -95,7 +95,9 @@ enum XML_Error { /* Added in 2.0. */ XML_ERROR_RESERVED_PREFIX_XML, XML_ERROR_RESERVED_PREFIX_XMLNS, - XML_ERROR_RESERVED_NAMESPACE_URI + XML_ERROR_RESERVED_NAMESPACE_URI, + /* Added in 2.2.1. */ + XML_ERROR_INVALID_ARGUMENT }; enum XML_Content_Type { @@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser); be called, despite an external subset being parsed. Note: If XML_DTD is not defined when Expat is compiled, returns XML_ERROR_FEATURE_REQUIRES_XML_DTD. + Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT. */ XMLPARSEAPI(enum XML_Error) XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD); @@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser); to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus this correspondds to an index into the atts array passed to the - XML_StartElementHandler. + XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to - XML_StartElementHandler, or -1 if there is no ID attribute. Each - attribute/value pair counts as 2; thus this correspondds to an - index into the atts array passed to the XML_StartElementHandler. + XML_StartElementHandler, or -1 if there is no ID attribute or + parser == NULL. Each attribute/value pair counts as 2; thus this + correspondds to an index into the atts array passed to the + XML_StartElementHandler. */ XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); @@ -901,6 +905,7 @@ enum XML_ParamEntityParsing { entities is requested; otherwise it will return non-zero. Note: If XML_SetParamEntityParsing is called after XML_Parse or XML_ParseBuffer, then it has no effect and will always return 0. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetParamEntityParsing(XML_Parser parser, @@ -910,6 +915,7 @@ XML_SetParamEntityParsing(XML_Parser parser, Helps in preventing DoS attacks based on predicting hash function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, @@ -936,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser); the location is the location of the character at which the error was detected; otherwise the location is the location of the last parse event, as described above. + + Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber + return 0 to indicate an error. + Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); @@ -1039,7 +1049,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 0 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 4860c5777aa018..4c9e5eabdee2ce 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -97,7 +97,10 @@ extern "C" { #endif #ifdef XML_UNICODE_WCHAR_T -#define XML_UNICODE +# define XML_UNICODE +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif #endif #ifdef XML_UNICODE /* Information is UTF-16 encoded. */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h new file mode 100644 index 00000000000000..23b56d2ae48754 --- /dev/null +++ b/Modules/expat/siphash.h @@ -0,0 +1,344 @@ +/* ========================================================================== + * siphash.h - SipHash-2-4 in a single header file + * -------------------------------------------------------------------------- + * Derived by William Ahern from the reference implementation[1] published[2] + * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. + * by Jean-Philippe Aumasson and Daniel J. Berstein. + * Minimal changes by Sebastian Pipping on top, details below. + * Licensed under the CC0 Public Domain Dedication license. + * + * 1. https://www.131002.net/siphash/siphash24.c + * 2. https://www.131002.net/siphash/ + * -------------------------------------------------------------------------- + * HISTORY: + * + * 2017-06-10 (Sebastian Pipping) + * - Clarify license note in the header + * - Address C89 issues: + * - Stop using inline keyword (and let compiler decide) + * - Turn integer suffix ULL to UL + * - Replace _Bool by int + * - Turn macro siphash24 into a function + * - Address invalid conversion (void pointer) by explicit cast + * - Always expose sip24_valid (for self-tests) + * + * 2012-11-04 - Born. (William Ahern) + * -------------------------------------------------------------------------- + * USAGE: + * + * SipHash-2-4 takes as input two 64-bit words as the key, some number of + * message bytes, and outputs a 64-bit word as the message digest. This + * implementation employs two data structures: a struct sipkey for + * representing the key, and a struct siphash for representing the hash + * state. + * + * For converting a 16-byte unsigned char array to a key, use either the + * macro sip_keyof or the routine sip_tokey. The former instantiates a + * compound literal key, while the latter requires a key object as a + * parameter. + * + * unsigned char secret[16]; + * arc4random_buf(secret, sizeof secret); + * struct sipkey *key = sip_keyof(secret); + * + * For hashing a message, use either the convenience macro siphash24 or the + * routines sip24_init, sip24_update, and sip24_final. + * + * struct siphash state; + * void *msg; + * size_t len; + * uint64_t hash; + * + * sip24_init(&state, key); + * sip24_update(&state, msg, len); + * hash = sip24_final(&state); + * + * or + * + * hash = siphash24(msg, len, key); + * + * To convert the 64-bit hash value to a canonical 8-byte little-endian + * binary representation, use either the macro sip_binof or the routine + * sip_tobin. The former instantiates and returns a compound literal array, + * while the latter requires an array object as a parameter. + * -------------------------------------------------------------------------- + * NOTES: + * + * o Neither sip_keyof, sip_binof, nor siphash24 will work with compilers + * lacking compound literal support. Instead, you must use the lower-level + * interfaces which take as parameters the temporary state objects. + * + * o Uppercase macros may evaluate parameters more than once. Lowercase + * macros should not exhibit any such side effects. + * ========================================================================== + */ +#ifndef SIPHASH_H +#define SIPHASH_H + +#include /* size_t */ +#include /* uint64_t uint32_t uint8_t */ + + +#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) + +#define SIP_U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v) >> 0); (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24); + +#define SIP_U64TO8_LE(p, v) \ + SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0)); \ + SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define SIP_U8TO64_LE(p) \ + (((uint64_t)((p)[0]) << 0) | \ + ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | \ + ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | \ + ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | \ + ((uint64_t)((p)[7]) << 56)) + + +#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 } + +struct siphash { + uint64_t v0, v1, v2, v3; + + unsigned char buf[8], *p; + uint64_t c; +}; /* struct siphash */ + + +#define SIP_KEYLEN 16 + +struct sipkey { + uint64_t k[2]; +}; /* struct sipkey */ + +#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k)) + +static struct sipkey *sip_tokey(struct sipkey *key, const void *src) { + key->k[0] = SIP_U8TO64_LE((const unsigned char *)src); + key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8); + return key; +} /* sip_tokey() */ + + +#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v)) + +static void *sip_tobin(void *dst, uint64_t u64) { + SIP_U64TO8_LE((unsigned char *)dst, u64); + return dst; +} /* sip_tobin() */ + + +static void sip_round(struct siphash *H, const int rounds) { + int i; + + for (i = 0; i < rounds; i++) { + H->v0 += H->v1; + H->v1 = SIP_ROTL(H->v1, 13); + H->v1 ^= H->v0; + H->v0 = SIP_ROTL(H->v0, 32); + + H->v2 += H->v3; + H->v3 = SIP_ROTL(H->v3, 16); + H->v3 ^= H->v2; + + H->v0 += H->v3; + H->v3 = SIP_ROTL(H->v3, 21); + H->v3 ^= H->v0; + + H->v2 += H->v1; + H->v1 = SIP_ROTL(H->v1, 17); + H->v1 ^= H->v2; + H->v2 = SIP_ROTL(H->v2, 32); + } +} /* sip_round() */ + + +static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { + H->v0 = 0x736f6d6570736575UL ^ key->k[0]; + H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; + H->v2 = 0x6c7967656e657261UL ^ key->k[0]; + H->v3 = 0x7465646279746573UL ^ key->k[1]; + + H->p = H->buf; + H->c = 0; + + return H; +} /* sip24_init() */ + + +#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) + +static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { + const unsigned char *p = (const unsigned char *)src, *pe = p + len; + uint64_t m; + + do { + while (p < pe && H->p < sip_endof(H->buf)) + *H->p++ = *p++; + + if (H->p < sip_endof(H->buf)) + break; + + m = SIP_U8TO64_LE(H->buf); + H->v3 ^= m; + sip_round(H, 2); + H->v0 ^= m; + + H->p = H->buf; + H->c += 8; + } while (p < pe); + + return H; +} /* sip24_update() */ + + +static uint64_t sip24_final(struct siphash *H) { + char left = H->p - H->buf; + uint64_t b = (H->c + left) << 56; + + switch (left) { + case 7: b |= (uint64_t)H->buf[6] << 48; + case 6: b |= (uint64_t)H->buf[5] << 40; + case 5: b |= (uint64_t)H->buf[4] << 32; + case 4: b |= (uint64_t)H->buf[3] << 24; + case 3: b |= (uint64_t)H->buf[2] << 16; + case 2: b |= (uint64_t)H->buf[1] << 8; + case 1: b |= (uint64_t)H->buf[0] << 0; + case 0: break; + } + + H->v3 ^= b; + sip_round(H, 2); + H->v0 ^= b; + H->v2 ^= 0xff; + sip_round(H, 4); + + return H->v0 ^ H->v1 ^ H->v2 ^ H->v3; +} /* sip24_final() */ + + +static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { + struct siphash state = SIPHASH_INITIALIZER; + return sip24_final(sip24_update(sip24_init(&state, key), src, len)); +} /* siphash24() */ + + +/* + * SipHash-2-4 output with + * k = 00 01 02 ... + * and + * in = (empty string) + * in = 00 (1 byte) + * in = 00 01 (2 bytes) + * in = 00 01 02 (3 bytes) + * ... + * in = 00 01 02 ... 3e (63 bytes) + */ +static int sip24_valid(void) { + static const unsigned char vectors[64][8] = { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + unsigned char in[64]; + struct sipkey k; + size_t i; + + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + + for (i = 0; i < sizeof in; ++i) { + in[i] = i; + + if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) + return 0; + } + + return 1; +} /* sip24_valid() */ + + +#if SIPHASH_MAIN + +#include + +int main(void) { + int ok = sip24_valid(); + + if (ok) + puts("OK"); + else + puts("FAIL"); + + return !ok; +} /* main() */ + +#endif /* SIPHASH_MAIN */ + + +#endif /* SIPHASH_H */ diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index c1b791d62d0d0c..9bf014d7fbabd6 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -17,6 +17,12 @@ #include #include + +#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ +# include +#else /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #define XML_NS 1 #define XML_DTD 1 #define XML_CONTEXT_BYTES 1024 @@ -27,4 +33,8 @@ /* Windows has memmove() available. */ #define HAVE_MEMMOVE + +#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b308e67e2d390d..76f078e2505f90 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,13 +1,19 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. + + 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ +#define _GNU_SOURCE /* syscall prototype */ + #include #include /* memset(), memcpy() */ #include #include /* UINT_MAX */ +#include /* fprintf */ +#include /* getenv */ -#ifdef WIN32 +#ifdef _WIN32 #define getpid GetCurrentProcessId #else #include /* gettimeofday() */ @@ -17,20 +23,15 @@ #define XML_BUILDING_EXPAT 1 -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #elif defined(HAVE_EXPAT_CONFIG_H) #include -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "ascii.h" #include "expat.h" +#include "siphash.h" #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX @@ -109,17 +110,11 @@ typedef struct { const XML_Memory_Handling_Suite *mem; } HASH_TABLE; -/* Basic character hash algorithm, taken from Python's string hash: - h = h * 1000003 ^ character, the constant being a prime number. +static size_t +keylen(KEY s); -*/ -#ifdef XML_UNICODE -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned short)(c)) -#else -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned char)(c)) -#endif +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key); /* For probing (after a collision) we need a step size relative prime to the hash table size, which is a power of 2. We use double-hashing, @@ -355,6 +350,8 @@ doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ +static void +freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, BINDING **bindingsPtr); @@ -697,10 +694,84 @@ static const XML_Char implicitContext[] = { ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0' }; + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# include + +# if defined(HAVE_GETRANDOM) +# include /* getrandom */ +# else +# include /* syscall */ +# include /* SYS_getrandom */ +# endif + +/* Obtain entropy on Linux 3.17+ */ +static int +writeRandomBytes_getrandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + const unsigned int getrandomFlags = 0; + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const int bytesWrittenMore = +#if defined(HAVE_GETRANDOM) + getrandom(currentTarget, bytesToWrite, getrandomFlags); +#else + syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); +#endif + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR || errno == EAGAIN)); + + return success; +} + +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + + +#ifdef _WIN32 + +typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); + +/* Obtain entropy on Windows XP / Windows Server 2003 and later. + * Hint on RtlGenRandom and the following article from libsodioum. + * + * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI + * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/ + */ +static int +writeRandomBytes_RtlGenRandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + + if (advapi32) { + const RTLGENRANDOM_FUNC RtlGenRandom + = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036"); + if (RtlGenRandom) { + if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) { + success = 1; + } + } + FreeLibrary(advapi32); + } + + return success; +} + +#endif /* _WIN32 */ + + static unsigned long gather_time_entropy(void) { -#ifdef WIN32 +#ifdef _WIN32 FILETIME ft; GetSystemTimeAsFileTime(&ft); /* never fails */ return ft.dwHighDateTime ^ ft.dwLowDateTime; @@ -716,20 +787,62 @@ gather_time_entropy(void) #endif } +#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) +# include +#endif + +static unsigned long +ENTROPY_DEBUG(const char * label, unsigned long entropy) { + const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); + if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { + fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", + label, + (int)sizeof(entropy) * 2, entropy, + (unsigned long)sizeof(entropy)); + } + return entropy; +} + static unsigned long generate_hash_secret_salt(XML_Parser parser) { - /* Process ID is 0 bits entropy if attacker has local access - * XML_Parser address is few bits of entropy if attacker has local access */ - const unsigned long entropy = - gather_time_entropy() ^ getpid() ^ (unsigned long)parser; + unsigned long entropy; + (void)parser; +#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) + (void)gather_time_entropy; + arc4random_buf(&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random_buf", entropy); +#else + /* Try high quality providers first .. */ +#ifdef _WIN32 + if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("RtlGenRandom", entropy); + } +#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getrandom", entropy); + } +#endif + /* .. and self-made low quality for backup: */ + + /* Process ID is 0 bits entropy if attacker has local access */ + entropy = gather_time_entropy() ^ getpid(); /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return entropy * 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { - return entropy * (unsigned long)2305843009213693951; + return ENTROPY_DEBUG("fallback(8)", + entropy * (unsigned long)2305843009213693951); } +#endif +} + +static unsigned long +get_hash_secret_salt(XML_Parser parser) { + if (parser->m_parentParser != NULL) + return get_hash_secret_salt(parser->m_parentParser); + return parser->m_hash_secret_salt; } static XML_Bool /* only valid for root parser */ @@ -960,6 +1073,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; OPEN_INTERNAL_ENTITY *openEntityList; + + if (parser == NULL) + return XML_FALSE; + if (parentParser) return XML_FALSE; /* move tagStack to freeTagList */ @@ -994,6 +1111,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { + if (parser == NULL) + return XML_STATUS_ERROR; /* Block after XML_Parse()/XML_ParseBuffer() has been called. XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. @@ -1017,52 +1136,88 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, { XML_Parser parser = oldParser; DTD *newDtd = NULL; - DTD *oldDtd = _dtd; - XML_StartElementHandler oldStartElementHandler = startElementHandler; - XML_EndElementHandler oldEndElementHandler = endElementHandler; - XML_CharacterDataHandler oldCharacterDataHandler = characterDataHandler; - XML_ProcessingInstructionHandler oldProcessingInstructionHandler - = processingInstructionHandler; - XML_CommentHandler oldCommentHandler = commentHandler; - XML_StartCdataSectionHandler oldStartCdataSectionHandler - = startCdataSectionHandler; - XML_EndCdataSectionHandler oldEndCdataSectionHandler - = endCdataSectionHandler; - XML_DefaultHandler oldDefaultHandler = defaultHandler; - XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler - = unparsedEntityDeclHandler; - XML_NotationDeclHandler oldNotationDeclHandler = notationDeclHandler; - XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler - = startNamespaceDeclHandler; - XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler - = endNamespaceDeclHandler; - XML_NotStandaloneHandler oldNotStandaloneHandler = notStandaloneHandler; - XML_ExternalEntityRefHandler oldExternalEntityRefHandler - = externalEntityRefHandler; - XML_SkippedEntityHandler oldSkippedEntityHandler = skippedEntityHandler; - XML_UnknownEncodingHandler oldUnknownEncodingHandler - = unknownEncodingHandler; - XML_ElementDeclHandler oldElementDeclHandler = elementDeclHandler; - XML_AttlistDeclHandler oldAttlistDeclHandler = attlistDeclHandler; - XML_EntityDeclHandler oldEntityDeclHandler = entityDeclHandler; - XML_XmlDeclHandler oldXmlDeclHandler = xmlDeclHandler; - ELEMENT_TYPE * oldDeclElementType = declElementType; - - void *oldUserData = userData; - void *oldHandlerArg = handlerArg; - XML_Bool oldDefaultExpandInternalEntities = defaultExpandInternalEntities; - XML_Parser oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; + DTD *oldDtd; + XML_StartElementHandler oldStartElementHandler; + XML_EndElementHandler oldEndElementHandler; + XML_CharacterDataHandler oldCharacterDataHandler; + XML_ProcessingInstructionHandler oldProcessingInstructionHandler; + XML_CommentHandler oldCommentHandler; + XML_StartCdataSectionHandler oldStartCdataSectionHandler; + XML_EndCdataSectionHandler oldEndCdataSectionHandler; + XML_DefaultHandler oldDefaultHandler; + XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler; + XML_NotationDeclHandler oldNotationDeclHandler; + XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler; + XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler; + XML_NotStandaloneHandler oldNotStandaloneHandler; + XML_ExternalEntityRefHandler oldExternalEntityRefHandler; + XML_SkippedEntityHandler oldSkippedEntityHandler; + XML_UnknownEncodingHandler oldUnknownEncodingHandler; + XML_ElementDeclHandler oldElementDeclHandler; + XML_AttlistDeclHandler oldAttlistDeclHandler; + XML_EntityDeclHandler oldEntityDeclHandler; + XML_XmlDeclHandler oldXmlDeclHandler; + ELEMENT_TYPE * oldDeclElementType; + + void *oldUserData; + void *oldHandlerArg; + XML_Bool oldDefaultExpandInternalEntities; + XML_Parser oldExternalEntityRefHandlerArg; #ifdef XML_DTD - enum XML_ParamEntityParsing oldParamEntityParsing = paramEntityParsing; - int oldInEntityValue = prologState.inEntityValue; + enum XML_ParamEntityParsing oldParamEntityParsing; + int oldInEntityValue; #endif - XML_Bool oldns_triplets = ns_triplets; + XML_Bool oldns_triplets; /* Note that the new parser shares the same hash secret as the old parser, so that dtdCopy and copyEntityTable can lookup values from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt = hash_secret_salt; + unsigned long oldhash_secret_salt; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) + return NULL; + + /* Stash the original parser contents on the stack */ + oldDtd = _dtd; + oldStartElementHandler = startElementHandler; + oldEndElementHandler = endElementHandler; + oldCharacterDataHandler = characterDataHandler; + oldProcessingInstructionHandler = processingInstructionHandler; + oldCommentHandler = commentHandler; + oldStartCdataSectionHandler = startCdataSectionHandler; + oldEndCdataSectionHandler = endCdataSectionHandler; + oldDefaultHandler = defaultHandler; + oldUnparsedEntityDeclHandler = unparsedEntityDeclHandler; + oldNotationDeclHandler = notationDeclHandler; + oldStartNamespaceDeclHandler = startNamespaceDeclHandler; + oldEndNamespaceDeclHandler = endNamespaceDeclHandler; + oldNotStandaloneHandler = notStandaloneHandler; + oldExternalEntityRefHandler = externalEntityRefHandler; + oldSkippedEntityHandler = skippedEntityHandler; + oldUnknownEncodingHandler = unknownEncodingHandler; + oldElementDeclHandler = elementDeclHandler; + oldAttlistDeclHandler = attlistDeclHandler; + oldEntityDeclHandler = entityDeclHandler; + oldXmlDeclHandler = xmlDeclHandler; + oldDeclElementType = declElementType; + + oldUserData = userData; + oldHandlerArg = handlerArg; + oldDefaultExpandInternalEntities = defaultExpandInternalEntities; + oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; +#ifdef XML_DTD + oldParamEntityParsing = paramEntityParsing; + oldInEntityValue = prologState.inEntityValue; +#endif + oldns_triplets = ns_triplets; + /* Note that the new parser shares the same hash secret as the old + parser, so that dtdCopy and copyEntityTable can lookup values + from hash tables associated with either parser without us having + to worry which hash secrets each table has. + */ + oldhash_secret_salt = hash_secret_salt; #ifdef XML_DTD if (!context) @@ -1228,12 +1383,15 @@ XML_ParserFree(XML_Parser parser) void XMLCALL XML_UseParserAsHandlerArg(XML_Parser parser) { - handlerArg = parser; + if (parser != NULL) + handlerArg = parser; } enum XML_Error XMLCALL XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) @@ -1248,6 +1406,8 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) void XMLCALL XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { + if (parser == NULL) + return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return; @@ -1257,6 +1417,8 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) void XMLCALL XML_SetUserData(XML_Parser parser, void *p) { + if (parser == NULL) + return; if (handlerArg == userData) handlerArg = userData = p; else @@ -1266,6 +1428,8 @@ XML_SetUserData(XML_Parser parser, void *p) enum XML_Status XMLCALL XML_SetBase(XML_Parser parser, const XML_Char *p) { + if (parser == NULL) + return XML_STATUS_ERROR; if (p) { p = poolCopyString(&_dtd->pool, p); if (!p) @@ -1280,18 +1444,24 @@ XML_SetBase(XML_Parser parser, const XML_Char *p) const XML_Char * XMLCALL XML_GetBase(XML_Parser parser) { + if (parser == NULL) + return NULL; return curBase; } int XMLCALL XML_GetSpecifiedAttributeCount(XML_Parser parser) { + if (parser == NULL) + return -1; return nSpecifiedAtts; } int XMLCALL XML_GetIdAttributeIndex(XML_Parser parser) { + if (parser == NULL) + return -1; return idAttIndex; } @@ -1299,6 +1469,8 @@ XML_GetIdAttributeIndex(XML_Parser parser) const XML_AttrInfo * XMLCALL XML_GetAttributeInfo(XML_Parser parser) { + if (parser == NULL) + return NULL; return attInfo; } #endif @@ -1308,6 +1480,8 @@ XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { + if (parser == NULL) + return; startElementHandler = start; endElementHandler = end; } @@ -1315,34 +1489,39 @@ XML_SetElementHandler(XML_Parser parser, void XMLCALL XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) { - startElementHandler = start; + if (parser != NULL) + startElementHandler = start; } void XMLCALL XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) { - endElementHandler = end; + if (parser != NULL) + endElementHandler = end; } void XMLCALL XML_SetCharacterDataHandler(XML_Parser parser, XML_CharacterDataHandler handler) { - characterDataHandler = handler; + if (parser != NULL) + characterDataHandler = handler; } void XMLCALL XML_SetProcessingInstructionHandler(XML_Parser parser, XML_ProcessingInstructionHandler handler) { - processingInstructionHandler = handler; + if (parser != NULL) + processingInstructionHandler = handler; } void XMLCALL XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) { - commentHandler = handler; + if (parser != NULL) + commentHandler = handler; } void XMLCALL @@ -1350,6 +1529,8 @@ XML_SetCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start, XML_EndCdataSectionHandler end) { + if (parser == NULL) + return; startCdataSectionHandler = start; endCdataSectionHandler = end; } @@ -1357,19 +1538,23 @@ XML_SetCdataSectionHandler(XML_Parser parser, void XMLCALL XML_SetStartCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start) { - startCdataSectionHandler = start; + if (parser != NULL) + startCdataSectionHandler = start; } void XMLCALL XML_SetEndCdataSectionHandler(XML_Parser parser, XML_EndCdataSectionHandler end) { - endCdataSectionHandler = end; + if (parser != NULL) + endCdataSectionHandler = end; } void XMLCALL XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_FALSE; } @@ -1378,6 +1563,8 @@ void XMLCALL XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_TRUE; } @@ -1387,6 +1574,8 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start, XML_EndDoctypeDeclHandler end) { + if (parser == NULL) + return; startDoctypeDeclHandler = start; endDoctypeDeclHandler = end; } @@ -1394,27 +1583,31 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start) { - startDoctypeDeclHandler = start; + if (parser != NULL) + startDoctypeDeclHandler = start; } void XMLCALL XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) { - endDoctypeDeclHandler = end; + if (parser != NULL) + endDoctypeDeclHandler = end; } void XMLCALL XML_SetUnparsedEntityDeclHandler(XML_Parser parser, XML_UnparsedEntityDeclHandler handler) { - unparsedEntityDeclHandler = handler; + if (parser != NULL) + unparsedEntityDeclHandler = handler; } void XMLCALL XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) { - notationDeclHandler = handler; + if (parser != NULL) + notationDeclHandler = handler; } void XMLCALL @@ -1422,6 +1615,8 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start, XML_EndNamespaceDeclHandler end) { + if (parser == NULL) + return; startNamespaceDeclHandler = start; endNamespaceDeclHandler = end; } @@ -1429,32 +1624,38 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start) { - startNamespaceDeclHandler = start; + if (parser != NULL) + startNamespaceDeclHandler = start; } void XMLCALL XML_SetEndNamespaceDeclHandler(XML_Parser parser, XML_EndNamespaceDeclHandler end) { - endNamespaceDeclHandler = end; + if (parser != NULL) + endNamespaceDeclHandler = end; } void XMLCALL XML_SetNotStandaloneHandler(XML_Parser parser, XML_NotStandaloneHandler handler) { - notStandaloneHandler = handler; + if (parser != NULL) + notStandaloneHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandler(XML_Parser parser, XML_ExternalEntityRefHandler handler) { - externalEntityRefHandler = handler; + if (parser != NULL) + externalEntityRefHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) { + if (parser == NULL) + return; if (arg) externalEntityRefHandlerArg = (XML_Parser)arg; else @@ -1465,7 +1666,8 @@ void XMLCALL XML_SetSkippedEntityHandler(XML_Parser parser, XML_SkippedEntityHandler handler) { - skippedEntityHandler = handler; + if (parser != NULL) + skippedEntityHandler = handler; } void XMLCALL @@ -1473,6 +1675,8 @@ XML_SetUnknownEncodingHandler(XML_Parser parser, XML_UnknownEncodingHandler handler, void *data) { + if (parser == NULL) + return; unknownEncodingHandler = handler; unknownEncodingHandlerData = data; } @@ -1481,33 +1685,39 @@ void XMLCALL XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) { - elementDeclHandler = eldecl; + if (parser != NULL) + elementDeclHandler = eldecl; } void XMLCALL XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) { - attlistDeclHandler = attdecl; + if (parser != NULL) + attlistDeclHandler = attdecl; } void XMLCALL XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) { - entityDeclHandler = handler; + if (parser != NULL) + entityDeclHandler = handler; } void XMLCALL XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) { - xmlDeclHandler = handler; + if (parser != NULL) + xmlDeclHandler = handler; } int XMLCALL XML_SetParamEntityParsing(XML_Parser parser, enum XML_ParamEntityParsing peParsing) { + if (parser == NULL) + return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1523,6 +1733,10 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + if (parser == NULL) + return 0; + if (parser->m_parentParser) + return XML_SetHashSalt(parser->m_parentParser, hash_salt); /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1533,6 +1747,10 @@ XML_SetHashSalt(XML_Parser parser, enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { + errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1585,6 +1803,13 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) const char *end; int nLeftOver; enum XML_Status result; + /* Detect overflow (a+b > MAX <==> b > MAX-a) */ + if (len > ((XML_Size)-1) / 2 - parseEndByteIndex) { + errorCode = XML_ERROR_NO_MEMORY; + eventPtr = eventEndPtr = NULL; + processor = errorProcessor; + return XML_STATUS_ERROR; + } parseEndByteIndex += len; positionPtr = s; ps_finalBuffer = (XML_Bool)isFinal; @@ -1617,11 +1842,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) nLeftOver = s + len - end; if (nLeftOver) { if (buffer == NULL || nLeftOver > bufferLim - buffer) { - /* FIXME avoid integer overflow */ - char *temp; - temp = (buffer == NULL - ? (char *)MALLOC(len * 2) - : (char *)REALLOC(buffer, len * 2)); + /* avoid _signed_ integer overflow */ + char *temp = NULL; + const int bytesToAllocate = (int)((unsigned)len * 2U); + if (bytesToAllocate > 0) { + temp = (buffer == NULL + ? (char *)MALLOC(bytesToAllocate) + : (char *)REALLOC(buffer, bytesToAllocate)); + } if (temp == NULL) { errorCode = XML_ERROR_NO_MEMORY; eventPtr = eventEndPtr = NULL; @@ -1629,7 +1857,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) return XML_STATUS_ERROR; } buffer = temp; - bufferLim = buffer + len * 2; + bufferLim = buffer + bytesToAllocate; } memcpy(buffer, end, nLeftOver); } @@ -1659,6 +1887,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) const char *start; enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1712,6 +1942,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) void * XMLCALL XML_GetBuffer(XML_Parser parser, int len) { + if (parser == NULL) + return NULL; if (len < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -1808,6 +2040,8 @@ XML_GetBuffer(XML_Parser parser, int len) enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: if (resumable) { @@ -1840,6 +2074,8 @@ XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; if (ps_parsing != XML_SUSPENDED) { errorCode = XML_ERROR_NOT_SUSPENDED; return XML_STATUS_ERROR; @@ -1876,6 +2112,8 @@ XML_ResumeParser(XML_Parser parser) void XMLCALL XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) { + if (parser == NULL) + return; assert(status != NULL); *status = parser->m_parsingStatus; } @@ -1883,12 +2121,16 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) enum XML_Error XMLCALL XML_GetErrorCode(XML_Parser parser) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; return errorCode; } XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { + if (parser == NULL) + return -1; if (eventPtr) return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; @@ -1897,6 +2139,8 @@ XML_GetCurrentByteIndex(XML_Parser parser) int XMLCALL XML_GetCurrentByteCount(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventEndPtr && eventPtr) return (int)(eventEndPtr - eventPtr); return 0; @@ -1906,11 +2150,19 @@ const char * XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { #ifdef XML_CONTEXT_BYTES + if (parser == NULL) + return NULL; if (eventPtr && buffer) { - *offset = (int)(eventPtr - buffer); - *size = (int)(bufferEnd - buffer); + if (offset != NULL) + *offset = (int)(eventPtr - buffer); + if (size != NULL) + *size = (int)(bufferEnd - buffer); return buffer; } +#else + (void)parser; + (void)offset; + (void)size; #endif /* defined XML_CONTEXT_BYTES */ return (char *) 0; } @@ -1918,6 +2170,8 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) XML_Size XMLCALL XML_GetCurrentLineNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1928,6 +2182,8 @@ XML_GetCurrentLineNumber(XML_Parser parser) XML_Size XMLCALL XML_GetCurrentColumnNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1938,30 +2194,38 @@ XML_GetCurrentColumnNumber(XML_Parser parser) void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - FREE(model); + if (parser != NULL) + FREE(model); } void * XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { + if (parser == NULL) + return NULL; return MALLOC(size); } void * XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { + if (parser == NULL) + return NULL; return REALLOC(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - FREE(ptr); + if (parser != NULL) + FREE(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { + if (parser == NULL) + return; if (defaultHandler) { if (openInternalEntities) reportDefault(parser, @@ -2468,7 +2732,7 @@ doContent(XML_Parser parser, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { + if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2511,8 +2775,10 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; poolFinish(&tempPool); result = storeAtts(parser, enc, s, &name, &bindings); - if (result) + if (result != XML_ERROR_NONE) { + freeBindings(parser, bindings); return result; + } poolFinish(&tempPool); if (startElementHandler) { startElementHandler(handlerArg, name.str, (const XML_Char **)atts); @@ -2527,15 +2793,7 @@ doContent(XML_Parser parser, if (noElmHandlers && defaultHandler) reportDefault(parser, enc, s, next); poolClear(&tempPool); - while (bindings) { - BINDING *b = bindings; - if (endNamespaceDeclHandler) - endNamespaceDeclHandler(handlerArg, b->prefix->name); - bindings = bindings->nextTagBinding; - b->nextTagBinding = freeBindingList; - freeBindingList = b; - b->prefix->binding = b->prevPrefixBinding; - } + freeBindings(parser, bindings); } if (tagLevel == 0) return epilogProcessor(parser, next, end, nextPtr); @@ -2733,6 +2991,29 @@ doContent(XML_Parser parser, /* not reached */ } +/* This function does not call free() on the allocated memory, merely + * moving it to the parser's freeBindingList where it can be freed or + * reused as appropriate. + */ +static void +freeBindings(XML_Parser parser, BINDING *bindings) +{ + while (bindings) { + BINDING *b = bindings; + + /* startNamespaceDeclHandler will have been called for this + * binding in addBindings(), so call the end handler now. + */ + if (endNamespaceDeclHandler) + endNamespaceDeclHandler(handlerArg, b->prefix->name); + + bindings = bindings->nextTagBinding; + b->nextTagBinding = freeBindingList; + freeBindingList = b; + b->prefix->binding = b->prevPrefixBinding; + } +} + /* Precondition: all arguments must be non-NULL; Purpose: - normalize attributes @@ -2957,7 +3238,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (s[-1] == 2) { /* prefixed */ ATTRIBUTE_ID *id; const BINDING *b; - unsigned long uriHash = hash_secret_salt; + unsigned long uriHash; + struct siphash sip_state; + struct sipkey sip_key; + + copy_salt_to_sipkey(parser, &sip_key); + sip24_init(&sip_state, &sip_key); + ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (!id || !id->prefix) @@ -2966,22 +3253,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (!b) return XML_ERROR_UNBOUND_PREFIX; - /* as we expand the name we also calculate its hash value */ for (j = 0; j < b->uriLen; j++) { const XML_Char c = b->uri[j]; if (!poolAppendChar(&tempPool, c)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } + + sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); + while (*s++ != XML_T(ASCII_COLON)) ; + + sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); + do { /* copies null terminator */ - const XML_Char c = *s; if (!poolAppendChar(&tempPool, *s)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } while (*s++); + uriHash = (unsigned long)sip24_final(&sip_state); + { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). */ @@ -3695,6 +3986,14 @@ entityValueInitProcessor(XML_Parser parser, *nextPtr = next; return XML_ERROR_NONE; } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "internalEventEndPtr = NULL; textStart = (char *)entity->textPtr; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4916,6 +5217,8 @@ internalEntityProcessor(XML_Parser parser, entity = openEntity->entity; textStart = ((char *)entity->textPtr) + entity->processed; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -5876,7 +6179,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H newE->defaultAtts = (DEFAULT_ATTRIBUTE *) ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (!newE->defaultAtts) { - ms->free_fcn(newE); return 0; } } @@ -6011,13 +6313,32 @@ keyeq(KEY s1, KEY s2) return XML_FALSE; } +static size_t +keylen(KEY s) +{ + size_t len = 0; + for (; *s; s++, len++); + return len; +} + +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key) +{ + key->k[0] = 0; + key->k[1] = get_hash_secret_salt(parser); +} + static unsigned long FASTCALL hash(XML_Parser parser, KEY s) { - unsigned long h = hash_secret_salt; - while (*s) - h = CHAR_HASH(h, *s++); - return h; + struct siphash state; + struct sipkey key; + (void)sip_tobin; + (void)sip24_valid; + copy_salt_to_sipkey(parser, &key); + sip24_init(&state, &key); + sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + return (unsigned long)sip24_final(&state); } static NAMED * @@ -6260,6 +6581,35 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, return pool->start; } +static size_t +poolBytesToAllocateFor(int blockSize) +{ + /* Unprotected math would be: + ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char); + ** + ** Detect overflow, avoiding _signed_ overflow undefined behavior + ** For a + b * c we check b * c in isolation first, so that addition of a + ** on top has no chance of making us accept a small non-negative number + */ + const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */ + + if (blockSize <= 0) + return 0; + + if (blockSize > (int)(INT_MAX / stretch)) + return 0; + + { + const int stretchedBlockSize = blockSize * (int)stretch; + const int bytesToAllocate = (int)( + offsetof(BLOCK, s) + (unsigned)stretchedBlockSize); + if (bytesToAllocate < 0) + return 0; + + return (size_t)bytesToAllocate; + } +} + static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { @@ -6289,14 +6639,17 @@ poolGrow(STRING_POOL *pool) if (pool->blocks && pool->start == pool->blocks->s) { BLOCK *temp; int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + temp = (BLOCK *) - pool->mem->realloc_fcn(pool->blocks, - (offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char))); + pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -6308,16 +6661,26 @@ poolGrow(STRING_POOL *pool) else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; - else + else { + /* Detect overflow, avoiding _signed_ overflow undefined behavior */ + if ((int)((unsigned)blockSize * 2U) < 0) { + return XML_FALSE; + } blockSize *= 2; - tem = (BLOCK *)pool->mem->malloc_fcn(offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char)); + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate); if (!tem) return XML_FALSE; tem->size = blockSize; diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index fcd0dc6948f442..a7c56302796957 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index a29d9e2f8ac386..cdf0720dd89689 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -369,24 +363,24 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - enum XML_Convert_Result res = XML_CONVERT_COMPLETED; char *to; const char *from; - if (fromLim - *fromP > toLim - *toP) { - /* Avoid copying partial characters. */ - res = XML_CONVERT_OUTPUT_EXHAUSTED; - fromLim = *fromP + (toLim - *toP); - align_limit_to_full_utf8_characters(*fromP, &fromLim); - } + const char *fromLimInitial = fromLim; + + /* Avoid copying partial characters. */ + align_limit_to_full_utf8_characters(*fromP, &fromLim); + for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++) *to = *from; *fromP = from; *toP = to; - if ((to == toLim) && (from < fromLim)) + if (fromLim < fromLimInitial) + return XML_CONVERT_INPUT_INCOMPLETE; + else if ((to == toLim) && (from < fromLim)) return XML_CONVERT_OUTPUT_EXHAUSTED; else - return res; + return XML_CONVERT_COMPLETED; } static enum XML_Convert_Result PTRCALL @@ -402,7 +396,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; @@ -410,7 +404,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD3: if (fromLim - from < 3) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); @@ -441,6 +435,8 @@ utf8_toUtf16(const ENCODING *enc, break; } } + if (from < fromLim) + res = XML_CONVERT_OUTPUT_EXHAUSTED; after: *fromP = from; *toP = to; From 5777e79ecbd1f2adf36456e09f210608ee221691 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 15:21:33 +0200 Subject: [PATCH 0535/1345] [3.6] bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) (#2349) * bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) bpo-30726, bpo-29591: libexpat 2.2.1 of Modules/expat/ now uses a winconfig.h configuration file which already defines: * XML_NS * XML_DTD * BYTEORDER=1234 * XML_CONTEXT_BYTES=1024 * HAVE_MEMMOVE Remove these defines from PCbuild/_elementtree.vcxproj to prevent compiler warnings. Co-Authored-By: Jeremy Kloth (cherry picked from commit c8fb58bd7917151e63398587a7fc2126db7c26de) * bpo-30726: Fix elementtree warnings on Windows due to expat upgrade (#2319) * bpo-30726: Fix elementtree warnings on Windows Caused by usage of `getenv` which should be safe. And a few integer truncations which should also be ok. * bpo-30726: Don't ignore libexpat warnings which haypo intends to fix upstream (cherry picked from commit 87c65550730a8f85ce339ba197bce4fb7e836619) (cherry picked from commit d32a05953130fb5cc2d3c0c9fcb20ad0859353f3) --- PCbuild/_elementtree.vcxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 725b5a967bebc9..639ba479db5786 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 @@ -103,4 +103,4 @@ - \ No newline at end of file + From b6012f982fabed6029d7e2db2a509a8b28b4f6e1 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Tue, 4 Jul 2017 21:38:47 +0300 Subject: [PATCH 0536/1345] [3.6] bpo-30726: Also fix pyexpat.vcxproj (GH-2375) (#2570) (cherry picked from commit 7526cadd64566725ffc56071a7208828a46ddbd8) (cherry picked from commit d02c8416fe1b29b3322004b73133bf6c8a2e353a) --- PCbuild/pyexpat.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 99c7286b2bdfde..a2763f02063cd5 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -59,7 +59,7 @@ $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) - PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions) From c794b643c9172d69afa46f85982befd82511d9df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 28 Jun 2017 02:28:51 +0200 Subject: [PATCH 0537/1345] bpo-30704, bpo-30604: Fix memleak in code_dealloc() (#2455) (#2456) Free also co_extra->ce_extras, not only co_extra. (cherry picked from commit 23e7944eba1968bb8432fdc4cc96d4fdd2c1a230) (cherry picked from commit 26daad4ee14693381d84a5235709d22aed1c22ed) --- Objects/codeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d38f185ba3b8e4..6de697ae3fd31c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -422,7 +422,8 @@ code_dealloc(PyCodeObject *co) } } - PyMem_FREE(co->co_extra); + PyMem_Free(co_extra->ce_extras); + PyMem_Free(co_extra); } Py_XDECREF(co->co_code); From a9b16cff35811f88cdfeb4f50758140dfff36ebc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 Jun 2017 20:17:38 +0300 Subject: [PATCH 0538/1345] [3.6] bpo-30730: Prevent environment variables injection in subprocess on Windows. (GH-2325) (#2360) Prevent passing other invalid environment variables and command arguments.. (cherry picked from commit d174d24a5d37d1516b885dc7c82f71ecd5930700) (cherry picked from commit e7135751b8e48af80665e40ac8fa6d0073e5affe) --- Lib/subprocess.py | 8 ++++++-- Lib/test/test_subprocess.py | 40 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_winapi.c | 26 +++++++++++++++++++----- Objects/abstract.c | 4 ++-- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e626a8afdbb7f0..b790cbd65ba208 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1239,8 +1239,12 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # and pass it to fork_exec() if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d8d6d82713cded..9a9e3ca5791256 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -644,6 +644,46 @@ def test_empty_env(self): # environment b"['__CF_USER_TEXT_ENCODING']")) + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' diff --git a/Misc/NEWS b/Misc/NEWS index 1f6161a1f10177..136c84fb9fe767 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- [Security] bpo-30730: Prevent environment variables injection in subprocess on + Windows. Prevent passing other environment variables and command arguments. + - [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security vulnerabilities including: CVE-2017-9233 (External entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 248f4582c60fb4..71426bc12dc655 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -744,6 +744,20 @@ getenvironment(PyObject* environment) "environment can only contain strings"); goto error; } + if (PyUnicode_FindChar(key, '\0', 0, PyUnicode_GET_LENGTH(key), 1) != -1 || + PyUnicode_FindChar(value, '\0', 0, PyUnicode_GET_LENGTH(value), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto error; + } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key) == 0 || + PyUnicode_FindChar(key, '=', 1, PyUnicode_GET_LENGTH(key), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } if (totalsize > PY_SSIZE_T_MAX - PyUnicode_GET_LENGTH(key) - 1) { PyErr_SetString(PyExc_OverflowError, "environment too long"); goto error; @@ -830,7 +844,8 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - wchar_t *wenvironment; + const wchar_t *wenvironment; + Py_ssize_t wenvironment_size; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); @@ -846,12 +861,13 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, if (env_mapping != Py_None) { environment = getenvironment(env_mapping); - if (! environment) + if (environment == NULL) { return NULL; + } + /* contains embedded null characters */ wenvironment = PyUnicode_AsUnicode(environment); - if (wenvironment == NULL) - { - Py_XDECREF(environment); + if (wenvironment == NULL) { + Py_DECREF(environment); return NULL; } } diff --git a/Objects/abstract.c b/Objects/abstract.c index 3585b34ca4d8bd..38de774de3a4db 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -3193,8 +3193,8 @@ _PySequence_BytesToCharpArray(PyObject* self) array[i] = NULL; goto fail; } - data = PyBytes_AsString(item); - if (data == NULL) { + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(item, &data, NULL) < 0) { /* NULL terminate before freeing. */ array[i] = NULL; goto fail; From d1d65015fca44b8d1f0b1df78694310270f03a6d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 24 Jun 2017 16:28:26 +0300 Subject: [PATCH 0539/1345] [3.6] bpo-30745: Fix compiler warnings introduced in bpo-30730. (GH-2376) (#2378) (cherry picked from commit 0ee32c1) (cherry picked from commit 0e1f9e8d3ea82262cbb9a403b70a884da5e6a6ac) --- Modules/_winapi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 71426bc12dc655..1606f0da772c95 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -844,8 +844,7 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - const wchar_t *wenvironment; - Py_ssize_t wenvironment_size; + wchar_t *wenvironment; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); From 05b72ede95521b2d897cb4c7b034139b5437c592 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 01:31:43 -0400 Subject: [PATCH 0540/1345] bpo-30797, bpo-30694: Avoid _GNU_SOURCE redefined warning in xmlparse.c (#2615) --- Modules/expat/xmlparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 76f078e2505f90..daec151e232f1c 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -4,7 +4,7 @@ 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ -#define _GNU_SOURCE /* syscall prototype */ +#define _GNU_SOURCE 1 /* syscall prototype */ #include #include /* memset(), memcpy() */ From bdabd7666032ce356d550da21c35e4bee5b3448c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 01:31:43 -0400 Subject: [PATCH 0541/1345] bpo-30797, bpo-30694: Avoid _GNU_SOURCE redefined warning in xmlparse.c (#2615) (cherry picked from commit 05b72ede95521b2d897cb4c7b034139b5437c592) --- Modules/expat/xmlparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 76f078e2505f90..daec151e232f1c 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -4,7 +4,7 @@ 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ -#define _GNU_SOURCE /* syscall prototype */ +#define _GNU_SOURCE 1 /* syscall prototype */ #include #include /* memset(), memcpy() */ From 891331134516784ff1c5ad8d1720fdc2d056a15f Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 03:16:00 -0400 Subject: [PATCH 0542/1345] Version bump to 3.6.2rc2 --- Include/patchlevel.h | 4 ++-- Misc/NEWS | 6 +++--- README.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9c22385f3b36c4..91b8ece15ba4a6 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 2 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.6.2rc1" +#define PY_VERSION "3.6.2rc2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 136c84fb9fe767..a76d2a1695b7e7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,11 +1,11 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ -What's New in Python 3.6.3 release candidate 1? +What's New in Python 3.6.2 release candidate 2? =============================================== -*Release date: XXXX-XX-XX* +*Release date: 2017-07-07* Core and Builtins ----------------- diff --git a/README.rst b/README.rst index 71eb257b529d4e..4b564759610206 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.6.2 release candidate 1 +This is Python version 3.6.2 release candidate 2 ================================================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 From edc034221f878b437748cef0f05c782d8190499d Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 7 Jul 2017 16:37:39 -0400 Subject: [PATCH 0543/1345] [3.6] bpo-30779: IDLE -- Factor ConfigChanges class from configdialog, put in config; test. (GH-2612) (#2625) * In config, put dump test code in a function; run it and unittest in 'if __name__ == '__main__'. * Add class config.ConfigChanges based on changes_class_v4.py on bpo issue. * Add class test_config.ChangesTest, partly based on configdialog_tests_v1.py on bpo issue. * Revise configdialog to use ConfigChanges, mostly as specified in tracker msg297804. * Revise test_configdialog to match configdialog changes. All tests pass in both files. * Remove configdialog functions unused or moved to ConfigChanges. Cheryl Sabella contributed parts of the patch. (cherry picked from commit 349abd9) --- Lib/idlelib/config.py | 140 ++++++++++++++--- Lib/idlelib/configdialog.py | 169 +++++++-------------- Lib/idlelib/idle_test/test_config.py | 86 ++++++++++- Lib/idlelib/idle_test/test_configdialog.py | 53 +++---- 4 files changed, 277 insertions(+), 171 deletions(-) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 6d683e2b80ac07..621e0bfc8b4073 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -44,7 +44,7 @@ def __init__(self, cfgFile, cfgDefaults=None): """ cfgFile - string, fully specified configuration file name """ - self.file = cfgFile + self.file = cfgFile # This is currently '' when testing. ConfigParser.__init__(self, defaults=cfgDefaults, strict=False) def Get(self, section, option, type=None, default=None, raw=False): @@ -73,7 +73,8 @@ def GetOptionList(self, section): def Load(self): "Load the configuration file from disk." - self.read(self.file) + if self.file: + self.read(self.file) class IdleUserConfParser(IdleConfParser): """ @@ -130,21 +131,22 @@ def RemoveFile(self): def Save(self): """Update user configuration file. - Remove empty sections. If resulting config isn't empty, write the file - to disk. If config is empty, remove the file from disk if it exists. + If self not empty after removing empty sections, write the file + to disk. Otherwise, remove the file from disk if it exists. """ - if not self.IsEmpty(): - fname = self.file - try: - cfgFile = open(fname, 'w') - except OSError: - os.unlink(fname) - cfgFile = open(fname, 'w') - with cfgFile: - self.write(cfgFile) - else: - self.RemoveFile() + fname = self.file + if fname: + if not self.IsEmpty(): + try: + cfgFile = open(fname, 'w') + except OSError: + os.unlink(fname) + cfgFile = open(fname, 'w') + with cfgFile: + self.write(cfgFile) + else: + self.RemoveFile() class IdleConf: """Hold config parsers for all idle config files in singleton instance. @@ -158,7 +160,7 @@ class IdleConf: (user home dir)/.idlerc/config-{config-type}.cfg """ def __init__(self): - self.config_types = ('main', 'extensions', 'highlight', 'keys') + self.config_types = ('main', 'highlight', 'keys', 'extensions') self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg @@ -766,7 +768,6 @@ def SaveUserCfgFiles(self): idleConf = IdleConf() - _warned = set() def _warn(msg, *key): key = (msg,) + key @@ -778,9 +779,100 @@ def _warn(msg, *key): _warned.add(key) +class ConfigChanges(dict): + """Manage a user's proposed configuration option changes. + + Names used across multiple methods: + page -- one of the 4 top-level dicts representing a + .idlerc/config-x.cfg file. + config_type -- name of a page. + section -- a section within a page/file. + option -- name of an option within a section. + value -- value for the option. + + Methods + add_option: Add option and value to changes. + save_option: Save option and value to config parser. + save_all: Save all the changes to the config parser and file. + delete_section: Delete section if it exists. + clear: Clear all changes by clearing each page. + """ + def __init__(self): + "Create a page for each configuration file" + self.pages = [] # List of unhashable dicts. + for config_type in idleConf.config_types: + self[config_type] = {} + self.pages.append(self[config_type]) + + def add_option(self, config_type, section, item, value): + "Add item/value pair for config_type and section." + page = self[config_type] + value = str(value) # Make sure we use a string. + if section not in page: + page[section] = {} + page[section][item] = value + + @staticmethod + def save_option(config_type, section, item, value): + """Return True if the configuration value was added or changed. + + Helper for save_all. + """ + if idleConf.defaultCfg[config_type].has_option(section, item): + if idleConf.defaultCfg[config_type].Get(section, item) == value: + # The setting equals a default setting, remove it from user cfg. + return idleConf.userCfg[config_type].RemoveOption(section, item) + # If we got here, set the option. + return idleConf.userCfg[config_type].SetOption(section, item, value) + + def save_all(self): + """Save configuration changes to the user config file. + + Then clear self in preparation for additional changes. + """ + idleConf.userCfg['main'].Save() + for config_type in self: + cfg_type_changed = False + page = self[config_type] + for section in page: + if section == 'HelpFiles': # Remove it for replacement. + idleConf.userCfg['main'].remove_section('HelpFiles') + cfg_type_changed = True + for item, value in page[section].items(): + if self.save_option(config_type, section, item, value): + cfg_type_changed = True + if cfg_type_changed: + idleConf.userCfg[config_type].Save() + for config_type in ['keys', 'highlight']: + # Save these even if unchanged! + idleConf.userCfg[config_type].Save() + self.clear() + # ConfigDialog caller must add the following call + # self.save_all_changed_extensions() # Uses a different mechanism. + + def delete_section(self, config_type, section): + """Delete a section from self, userCfg, and file. + + Used to delete custom themes and keysets. + """ + if section in self[config_type]: + del self[config_type][section] + configpage = idleConf.userCfg[config_type] + configpage.remove_section(section) + configpage.Save() + + def clear(self): + """Clear all 4 pages. + + Called in save_all after saving to idleConf. + XXX Mark window *title* when there are changes; unmark here. + """ + for page in self.pages: + page.clear() + + # TODO Revise test output, write expanded unittest -# -if __name__ == '__main__': +def _dump(): # htest # (not really, but ignore in coverage) from zlib import crc32 line, crc = 0, 0 @@ -790,10 +882,10 @@ def sprint(obj): line += 1 crc = crc32(txt.encode(encoding='utf-8'), crc) print(txt) - #print('***', line, crc, '***') # uncomment for diagnosis + #print('***', line, crc, '***') # Uncomment for diagnosis. def dumpCfg(cfg): - print('\n', cfg, '\n') # has variable '0xnnnnnnnn' addresses + print('\n', cfg, '\n') # Cfg has variable '0xnnnnnnnn' address. for key in sorted(cfg.keys()): sections = cfg[key].sections() sprint(key) @@ -808,3 +900,9 @@ def dumpCfg(cfg): dumpCfg(idleConf.defaultCfg) dumpCfg(idleConf.userCfg) print('\nlines = ', line, ', crc = ', crc, sep='') + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_config', + verbosity=2, exit=False) + #_dump() diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 84092275183685..ade6710001d69a 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -20,7 +20,7 @@ import tkinter.font as tkFont import tkinter.messagebox as tkMessageBox -from idlelib.config import idleConf +from idlelib.config import idleConf, ConfigChanges from idlelib.config_key import GetKeysDialog from idlelib.dynoption import DynOptionMenu from idlelib import macosx @@ -28,6 +28,8 @@ from idlelib.tabbedpages import TabbedPageSet from idlelib.textview import view_text +changes = ConfigChanges() + class ConfigDialog(Toplevel): """Config dialog for IDLE. @@ -71,7 +73,6 @@ def __init__(self, parent, title='', _htest=False, _utest=False): 'Shell Stdout Text': ('stdout', '12'), 'Shell Stderr Text': ('stderr', '13'), } - self.reset_changed_items() # Initialize changed_items dict. self.create_widgets() self.resizable(height=FALSE, width=FALSE) self.transient(parent) @@ -559,16 +560,16 @@ def var_changed_font(self, *params): overriding the default font, we need to write out everything. """ value = self.font_name.get() - self.add_changed_item('main', 'EditorWindow', 'font', value) + changes.add_option('main', 'EditorWindow', 'font', value) value = self.font_size.get() - self.add_changed_item('main', 'EditorWindow', 'font-size', value) + changes.add_option('main', 'EditorWindow', 'font-size', value) value = self.font_bold.get() - self.add_changed_item('main', 'EditorWindow', 'font-bold', value) + changes.add_option('main', 'EditorWindow', 'font-bold', value) def var_changed_space_num(self, *params): "Store change to indentation size." value = self.space_num.get() - self.add_changed_item('main', 'Indent', 'num-spaces', value) + changes.add_option('main', 'Indent', 'num-spaces', value) def var_changed_colour(self, *params): "Process change to color choice." @@ -584,13 +585,13 @@ def var_changed_builtin_theme(self, *params): value = self.builtin_theme.get() if value not in old_themes: if idleConf.GetOption('main', 'Theme', 'name') not in old_themes: - self.add_changed_item('main', 'Theme', 'name', old_themes[0]) - self.add_changed_item('main', 'Theme', 'name2', value) + changes.add_option('main', 'Theme', 'name', old_themes[0]) + changes.add_option('main', 'Theme', 'name2', value) self.new_custom_theme.config(text='New theme, see Help', fg='#500000') else: - self.add_changed_item('main', 'Theme', 'name', value) - self.add_changed_item('main', 'Theme', 'name2', '') + changes.add_option('main', 'Theme', 'name', value) + changes.add_option('main', 'Theme', 'name2', '') self.new_custom_theme.config(text='', fg='black') self.paint_theme_sample() @@ -602,7 +603,7 @@ def var_changed_custom_theme(self, *params): """ value = self.custom_theme.get() if value != '- no custom themes -': - self.add_changed_item('main', 'Theme', 'name', value) + changes.add_option('main', 'Theme', 'name', value) self.paint_theme_sample() def var_changed_is_builtin_theme(self, *params): @@ -612,7 +613,7 @@ def var_changed_is_builtin_theme(self, *params): selected theme type. """ value = self.is_builtin_theme.get() - self.add_changed_item('main', 'Theme', 'default', value) + changes.add_option('main', 'Theme', 'default', value) if value: self.var_changed_builtin_theme() else: @@ -628,11 +629,11 @@ def var_changed_keybinding(self, *params): key_set = self.custom_keys.get() event = self.list_bindings.get(ANCHOR).split()[0] if idleConf.IsCoreBinding(event): - self.add_changed_item('keys', key_set, event, value) + changes.add_option('keys', key_set, event, value) else: # Event is an extension binding. ext_name = idleConf.GetExtnNameForEvent(event) ext_keybind_section = ext_name + '_cfgBindings' - self.add_changed_item('extensions', ext_keybind_section, event, value) + changes.add_option('extensions', ext_keybind_section, event, value) def var_changed_builtin_keys(self, *params): "Process selection of builtin key set." @@ -645,13 +646,13 @@ def var_changed_builtin_keys(self, *params): value = self.builtin_keys.get() if value not in old_keys: if idleConf.GetOption('main', 'Keys', 'name') not in old_keys: - self.add_changed_item('main', 'Keys', 'name', old_keys[0]) - self.add_changed_item('main', 'Keys', 'name2', value) + changes.add_option('main', 'Keys', 'name', old_keys[0]) + changes.add_option('main', 'Keys', 'name2', value) self.new_custom_keys.config(text='New key set, see Help', fg='#500000') else: - self.add_changed_item('main', 'Keys', 'name', value) - self.add_changed_item('main', 'Keys', 'name2', '') + changes.add_option('main', 'Keys', 'name', value) + changes.add_option('main', 'Keys', 'name2', '') self.new_custom_keys.config(text='', fg='black') self.load_keys_list(value) @@ -659,13 +660,13 @@ def var_changed_custom_keys(self, *params): "Process selection of custom key set." value = self.custom_keys.get() if value != '- no custom keys -': - self.add_changed_item('main', 'Keys', 'name', value) + changes.add_option('main', 'Keys', 'name', value) self.load_keys_list(value) def var_changed_are_keys_builtin(self, *params): "Process toggle between builtin key set and custom key set." value = self.are_keys_builtin.get() - self.add_changed_item('main', 'Keys', 'default', value) + changes.add_option('main', 'Keys', 'default', value) if value: self.var_changed_builtin_keys() else: @@ -674,59 +675,27 @@ def var_changed_are_keys_builtin(self, *params): def var_changed_win_width(self, *params): "Store change to window width." value = self.win_width.get() - self.add_changed_item('main', 'EditorWindow', 'width', value) + changes.add_option('main', 'EditorWindow', 'width', value) def var_changed_win_height(self, *params): "Store change to window height." value = self.win_height.get() - self.add_changed_item('main', 'EditorWindow', 'height', value) + changes.add_option('main', 'EditorWindow', 'height', value) def var_changed_startup_edit(self, *params): "Store change to toggle for starting IDLE in the editor or shell." value = self.startup_edit.get() - self.add_changed_item('main', 'General', 'editor-on-startup', value) + changes.add_option('main', 'General', 'editor-on-startup', value) def var_changed_autosave(self, *params): "Store change to autosave." value = self.autosave.get() - self.add_changed_item('main', 'General', 'autosave', value) + changes.add_option('main', 'General', 'autosave', value) def var_changed_encoding(self, *params): "Store change to encoding." value = self.encoding.get() - self.add_changed_item('main', 'EditorWindow', 'encoding', value) - - def reset_changed_items(self): - """Reset dictionary containing the items changed on each tab. - - When any config item is changed in this dialog, an entry - should be made in the relevant section (config type) of this - dictionary. The key should be the config file section name and the - value a dictionary, whose key:value pairs are item=value pairs for - that config file section. - """ - self.changed_items = {'main':{}, 'highlight':{}, 'keys':{}, - 'extensions':{}} - - def add_changed_item(self, typ, section, item, value): - "Add item/value pair to changed items dictionary for typ and section." - value = str(value) # Make sure we use a string. - if section not in self.changed_items[typ]: - self.changed_items[typ][section] = {} - self.changed_items[typ][section][item] = value - - def GetDefaultItems(self): - "Return dictionary of default configuration settings." - d_items={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} - for config_type in d_items: - sections = idleConf.GetSectionList('default', config_type) - for section in sections: - d_items[config_type][section] = {} - options = idleConf.defaultCfg[config_type].GetOptionList(section) - for option in options: - d_items[config_type][section][option] = ( - idleConf.defaultCfg[config_type].Get(section, option)) - return d_items + changes.add_option('main', 'EditorWindow', 'encoding', value) def set_theme_type(self): "Set available screen options based on builtin or custom theme." @@ -769,8 +738,8 @@ def get_new_keys(self): else: current_key_set_name = self.custom_keys.get() current_bindings = idleConf.GetCurrentKeySet() - if current_key_set_name in self.changed_items['keys']: # unsaved changes - key_set_changes = self.changed_items['keys'][current_key_set_name] + if current_key_set_name in changes['keys']: # unsaved changes + key_set_changes = changes['keys'][current_key_set_name] for event in key_set_changes: current_bindings[event] = key_set_changes[event].split() current_key_sequences = list(current_bindings.values()) @@ -832,8 +801,8 @@ def create_new_key_set(self, new_key_set_name): binding = ' '.join(prev_keys[event]) new_keys[event_name] = binding # Handle any unsaved changes to prev key set. - if prev_key_set_name in self.changed_items['keys']: - key_set_changes = self.changed_items['keys'][prev_key_set_name] + if prev_key_set_name in changes['keys']: + key_set_changes = changes['keys'][prev_key_set_name] for event in key_set_changes: new_keys[event] = key_set_changes[event] # Save the new key set. @@ -863,10 +832,10 @@ def load_keys_list(self, keyset_name): for bind_name in bind_names: key = ' '.join(keyset[bind_name]) bind_name = bind_name[2:-2] # Trim off the angle brackets. - if keyset_name in self.changed_items['keys']: + if keyset_name in changes['keys']: # Handle any unsaved changes to this key set. - if bind_name in self.changed_items['keys'][keyset_name]: - key = self.changed_items['keys'][keyset_name][bind_name] + if bind_name in changes['keys'][keyset_name]: + key = changes['keys'][keyset_name][bind_name] self.list_bindings.insert(END, bind_name+' - '+key) if reselect: self.list_bindings.see(list_index) @@ -886,12 +855,8 @@ def delete_custom_keys(self): 'Delete Key Set', delmsg % keyset_name, parent=self): return self.deactivate_current_config() - # Remove key set from config. - idleConf.userCfg['keys'].remove_section(keyset_name) - if keyset_name in self.changed_items['keys']: - del(self.changed_items['keys'][keyset_name]) - # Write changes. - idleConf.userCfg['keys'].Save() + # Remove key set from changes, config, and file. + changes.remove(keyset_name) # Reload user key set list. item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() @@ -906,7 +871,8 @@ def delete_custom_keys(self): self.builtin_keys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') or idleConf.default_keys()) # User can't back out of these changes, they must be applied now. - self.save_all_changed_configs() + changes.save_all() + self.save_all_changed_extensions() self.activate_config_changes() self.set_keys_type() @@ -923,12 +889,8 @@ def delete_custom_theme(self): 'Delete Theme', delmsg % theme_name, parent=self): return self.deactivate_current_config() - # Remove theme from config. - idleConf.userCfg['highlight'].remove_section(theme_name) - if theme_name in self.changed_items['highlight']: - del(self.changed_items['highlight'][theme_name]) - # Write changes. - idleConf.userCfg['highlight'].Save() + # Remove theme from changes, config, and file. + changes.delete_section('highlight') # Reload user theme list. item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() @@ -941,7 +903,8 @@ def delete_custom_theme(self): self.is_builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) self.builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) # User can't back out of these changes, they must be applied now. - self.save_all_changed_configs() + changes.save_all() + self.save_all_changed_extensions() self.activate_config_changes() self.set_theme_type() @@ -979,7 +942,7 @@ def on_new_colour_set(self): self.text_highlight_sample.tag_config(sample_element, **{plane:new_colour}) theme = self.custom_theme.get() theme_element = sample_element + '-' + plane - self.add_changed_item('highlight', theme, theme_element, new_colour) + changes.add_option('highlight', theme, theme_element, new_colour) def get_new_theme_name(self, message): "Return name of new theme from query popup." @@ -1010,8 +973,8 @@ def create_new_theme(self, new_theme_name): theme_name = self.custom_theme.get() new_theme = idleConf.GetThemeDict(theme_type, theme_name) # Apply any of the old theme's unsaved changes to the new theme. - if theme_name in self.changed_items['highlight']: - theme_changes = self.changed_items['highlight'][theme_name] + if theme_name in changes['highlight']: + theme_changes = changes['highlight'][theme_name] for element in theme_changes: new_theme[element] = theme_changes[element] # Save the new theme. @@ -1078,8 +1041,8 @@ def paint_theme_sample(self): colours['background'] = idleConf.GetHighlight( theme, 'normal', fgBg='bg') # Handle any unsaved changes to this theme. - if theme in self.changed_items['highlight']: - theme_dict = self.changed_items['highlight'][theme] + if theme in changes['highlight']: + theme_dict = changes['highlight'][theme] if element + '-foreground' in theme_dict: colours['foreground'] = theme_dict[element + '-foreground'] if element + '-background' in theme_dict: @@ -1150,10 +1113,10 @@ def helplist_item_remove(self): self.set_helplist_button_states() def update_user_help_changed_items(self): - "Clear and rebuild the HelpFiles section in self.changed_items" - self.changed_items['main']['HelpFiles'] = {} + "Clear and rebuild the HelpFiles section in changes" + changes['main']['HelpFiles'] = {} for num in range(1, len(self.user_helplist) + 1): - self.add_changed_item( + changes.add_option( 'main', 'HelpFiles', str(num), ';'.join(self.user_helplist[num-1][:2])) @@ -1317,37 +1280,6 @@ def save_new_theme(self, theme_name, theme): value = theme[element] idleConf.userCfg['highlight'].SetOption(theme_name, element, value) - def set_user_value(self, config_type, section, item, value): - "Return True if the configuration value was added or changed." - if idleConf.defaultCfg[config_type].has_option(section, item): - if idleConf.defaultCfg[config_type].Get(section, item) == value: - # The setting equals a default setting, remove it from user cfg. - return idleConf.userCfg[config_type].RemoveOption(section, item) - # If we got here, set the option. - return idleConf.userCfg[config_type].SetOption(section, item, value) - - def save_all_changed_configs(self): - "Save all configuration changes to the user config file." - idleConf.userCfg['main'].Save() - for config_type in self.changed_items: - cfg_type_changed = False - for section in self.changed_items[config_type]: - if section == 'HelpFiles': - # This section gets completely replaced. - idleConf.userCfg['main'].remove_section('HelpFiles') - cfg_type_changed = True - for item in self.changed_items[config_type][section]: - value = self.changed_items[config_type][section][item] - if self.set_user_value(config_type, section, item, value): - cfg_type_changed = True - if cfg_type_changed: - idleConf.userCfg[config_type].Save() - for config_type in ['keys', 'highlight']: - # Save these even if unchanged! - idleConf.userCfg[config_type].Save() - self.reset_changed_items() # Clear the changed items dict. - self.save_all_changed_extensions() # Uses a different mechanism. - def deactivate_current_config(self): "Remove current key bindings." # Before a config is saved, some cleanup of current @@ -1378,7 +1310,8 @@ def ok(self): def apply(self): "Apply config changes and leave dialog open." self.deactivate_current_config() - self.save_all_changed_configs() + changes.save_all() + self.save_all_changed_extensions() self.activate_config_changes() def help(self): diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index e678cc6f332670..e157bbb5b52c46 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -9,12 +9,13 @@ # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. -# Replace user parsers with empty parsers that cannot be saved. +# Replace user parsers with empty parsers that cannot be saved +# due to getting '' as the filename when created. idleConf = config.idleConf usercfg = idleConf.userCfg testcfg = {} -usermain = testcfg['main'] = config.IdleUserConfParser('') # filename +usermain = testcfg['main'] = config.IdleUserConfParser('') userhigh = testcfg['highlight'] = config.IdleUserConfParser('') userkeys = testcfg['keys'] = config.IdleUserConfParser('') @@ -136,6 +137,87 @@ def test_user_override_keys(self): userkeys.remove_section('Custom Keys') +class ChangesTest(unittest.TestCase): + + empty = {'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} + + def load(self): # Test_add_option verifies that this works. + changes = self.changes + changes.add_option('main', 'Msec', 'mitem', 'mval') + changes.add_option('highlight', 'Hsec', 'hitem', 'hval') + changes.add_option('keys', 'Ksec', 'kitem', 'kval') + return changes + + loaded = {'main': {'Msec': {'mitem': 'mval'}}, + 'highlight': {'Hsec': {'hitem': 'hval'}}, + 'keys': {'Ksec': {'kitem':'kval'}}, + 'extensions': {}} + + def setUp(self): + self.changes = config.ConfigChanges() + + def test_init(self): + self.assertEqual(self.changes, self.empty) + + def test_add_option(self): + changes = self.load() + self.assertEqual(changes, self.loaded) + changes.add_option('main', 'Msec', 'mitem', 'mval') + self.assertEqual(changes, self.loaded) + + def test_save_option(self): # Static function does not touch changes. + save_option = self.changes.save_option + self.assertTrue(save_option('main', 'Indent', 'what', '0')) + self.assertFalse(save_option('main', 'Indent', 'what', '0')) + self.assertEqual(usermain['Indent']['what'], '0') + + self.assertTrue(save_option('main', 'Indent', 'use-spaces', '0')) + self.assertEqual(usermain['Indent']['use-spaces'], '0') + self.assertTrue(save_option('main', 'Indent', 'use-spaces', '1')) + self.assertFalse(usermain.has_option('Indent', 'use-spaces')) + usermain.remove_section('Indent') + + def test_save_added(self): + changes = self.load() + changes.save_all() + self.assertEqual(usermain['Msec']['mitem'], 'mval') + self.assertEqual(userhigh['Hsec']['hitem'], 'hval') + self.assertEqual(userkeys['Ksec']['kitem'], 'kval') + usermain.remove_section('Msec') + userhigh.remove_section('Hsec') + userkeys.remove_section('Ksec') + + def test_save_help(self): + changes = self.changes + changes.save_option('main', 'HelpFiles', 'IDLE', 'idledoc') + changes.add_option('main', 'HelpFiles', 'ELDI', 'codeldi') + changes.save_all() + self.assertFalse(usermain.has_option('HelpFiles', 'IDLE')) + self.assertTrue(usermain.has_option('HelpFiles', 'ELDI')) + + def test_save_default(self): # Cover 2nd and 3rd false branches. + changes = self.changes + changes.add_option('main', 'Indent', 'use-spaces', '1') + # save_option returns False; cfg_type_changed remains False. + + # TODO: test that save_all calls usercfg Saves. + + def test_delete_section(self): + changes = self.load() + changes.delete_section('main', 'fake') # Test no exception. + self.assertEqual(changes, self.loaded) # Test nothing deleted. + for cfgtype, section in (('main', 'Msec'), ('keys', 'Ksec')): + changes.delete_section(cfgtype, section) + with self.assertRaises(KeyError): + changes[cfgtype][section] # Test section gone. + # TODO Test change to userkeys and maybe save call. + + def test_clear(self): + changes = self.load() + changes.clear() + self.assertEqual(changes, self.empty) + + class WarningTest(unittest.TestCase): def test_warn(self): diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index ffc8a723ca0e6a..26b045d353550b 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -3,7 +3,7 @@ Half the class creates dialog, half works with user customizations. Coverage: 46% just by creating dialog, 56% with current tests. """ -from idlelib.configdialog import ConfigDialog, idleConf # test import +from idlelib.configdialog import ConfigDialog, idleConf, changes from test.support import requires requires('gui') from tkinter import Tk @@ -21,17 +21,13 @@ 'extensions': config.IdleUserConfParser(''), } -# ConfigDialog.changed_items is a 3-level hierarchical dictionary of -# pending changes that mirrors the multilevel user config dict. -# For testing, record args in a list for comparison with expected. -changes = [] root = None configure = None +mainpage = changes['main'] +highpage = changes['highlight'] +keyspage = changes['keys'] - -class TestDialog(ConfigDialog): - def add_changed_item(self, *args): - changes.append(args) +class TestDialog(ConfigDialog): pass # Delete? def setUpModule(): @@ -63,31 +59,28 @@ def test_font(self): default_size = str(default_font[1]) default_bold = default_font[2] == 'bold' configure.font_name.set('Test Font') - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', default_size), - ('main', 'EditorWindow', 'font-bold', default_bold)] - self.assertEqual(changes, expected) + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': default_size, + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) changes.clear() configure.font_size.set(20) - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', default_bold)] - self.assertEqual(changes, expected) + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) changes.clear() configure.font_bold.set(not default_bold) - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', not default_bold)] - self.assertEqual(changes, expected) + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(not default_bold)}} + self.assertEqual(mainpage, expected) #def test_sample(self): pass # TODO def test_tabspace(self): configure.space_num.set(6) - self.assertEqual(changes, [('main', 'Indent', 'num-spaces', 6)]) + self.assertEqual(mainpage, {'Indent': {'num-spaces': '6'}}) class HighlightTest(unittest.TestCase): @@ -111,19 +104,19 @@ def setUp(self): def test_startup(self): configure.radio_startup_edit.invoke() - self.assertEqual(changes, - [('main', 'General', 'editor-on-startup', 1)]) + self.assertEqual(mainpage, + {'General': {'editor-on-startup': '1'}}) def test_autosave(self): configure.radio_save_auto.invoke() - self.assertEqual(changes, [('main', 'General', 'autosave', 1)]) + self.assertEqual(mainpage, {'General': {'autosave': '1'}}) def test_editor_size(self): configure.entry_win_height.insert(0, '1') - self.assertEqual(changes, [('main', 'EditorWindow', 'height', '140')]) + self.assertEqual(mainpage, {'EditorWindow': {'height': '140'}}) changes.clear() configure.entry_win_width.insert(0, '1') - self.assertEqual(changes, [('main', 'EditorWindow', 'width', '180')]) + self.assertEqual(mainpage, {'EditorWindow': {'width': '180'}}) #def test_help_sources(self): pass # TODO From 9d8abf31c430dd83d073660cc92f1fe4ca6f2cd4 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 7 Jul 2017 22:26:53 -0400 Subject: [PATCH 0544/1345] [3.6] bpo-30779: News (GH-2627) (#2630) (cherry picked from commit 24f2e19) --- .../next/IDLE/2017-07-07-20-26-37.bpo-30779.8KXEXN.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-07-20-26-37.bpo-30779.8KXEXN.rst diff --git a/Misc/NEWS.d/next/IDLE/2017-07-07-20-26-37.bpo-30779.8KXEXN.rst b/Misc/NEWS.d/next/IDLE/2017-07-07-20-26-37.bpo-30779.8KXEXN.rst new file mode 100644 index 00000000000000..1d0dc181304217 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-07-20-26-37.bpo-30779.8KXEXN.rst @@ -0,0 +1,8 @@ +IDLE: Factor ConfigChanges class from configdialog, put in config; test. * +In config, put dump test code in a function; run it and unittest in 'if +__name__ == '__main__'. * Add class config.ConfigChanges based on +changes_class_v4.py on bpo issue. * Add class test_config.ChangesTest, +partly using configdialog_tests_v1.py. * Revise configdialog to use +ConfigChanges; see tracker msg297804. * Revise test_configdialog to match +configdialog changes. * Remove configdialog functions unused or moved to +ConfigChanges. Cheryl Sabella contributed parts of the patch. From 552f26680d3806df7c27dd7161fd7d57ac815f78 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 7 Jul 2017 22:47:37 -0400 Subject: [PATCH 0545/1345] [3.6] bpo-8231: Call idlelib.IdleConf.GetUserCfgDir only once. (GH-2629) (#2631) (cherry picked from commit 223c7e7) --- Lib/idlelib/config.py | 6 +++--- Lib/idlelib/editor.py | 4 ++-- Lib/idlelib/pyshell.py | 4 ++-- .../next/IDLE/2017-07-07-21-10-55.bpo-8231.yEge3L.rst | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-07-21-10-55.bpo-8231.yEge3L.rst diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 621e0bfc8b4073..ed37f11a9cb8e0 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -172,10 +172,10 @@ def CreateConfigHandlers(self): "Populate default and user config parser dictionaries." #build idle install path if __name__ != '__main__': # we were imported - idleDir=os.path.dirname(__file__) + idleDir = os.path.dirname(__file__) else: # we were exec'ed (for testing only) - idleDir=os.path.abspath(sys.path[0]) - userDir=self.GetUserCfgDir() + idleDir = os.path.abspath(sys.path[0]) + self.userdir = userDir = self.GetUserCfgDir() defCfgFiles = {} usrCfgFiles = {} diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 5b64ef4630e737..43b105f726573d 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -103,8 +103,8 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.tkinter_vars = {} # keys: Tkinter event names # values: Tkinter variable instances self.top.instance_dict = {} - self.recent_files_path = os.path.join(idleConf.GetUserCfgDir(), - 'recent-files.lst') + self.recent_files_path = os.path.join( + idleConf.userdir, 'recent-files.lst') self.text_frame = text_frame = Frame(top) self.vbar = vbar = Scrollbar(text_frame, name='vbar') self.width = idleConf.GetOption('main', 'EditorWindow', diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index be7bd3dfcfb30a..47df74433cf87a 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -117,8 +117,8 @@ def __init__(self, *args): self.text.bind("<>", self.clear_breakpoint_here) self.text.bind("<>", self.flist.open_shell) - self.breakpointPath = os.path.join(idleConf.GetUserCfgDir(), - 'breakpoints.lst') + self.breakpointPath = os.path.join( + idleConf.userdir, 'breakpoints.lst') # whenever a file is changed, restore breakpoints def filename_changed_hook(old_hook=self.io.filename_change_hook, self=self): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-07-21-10-55.bpo-8231.yEge3L.rst b/Misc/NEWS.d/next/IDLE/2017-07-07-21-10-55.bpo-8231.yEge3L.rst new file mode 100644 index 00000000000000..c96cc4e1438245 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-07-21-10-55.bpo-8231.yEge3L.rst @@ -0,0 +1 @@ +IDLE: call config.IdleConf.GetUserCfgDir only once. From 5fd33b5926eb8c9352bf5718369b4a8d72c4bb44 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 23:33:27 -0400 Subject: [PATCH 0546/1345] Version bump to 3.6.2final --- Include/patchlevel.h | 6 +++--- Misc/NEWS | 8 ++++++++ README.rst | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 91b8ece15ba4a6..312a6fbab7f170 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 2 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.2rc2" +#define PY_VERSION "3.6.2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index a76d2a1695b7e7..644b4c37c862c4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,14 @@ Python News +++++++++++ +What's New in Python 3.6.2? +=========================== + +*Release date: 2017-07-17* + +- No changes since release candidate 2 + + What's New in Python 3.6.2 release candidate 2? =============================================== diff --git a/README.rst b/README.rst index 4b564759610206..c43cd4424baa14 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.6.2 release candidate 2 -================================================ +This is Python version 3.6.2 +============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI From 03e0df66b8ce0a8d980eb2092b7c6464d26db14e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 8 Jul 2017 00:49:40 -0400 Subject: [PATCH 0547/1345] Update for post-3.6.2rc2 --- Include/patchlevel.h | 4 ++-- Misc/NEWS | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b4996d55ec841f..0c82f5c0db855a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 2 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.6.2rc1+" +#define PY_VERSION "3.6.2rc2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 89876f8fa24543..3c16a66e6ffba0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -21,6 +21,21 @@ Library - bpo-30746: Prohibited the '=' character in environment variable names in ``os.putenv()`` and ``os.spawn*()``. +- bpo-29755: Fixed the lgettext() family of functions in the gettext module. + They now always return bytes. + + +What's New in Python 3.6.2 release candidate 2? +=============================================== + +*Release date: 2017-07-07* + +Core and Builtins +----------------- + +Library +------- + - [Security] bpo-30730: Prevent environment variables injection in subprocess on Windows. Prevent passing other environment variables and command arguments. @@ -33,9 +48,6 @@ Library doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. -- bpo-29755: Fixed the lgettext() family of functions in the gettext module. - They now always return bytes. - - [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` From 04f77d4677e7508b6ec8de9d0331fdabbcd11d30 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sat, 8 Jul 2017 21:51:21 +0300 Subject: [PATCH 0548/1345] [3.6] bpo-29854: Fix segfault in call_readline() (GH-728) If history-length is set in .inputrc, and the history file is double the history size (or more), history_get(N) returns NULL, and python segfaults. Fix that by checking for NULL return value. It seems that the root cause is incorrect handling of bigger history in readline, but Python should not segfault even if readline returns unexpected value. This issue affects only GNU readline. When using libedit emulation system history size option does not work. --- Lib/test/test_readline.py | 50 +++++++++++++++++-- .../2017-07-07-02-18-57.bpo-29854.J8wKb_.rst | 2 + Modules/readline.c | 10 ++-- 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-07-02-18-57.bpo-29854.J8wKb_.rst diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 06a9149374aeab..9c979652a1d0e9 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -9,7 +9,7 @@ import sys import tempfile import unittest -from test.support import import_module, unlink, TESTFN +from test.support import import_module, unlink, temp_dir, TESTFN from test.support.script_helper import assert_python_ok # Skip tests if there is no readline module @@ -210,13 +210,57 @@ def display(substitution, matches, longest_match_length): self.assertIn(b"result " + expected + b"\r\n", output) self.assertIn(b"history " + expected + b"\r\n", output) + # We have 2 reasons to skip this test: + # - readline: history size was added in 6.0 + # See https://cnswww.cns.cwru.edu/php/chet/readline/CHANGES + # - editline: history size is broken on OS X 10.11.6. + # Newer versions were not tested yet. + @unittest.skipIf(readline._READLINE_VERSION < 0x600, + "this readline version does not support history-size") + @unittest.skipIf(is_editline, + "editline history size configuration is broken") + def test_history_size(self): + history_size = 10 + with temp_dir() as test_dir: + inputrc = os.path.join(test_dir, "inputrc") + with open(inputrc, "wb") as f: + f.write(b"set history-size %d\n" % history_size) + + history_file = os.path.join(test_dir, "history") + with open(history_file, "wb") as f: + # history_size * 2 items crashes readline + data = b"".join(b"item %d\n" % i + for i in range(history_size * 2)) + f.write(data) + + script = """ +import os +import readline + +history_file = os.environ["HISTORY_FILE"] +readline.read_history_file(history_file) +input() +readline.write_history_file(history_file) +""" + + env = dict(os.environ) + env["INPUTRC"] = inputrc + env["HISTORY_FILE"] = history_file + + run_pty(script, input=b"last input\r", env=env) + + with open(history_file, "rb") as f: + lines = f.readlines() + self.assertEqual(len(lines), history_size) + self.assertEqual(lines[-1].strip(), b"last input") + -def run_pty(script, input=b"dummy input\r"): +def run_pty(script, input=b"dummy input\r", env=None): pty = import_module('pty') output = bytearray() [master, slave] = pty.openpty() args = (sys.executable, '-c', script) - proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) os.close(slave) with ExitStack() as cleanup: cleanup.enter_context(proc) diff --git a/Misc/NEWS.d/next/Library/2017-07-07-02-18-57.bpo-29854.J8wKb_.rst b/Misc/NEWS.d/next/Library/2017-07-07-02-18-57.bpo-29854.J8wKb_.rst new file mode 100644 index 00000000000000..5c439087dcb344 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-07-02-18-57.bpo-29854.J8wKb_.rst @@ -0,0 +1,2 @@ +Fix segfault in readline when using readline's history-size option. Patch +by Nir Soffer. diff --git a/Modules/readline.c b/Modules/readline.c index 3b94d4a0d1d759..575479cff249be 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1347,15 +1347,17 @@ call_readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) if (should_auto_add_history && n > 0) { const char *line; int length = _py_get_history_length(); - if (length > 0) + if (length > 0) { + HIST_ENTRY *hist_ent; #ifdef __APPLE__ if (using_libedit_emulation) { /* handle older 0-based or newer 1-based indexing */ - line = (const char *)history_get(length + libedit_history_start - 1)->line; + hist_ent = history_get(length + libedit_history_start - 1); } else #endif /* __APPLE__ */ - line = (const char *)history_get(length)->line; - else + hist_ent = history_get(length); + line = hist_ent ? hist_ent->line : ""; + } else line = ""; if (strcmp(p, line)) add_history(p); From 7ab334233394070a25344d481c8de1402fa12b29 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 9 Jul 2017 19:26:32 -0400 Subject: [PATCH 0549/1345] [3.6] bpo-30870: IDLE: Change sample font when select by key-up/down (GH-2617) (#2640) Patch by Louie Lu. (cherry picked from commit bb2bae8) --- Lib/idlelib/configdialog.py | 52 +++++++++---------- .../2017-07-08-17-57-04.bpo-30870.IcR2pf.rst | 2 + 2 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-08-17-57-04.bpo-30870.IcR2pf.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index ade6710001d69a..1e1561f7457310 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -14,7 +14,7 @@ StringVar, BooleanVar, IntVar, TRUE, FALSE, TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, - HORIZONTAL, VERTICAL, ANCHOR, END) + HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import Scrollbar import tkinter.colorchooser as tkColorChooser import tkinter.font as tkFont @@ -78,7 +78,7 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.cancel) - self.tab_pages.focus_set() + self.fontlist.focus_set() # XXX Decide whether to keep or delete these key bindings. # Key bindings for this dialog. # self.bind('', self.Cancel) #dismiss dialog, no save @@ -143,26 +143,24 @@ def create_page_font_tab(self): self.space_num = IntVar(parent) self.edit_font = tkFont.Font(parent, ('courier', 10, 'normal')) - ##widget creation - #body frame + # Create widgets. + # body and body section frames. frame = self.tab_pages.pages['Fonts/Tabs'].frame - #body section frames frame_font = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') frame_indent = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') - #frame_font + # frame_font frame_font_name = Frame(frame_font) frame_font_param = Frame(frame_font) font_name_title = Label( frame_font_name, justify=LEFT, text='Font Face :') - self.list_fonts = Listbox( + self.fontlist = Listbox( frame_font_name, height=5, takefocus=FALSE, exportselection=FALSE) - self.list_fonts.bind( - '', self.on_list_fonts_button_release) + self.fontlist.bind('<>', self.on_fontlist_select) scroll_font = Scrollbar(frame_font_name) - scroll_font.config(command=self.list_fonts.yview) - self.list_fonts.config(yscrollcommand=scroll_font.set) + scroll_font.config(command=self.fontlist.yview) + self.fontlist.config(yscrollcommand=scroll_font.set) font_size_title = Label(frame_font_param, text='Size :') self.opt_menu_font_size = DynOptionMenu( frame_font_param, self.font_size, None, command=self.set_font_sample) @@ -173,7 +171,7 @@ def create_page_font_tab(self): self.font_sample = Label( frame_font_sample, justify=LEFT, font=self.edit_font, text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') - #frame_indent + # frame_indent frame_indent_size = Frame(frame_indent) indent_size_title = Label( frame_indent_size, justify=LEFT, @@ -182,25 +180,26 @@ def create_page_font_tab(self): frame_indent_size, variable=self.space_num, orient='horizontal', tickinterval=2, from_=2, to=16) - #widget packing - #body + # Pack widgets. + # body frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) - #frame_font + # frame_font frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) font_name_title.pack(side=TOP, anchor=W) - self.list_fonts.pack(side=LEFT, expand=TRUE, fill=X) + self.fontlist.pack(side=LEFT, expand=TRUE, fill=X) scroll_font.pack(side=LEFT, fill=Y) font_size_title.pack(side=LEFT, anchor=W) self.opt_menu_font_size.pack(side=LEFT, anchor=W) check_font_bold.pack(side=LEFT, anchor=W, padx=20) frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) self.font_sample.pack(expand=TRUE, fill=BOTH) - #frame_indent + # frame_indent frame_indent_size.pack(side=TOP, fill=X) indent_size_title.pack(side=TOP, anchor=W, padx=5) self.scale_indent_size.pack(side=TOP, padx=5, fill=X) + return frame def create_page_highlight(self): @@ -986,13 +985,13 @@ def create_new_theme(self, new_theme_name): self.is_builtin_theme.set(0) self.set_theme_type() - def on_list_fonts_button_release(self, event): - """Handle event of selecting a font from the list. + def on_fontlist_select(self, event): + """Handle selecting a font from the list. - Change the font name to the font selected from the list - and update sample text to show that font. + Event can result from either mouse click or Up or Down key. + Set font_name and example display to selection. """ - font = self.list_fonts.get(ANCHOR) + font = self.fontlist.get(ANCHOR if event.type == 3 else ACTIVE) self.font_name.set(font.lower()) self.set_font_sample() @@ -1126,7 +1125,7 @@ def load_font_cfg(self): fonts = list(tkFont.families(self)) fonts.sort() for font in fonts: - self.list_fonts.insert(END, font) + self.fontlist.insert(END, font) configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') font_name = configured_font[0].lower() font_size = configured_font[1] @@ -1135,9 +1134,10 @@ def load_font_cfg(self): lc_fonts = [s.lower() for s in fonts] try: current_font_index = lc_fonts.index(font_name) - self.list_fonts.see(current_font_index) - self.list_fonts.select_set(current_font_index) - self.list_fonts.select_anchor(current_font_index) + self.fontlist.see(current_font_index) + self.fontlist.select_set(current_font_index) + self.fontlist.select_anchor(current_font_index) + self.fontlist.activate(current_font_index) except ValueError: pass # Set font size dropdown. diff --git a/Misc/NEWS.d/next/IDLE/2017-07-08-17-57-04.bpo-30870.IcR2pf.rst b/Misc/NEWS.d/next/IDLE/2017-07-08-17-57-04.bpo-30870.IcR2pf.rst new file mode 100644 index 00000000000000..1bd0cbb6abffaa --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-08-17-57-04.bpo-30870.IcR2pf.rst @@ -0,0 +1,2 @@ +IDLE: In Settings dialog, select font with Up, Down keys as well as mouse. +Initial patch by Louie Lu. From 7f3d65d6e4f8bebaaf996efb1c1adb67eb1724cb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jul 2017 13:43:20 +0200 Subject: [PATCH 0550/1345] bpo-30886: Fix multiprocessing.Queue.join_thread() (#2642) (#2643) multiprocessing.Queue.join_thread() now waits until the thread completes, even if the thread was started by the same process which created the queue. Fix the following warning which occurs randomly when running test_handle_called_with_mp_queue of test_logging.QueueListenerTest: Warning -- threading_cleanup() failed to cleanup -1 threads after 4 sec (count: 0, dangling: 1) (cherry picked from commit 3b69d911c57ef591ac0c0f47a66dbcad8337f33a) --- Lib/multiprocessing/queues.py | 9 +-------- .../Library/2017-07-10-12-14-22.bpo-30886.nqQj34.rst | 3 +++ 2 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-10-12-14-22.bpo-30886.nqQj34.rst diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 7f77837a74acb6..76684172528f93 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -169,14 +169,7 @@ def _start_thread(self): self._thread.start() debug('... done self._thread.start()') - # On process exit we will wait for data to be flushed to pipe. - # - # However, if this process created the queue then all - # processes which use the queue will be descendants of this - # process. Therefore waiting for the queue to be flushed - # is pointless once all the child processes have been joined. - created_by_this_process = (self._opid == os.getpid()) - if not self._joincancelled and not created_by_this_process: + if not self._joincancelled: self._jointhread = Finalize( self._thread, Queue._finalize_join, [weakref.ref(self._thread)], diff --git a/Misc/NEWS.d/next/Library/2017-07-10-12-14-22.bpo-30886.nqQj34.rst b/Misc/NEWS.d/next/Library/2017-07-10-12-14-22.bpo-30886.nqQj34.rst new file mode 100644 index 00000000000000..fedd6cd04741fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-10-12-14-22.bpo-30886.nqQj34.rst @@ -0,0 +1,3 @@ +Fix multiprocessing.Queue.join_thread(): it now waits until the thread +completes, even if the thread was started by the same process which created +the queue. From 8e3f73e5499a5bc575a5e7ad4ee0f16b1aaae829 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Mon, 10 Jul 2017 15:11:45 -0400 Subject: [PATCH 0551/1345] [3.6] bpo-30851: IDLE: Remove unused tk variables in configdialog. (GH-2626) (#2648) One is a duplicate, one is set but cannot be altered by users. Patch by Cheryl Sabella. (cherry picked from commit aa8d0a2) --- Lib/idlelib/configdialog.py | 23 +------------------ .../2017-07-09-23-53-00.bpo-30851.AHXBYa.rst | 2 ++ 2 files changed, 3 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-09-23-53-00.bpo-30851.AHXBYa.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 1e1561f7457310..cf5bea7ec7ece6 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -218,8 +218,6 @@ def create_page_highlight(self): self.custom_theme = StringVar(parent) self.fg_bg_toggle = BooleanVar(parent) self.colour = StringVar(parent) - # XXX - font_name is defined in create_page_font_tab. Needed here too? - self.font_name = StringVar(parent) self.is_builtin_theme = BooleanVar(parent) self.highlight_target = StringVar(parent) @@ -334,8 +332,6 @@ def create_page_keys(self): keybinding: Action/key bindings. """ parent = self.parent - # XXX - binding_target isn't used. - self.binding_target = StringVar(parent) self.builtin_keys = StringVar(parent) self.custom_keys = StringVar(parent) self.are_keys_builtin = BooleanVar(parent) @@ -420,18 +416,12 @@ def create_page_general(self): win_height: Initial window height in characters. startup_edit: Selector for opening in editor or shell mode. autosave: Selector for save prompt popup when using Run. - encoding: ? """ parent = self.parent self.win_width = StringVar(parent) self.win_height = StringVar(parent) self.startup_edit = IntVar(parent) self.autosave = IntVar(parent) - # XXX - encoding isn't on the screen to be set, but is saved to config. - self.encoding = StringVar(parent) - # XXX - user_help_browser and help_browser aren't used. - self.user_help_browser = BooleanVar(parent) - self.help_browser = StringVar(parent) #widget creation #body @@ -538,7 +528,6 @@ def attach_var_callbacks(self): self.win_height.trace_add('write', self.var_changed_win_height) self.startup_edit.trace_add('write', self.var_changed_startup_edit) self.autosave.trace_add('write', self.var_changed_autosave) - self.encoding.trace_add('write', self.var_changed_encoding) def remove_var_callbacks(self): "Remove callbacks to prevent memory leaks." @@ -548,7 +537,7 @@ def remove_var_callbacks(self): self.custom_theme, self.is_builtin_theme, self.highlight_target, self.keybinding, self.builtin_keys, self.custom_keys, self.are_keys_builtin, self.win_width, self.win_height, - self.startup_edit, self.autosave, self.encoding,): + self.startup_edit, self.autosave,): var.trace_remove('write', var.trace_info()[0][1]) def var_changed_font(self, *params): @@ -691,11 +680,6 @@ def var_changed_autosave(self, *params): value = self.autosave.get() changes.add_option('main', 'General', 'autosave', value) - def var_changed_encoding(self, *params): - "Store change to encoding." - value = self.encoding.get() - changes.add_option('main', 'EditorWindow', 'encoding', value) - def set_theme_type(self): "Set available screen options based on builtin or custom theme." if self.is_builtin_theme.get(): @@ -819,8 +803,6 @@ def load_keys_list(self, keyset_name): An action/key binding can be selected to change the key binding. """ reselect = 0 - # XXX - new_keyset isn't used in this function. - new_keyset = 0 if self.list_bindings.curselection(): reselect = 1 list_index = self.list_bindings.index(ANCHOR) @@ -1234,9 +1216,6 @@ def load_general_cfg(self): 'main', 'EditorWindow', 'width', type='int')) self.win_height.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) - # Set default source encoding. - self.encoding.set(idleConf.GetOption( - 'main', 'EditorWindow', 'encoding', default='none')) # Set additional help sources. self.user_helplist = idleConf.GetAllExtraHelpSourcesList() for help_item in self.user_helplist: diff --git a/Misc/NEWS.d/next/IDLE/2017-07-09-23-53-00.bpo-30851.AHXBYa.rst b/Misc/NEWS.d/next/IDLE/2017-07-09-23-53-00.bpo-30851.AHXBYa.rst new file mode 100644 index 00000000000000..5ed7c8bb1af630 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-09-23-53-00.bpo-30851.AHXBYa.rst @@ -0,0 +1,2 @@ +IDLE: Remove unused variables in configdialog. One is a duplicate, one is +set but cannot be altered by users. Patch by Cheryl Sabella. From 044e156426825acac8b6c6d1ce14d5b7bcb20bc9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jul 2017 23:12:37 +0200 Subject: [PATCH 0552/1345] bpo-30892: Fix _elementtree module initialization (#2647) (#2649) Handle getattr(copy, 'deepcopy') error in _elementtree module initialization. (cherry picked from commit b136f11f3a51f9282ae992bac68f170ca5563b55) --- Modules/_elementtree.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index bef702ebe69c93..599ca6a4cc0d7c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3947,6 +3947,11 @@ PyInit__elementtree(void) st->deepcopy_obj = PyObject_GetAttrString(temp, "deepcopy"); Py_XDECREF(temp); + if (st->deepcopy_obj == NULL) { + return NULL; + } + + assert(!PyErr_Occurred()); if (!(st->elementpath_obj = PyImport_ImportModule("xml.etree.ElementPath"))) return NULL; From fe6e686c27ce3a3ecdf03803cff7e230dee5530d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jul 2017 23:16:27 +0200 Subject: [PATCH 0553/1345] bpo-30891: Fix importlib _find_and_load() race condition (#2646) (#2651) * Rewrite importlib _get_module_lock(): it is now responsible to hold the imp lock directly. * _find_and_load() now holds the module lock to check if name is in sys.modules to prevent a race condition (cherry picked from commit 4f9a446f3fb42f800e73cd9414dd1eccb3ca4fa7) --- Lib/importlib/_bootstrap.py | 58 +- Python/import.c | 4 - Python/importlib.h | 3044 +++++++++++++++++------------------ 3 files changed, 1548 insertions(+), 1558 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index f812a18cde3247..f7790c1db3a0af 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -39,6 +39,7 @@ def _new_module(name): # Module-level locking ######################################################## # A dict mapping module names to weakrefs of _ModuleLock instances +# Dictionary protected by the global import lock _module_locks = {} # A dict mapping thread ids to _ModuleLock instances _blocking_on = {} @@ -144,10 +145,7 @@ def __init__(self, name): self._lock = None def __enter__(self): - try: - self._lock = _get_module_lock(self._name) - finally: - _imp.release_lock() + self._lock = _get_module_lock(self._name) self._lock.acquire() def __exit__(self, *args, **kwargs): @@ -159,31 +157,37 @@ def __exit__(self, *args, **kwargs): def _get_module_lock(name): """Get or create the module lock for a given module name. - Should only be called with the import lock taken.""" - lock = None + Acquire/release internally the global import lock to protect + _module_locks.""" + + _imp.acquire_lock() try: - lock = _module_locks[name]() - except KeyError: - pass - if lock is None: - if _thread is None: - lock = _DummyModuleLock(name) - else: - lock = _ModuleLock(name) - def cb(_): - del _module_locks[name] - _module_locks[name] = _weakref.ref(lock, cb) + try: + lock = _module_locks[name]() + except KeyError: + lock = None + + if lock is None: + if _thread is None: + lock = _DummyModuleLock(name) + else: + lock = _ModuleLock(name) + def cb(_): + del _module_locks[name] + _module_locks[name] = _weakref.ref(lock, cb) + finally: + _imp.release_lock() + return lock + def _lock_unlock_module(name): - """Release the global import lock, and acquires then release the - module lock for a given module name. + """Acquires then releases the module lock for a given module name. + This is used to ensure a module is completely initialized, in the event it is being imported by another thread. - - Should only be called with the import lock taken.""" + """ lock = _get_module_lock(name) - _imp.release_lock() try: lock.acquire() except _DeadlockError: @@ -587,7 +591,6 @@ def _module_repr_from_spec(spec): def _exec(spec, module): """Execute the spec's specified module in an existing module's namespace.""" name = spec.name - _imp.acquire_lock() with _ModuleLockManager(name): if sys.modules.get(name) is not module: msg = 'module {!r} not in sys.modules'.format(name) @@ -670,7 +673,6 @@ def _load(spec): clobbered. """ - _imp.acquire_lock() with _ModuleLockManager(spec.name): return _load_unlocked(spec) @@ -957,16 +959,16 @@ def _find_and_load_unlocked(name, import_): def _find_and_load(name, import_): """Find and load the module.""" - _imp.acquire_lock() - if name not in sys.modules: - with _ModuleLockManager(name): + with _ModuleLockManager(name): + if name not in sys.modules: return _find_and_load_unlocked(name, import_) + module = sys.modules[name] if module is None: - _imp.release_lock() message = ('import of {} halted; ' 'None in sys.modules'.format(name)) raise ModuleNotFoundError(message, name=name) + _lock_unlock_module(name) return module diff --git a/Python/import.c b/Python/import.c index a9aa5b3c25bd2e..40e02c01ec95d4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1560,10 +1560,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (initializing == -1) PyErr_Clear(); if (initializing > 0) { -#ifdef WITH_THREAD - _PyImport_AcquireLock(); -#endif - /* _bootstrap._lock_unlock_module() releases the import lock */ value = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__lock_unlock_module, abs_name, NULL); diff --git a/Python/importlib.h b/Python/importlib.h index 0584477f190397..e44285036ea5b0 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -86,7 +86,7 @@ const unsigned char _Py_M__importlib[] = { 69,114,114,111,114,78,41,3,114,1,0,0,0,114,0,0, 0,0,114,2,0,0,0,114,10,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,114,17,0,0,0, - 47,0,0,0,115,2,0,0,0,8,1,114,17,0,0,0, + 48,0,0,0,115,2,0,0,0,8,1,114,17,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, 0,64,0,0,0,115,56,0,0,0,101,0,90,1,100,0, 90,2,100,1,90,3,100,2,100,3,132,0,90,4,100,4, @@ -115,7 +115,7 @@ const unsigned char _Py_M__importlib[] = { 101,114,218,5,99,111,117,110,116,218,7,119,97,105,116,101, 114,115,41,2,218,4,115,101,108,102,114,15,0,0,0,114, 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8, - 95,95,105,110,105,116,95,95,57,0,0,0,115,12,0,0, + 95,95,105,110,105,116,95,95,58,0,0,0,115,12,0,0, 0,0,1,10,1,10,1,6,1,6,1,6,1,122,20,95, 77,111,100,117,108,101,76,111,99,107,46,95,95,105,110,105, 116,95,95,99,1,0,0,0,0,0,0,0,4,0,0,0, @@ -129,7 +129,7 @@ const unsigned char _Py_M__importlib[] = { 107,105,110,103,95,111,110,218,3,103,101,116,41,4,114,26, 0,0,0,90,2,109,101,218,3,116,105,100,114,21,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,12,104,97,115,95,100,101,97,100,108,111,99,107,65,0, + 218,12,104,97,115,95,100,101,97,100,108,111,99,107,66,0, 0,0,115,18,0,0,0,0,2,8,1,6,1,2,1,10, 1,8,1,4,1,6,1,8,1,122,24,95,77,111,100,117, 108,101,76,111,99,107,46,104,97,115,95,100,101,97,100,108, @@ -165,7 +165,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,114,22,0,0,0,218,7,97,99,113,117,105,114,101, 114,25,0,0,0,218,7,114,101,108,101,97,115,101,41,2, 114,26,0,0,0,114,31,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,34,0,0,0,77,0, + 10,0,0,0,114,11,0,0,0,114,34,0,0,0,78,0, 0,0,115,32,0,0,0,0,6,8,1,8,1,2,1,2, 1,8,1,20,1,6,1,14,1,4,1,8,1,12,1,12, 1,24,2,10,1,18,2,122,19,95,77,111,100,117,108,101, @@ -188,7 +188,7 @@ const unsigned char _Py_M__importlib[] = { 111,110,69,114,114,111,114,114,25,0,0,0,114,22,0,0, 0,114,35,0,0,0,41,2,114,26,0,0,0,114,31,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,35,0,0,0,102,0,0,0,115,22,0,0,0,0, + 0,114,35,0,0,0,103,0,0,0,115,22,0,0,0,0, 1,8,1,8,1,10,1,8,1,14,1,14,1,10,1,6, 1,6,1,14,1,122,19,95,77,111,100,117,108,101,76,111, 99,107,46,114,101,108,101,97,115,101,99,1,0,0,0,0, @@ -199,13 +199,13 @@ const unsigned char _Py_M__importlib[] = { 123,125,41,3,218,6,102,111,114,109,97,116,114,15,0,0, 0,218,2,105,100,41,1,114,26,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,218,8,95,95,114, - 101,112,114,95,95,115,0,0,0,115,2,0,0,0,0,1, + 101,112,114,95,95,116,0,0,0,115,2,0,0,0,0,1, 122,20,95,77,111,100,117,108,101,76,111,99,107,46,95,95, 114,101,112,114,95,95,78,41,9,114,1,0,0,0,114,0, 0,0,0,114,2,0,0,0,114,3,0,0,0,114,27,0, 0,0,114,32,0,0,0,114,34,0,0,0,114,35,0,0, 0,114,40,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,114,18,0,0,0,51, + 114,10,0,0,0,114,11,0,0,0,114,18,0,0,0,52, 0,0,0,115,12,0,0,0,8,4,4,2,8,8,8,12, 8,25,8,13,114,18,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,48, @@ -224,7 +224,7 @@ const unsigned char _Py_M__importlib[] = { 1,124,0,95,1,100,0,83,0,41,2,78,114,19,0,0, 0,41,2,114,15,0,0,0,114,24,0,0,0,41,2,114, 26,0,0,0,114,15,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,27,0,0,0,123,0,0, + 0,0,0,114,11,0,0,0,114,27,0,0,0,124,0,0, 0,115,4,0,0,0,0,1,6,1,122,25,95,68,117,109, 109,121,77,111,100,117,108,101,76,111,99,107,46,95,95,105, 110,105,116,95,95,99,1,0,0,0,0,0,0,0,1,0, @@ -232,7 +232,7 @@ const unsigned char _Py_M__importlib[] = { 0,4,0,106,0,100,1,55,0,2,0,95,0,100,2,83, 0,41,3,78,114,33,0,0,0,84,41,1,114,24,0,0, 0,41,1,114,26,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,114,34,0,0,0,127,0,0,0, + 0,0,114,11,0,0,0,114,34,0,0,0,128,0,0,0, 115,4,0,0,0,0,1,14,1,122,24,95,68,117,109,109, 121,77,111,100,117,108,101,76,111,99,107,46,97,99,113,117, 105,114,101,99,1,0,0,0,0,0,0,0,1,0,0,0, @@ -244,7 +244,7 @@ const unsigned char _Py_M__importlib[] = { 117,105,114,101,100,32,108,111,99,107,114,33,0,0,0,41, 2,114,24,0,0,0,114,36,0,0,0,41,1,114,26,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,35,0,0,0,131,0,0,0,115,6,0,0,0,0, + 0,114,35,0,0,0,132,0,0,0,115,6,0,0,0,0, 1,10,1,8,1,122,24,95,68,117,109,109,121,77,111,100, 117,108,101,76,111,99,107,46,114,101,108,101,97,115,101,99, 1,0,0,0,0,0,0,0,1,0,0,0,4,0,0,0, @@ -254,14 +254,14 @@ const unsigned char _Py_M__importlib[] = { 40,123,33,114,125,41,32,97,116,32,123,125,41,3,114,38, 0,0,0,114,15,0,0,0,114,39,0,0,0,41,1,114, 26,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,40,0,0,0,136,0,0,0,115,2,0,0, + 0,0,0,114,40,0,0,0,137,0,0,0,115,2,0,0, 0,0,1,122,25,95,68,117,109,109,121,77,111,100,117,108, 101,76,111,99,107,46,95,95,114,101,112,114,95,95,78,41, 8,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0, 114,3,0,0,0,114,27,0,0,0,114,34,0,0,0,114, 35,0,0,0,114,40,0,0,0,114,10,0,0,0,114,10, 0,0,0,114,10,0,0,0,114,11,0,0,0,114,41,0, - 0,0,119,0,0,0,115,10,0,0,0,8,2,4,2,8, + 0,0,120,0,0,0,115,10,0,0,0,8,2,4,2,8, 4,8,4,8,5,114,41,0,0,0,99,0,0,0,0,0, 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, 36,0,0,0,101,0,90,1,100,0,90,2,100,1,100,2, @@ -273,774 +273,768 @@ const unsigned char _Py_M__importlib[] = { 124,0,95,1,100,0,83,0,41,1,78,41,2,218,5,95, 110,97,109,101,218,5,95,108,111,99,107,41,2,114,26,0, 0,0,114,15,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,27,0,0,0,142,0,0,0,115, + 0,114,11,0,0,0,114,27,0,0,0,143,0,0,0,115, 4,0,0,0,0,1,6,1,122,27,95,77,111,100,117,108, 101,76,111,99,107,77,97,110,97,103,101,114,46,95,95,105, 110,105,116,95,95,99,1,0,0,0,0,0,0,0,1,0, - 0,0,10,0,0,0,67,0,0,0,115,42,0,0,0,122, - 16,116,0,124,0,106,1,131,1,124,0,95,2,87,0,100, - 0,116,3,106,4,131,0,1,0,88,0,124,0,106,2,106, - 5,131,0,1,0,100,0,83,0,41,1,78,41,6,218,16, + 0,0,2,0,0,0,67,0,0,0,115,26,0,0,0,116, + 0,124,0,106,1,131,1,124,0,95,2,124,0,106,2,106, + 3,131,0,1,0,100,0,83,0,41,1,78,41,4,218,16, 95,103,101,116,95,109,111,100,117,108,101,95,108,111,99,107, - 114,43,0,0,0,114,44,0,0,0,218,4,95,105,109,112, - 218,12,114,101,108,101,97,115,101,95,108,111,99,107,114,34, - 0,0,0,41,1,114,26,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,218,9,95,95,101,110,116, - 101,114,95,95,146,0,0,0,115,8,0,0,0,0,1,2, - 1,16,2,10,1,122,28,95,77,111,100,117,108,101,76,111, - 99,107,77,97,110,97,103,101,114,46,95,95,101,110,116,101, - 114,95,95,99,1,0,0,0,0,0,0,0,3,0,0,0, - 1,0,0,0,79,0,0,0,115,14,0,0,0,124,0,106, - 0,106,1,131,0,1,0,100,0,83,0,41,1,78,41,2, - 114,44,0,0,0,114,35,0,0,0,41,3,114,26,0,0, - 0,218,4,97,114,103,115,90,6,107,119,97,114,103,115,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8, - 95,95,101,120,105,116,95,95,153,0,0,0,115,2,0,0, - 0,0,1,122,27,95,77,111,100,117,108,101,76,111,99,107, - 77,97,110,97,103,101,114,46,95,95,101,120,105,116,95,95, - 78,41,6,114,1,0,0,0,114,0,0,0,0,114,2,0, - 0,0,114,27,0,0,0,114,48,0,0,0,114,50,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,42,0,0,0,140,0,0,0,115,6, - 0,0,0,8,2,8,4,8,7,114,42,0,0,0,99,1, - 0,0,0,0,0,0,0,3,0,0,0,11,0,0,0,3, - 0,0,0,115,106,0,0,0,100,1,125,1,121,14,116,0, - 136,0,25,0,131,0,125,1,87,0,110,20,4,0,116,1, - 107,10,114,38,1,0,1,0,1,0,89,0,110,2,88,0, - 124,1,100,1,107,8,114,102,116,2,100,1,107,8,114,66, - 116,3,136,0,131,1,125,1,110,8,116,4,136,0,131,1, - 125,1,135,0,102,1,100,2,100,3,132,8,125,2,116,5, - 106,6,124,1,124,2,131,2,116,0,136,0,60,0,124,1, - 83,0,41,4,122,109,71,101,116,32,111,114,32,99,114,101, + 114,43,0,0,0,114,44,0,0,0,114,34,0,0,0,41, + 1,114,26,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,9,95,95,101,110,116,101,114,95,95, + 147,0,0,0,115,4,0,0,0,0,1,12,1,122,28,95, + 77,111,100,117,108,101,76,111,99,107,77,97,110,97,103,101, + 114,46,95,95,101,110,116,101,114,95,95,99,1,0,0,0, + 0,0,0,0,3,0,0,0,1,0,0,0,79,0,0,0, + 115,14,0,0,0,124,0,106,0,106,1,131,0,1,0,100, + 0,83,0,41,1,78,41,2,114,44,0,0,0,114,35,0, + 0,0,41,3,114,26,0,0,0,218,4,97,114,103,115,90, + 6,107,119,97,114,103,115,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,218,8,95,95,101,120,105,116,95,95, + 151,0,0,0,115,2,0,0,0,0,1,122,27,95,77,111, + 100,117,108,101,76,111,99,107,77,97,110,97,103,101,114,46, + 95,95,101,120,105,116,95,95,78,41,6,114,1,0,0,0, + 114,0,0,0,0,114,2,0,0,0,114,27,0,0,0,114, + 46,0,0,0,114,48,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,42,0, + 0,0,141,0,0,0,115,6,0,0,0,8,2,8,4,8, + 4,114,42,0,0,0,99,1,0,0,0,0,0,0,0,3, + 0,0,0,17,0,0,0,3,0,0,0,115,130,0,0,0, + 116,0,106,1,131,0,1,0,122,106,121,14,116,2,136,0, + 25,0,131,0,125,1,87,0,110,24,4,0,116,3,107,10, + 114,48,1,0,1,0,1,0,100,1,125,1,89,0,110,2, + 88,0,124,1,100,1,107,8,114,112,116,4,100,1,107,8, + 114,76,116,5,136,0,131,1,125,1,110,8,116,6,136,0, + 131,1,125,1,135,0,102,1,100,2,100,3,132,8,125,2, + 116,7,106,8,124,1,124,2,131,2,116,2,136,0,60,0, + 87,0,100,1,116,0,106,9,131,0,1,0,88,0,124,1, + 83,0,41,4,122,139,71,101,116,32,111,114,32,99,114,101, 97,116,101,32,116,104,101,32,109,111,100,117,108,101,32,108, 111,99,107,32,102,111,114,32,97,32,103,105,118,101,110,32, 109,111,100,117,108,101,32,110,97,109,101,46,10,10,32,32, - 32,32,83,104,111,117,108,100,32,111,110,108,121,32,98,101, - 32,99,97,108,108,101,100,32,119,105,116,104,32,116,104,101, - 32,105,109,112,111,114,116,32,108,111,99,107,32,116,97,107, - 101,110,46,78,99,1,0,0,0,0,0,0,0,1,0,0, - 0,2,0,0,0,19,0,0,0,115,10,0,0,0,116,0, - 136,0,61,0,100,0,83,0,41,1,78,41,1,218,13,95, - 109,111,100,117,108,101,95,108,111,99,107,115,41,1,218,1, - 95,41,1,114,15,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,2,99,98,173,0,0,0,115,2,0,0,0,0, - 1,122,28,95,103,101,116,95,109,111,100,117,108,101,95,108, - 111,99,107,46,60,108,111,99,97,108,115,62,46,99,98,41, - 7,114,51,0,0,0,218,8,75,101,121,69,114,114,111,114, - 114,20,0,0,0,114,41,0,0,0,114,18,0,0,0,218, - 8,95,119,101,97,107,114,101,102,90,3,114,101,102,41,3, - 114,15,0,0,0,114,21,0,0,0,114,53,0,0,0,114, + 32,32,65,99,113,117,105,114,101,47,114,101,108,101,97,115, + 101,32,105,110,116,101,114,110,97,108,108,121,32,116,104,101, + 32,103,108,111,98,97,108,32,105,109,112,111,114,116,32,108, + 111,99,107,32,116,111,32,112,114,111,116,101,99,116,10,32, + 32,32,32,95,109,111,100,117,108,101,95,108,111,99,107,115, + 46,78,99,1,0,0,0,0,0,0,0,1,0,0,0,2, + 0,0,0,19,0,0,0,115,10,0,0,0,116,0,136,0, + 61,0,100,0,83,0,41,1,78,41,1,218,13,95,109,111, + 100,117,108,101,95,108,111,99,107,115,41,1,218,1,95,41, + 1,114,15,0,0,0,114,10,0,0,0,114,11,0,0,0, + 218,2,99,98,175,0,0,0,115,2,0,0,0,0,1,122, + 28,95,103,101,116,95,109,111,100,117,108,101,95,108,111,99, + 107,46,60,108,111,99,97,108,115,62,46,99,98,41,10,218, + 4,95,105,109,112,218,12,97,99,113,117,105,114,101,95,108, + 111,99,107,114,49,0,0,0,218,8,75,101,121,69,114,114, + 111,114,114,20,0,0,0,114,41,0,0,0,114,18,0,0, + 0,218,8,95,119,101,97,107,114,101,102,90,3,114,101,102, + 218,12,114,101,108,101,97,115,101,95,108,111,99,107,41,3, + 114,15,0,0,0,114,21,0,0,0,114,51,0,0,0,114, 10,0,0,0,41,1,114,15,0,0,0,114,11,0,0,0, - 114,45,0,0,0,159,0,0,0,115,24,0,0,0,0,4, - 4,1,2,1,14,1,14,1,6,1,8,1,8,1,10,2, - 8,1,12,2,16,1,114,45,0,0,0,99,1,0,0,0, - 0,0,0,0,2,0,0,0,11,0,0,0,67,0,0,0, - 115,62,0,0,0,116,0,124,0,131,1,125,1,116,1,106, - 2,131,0,1,0,121,12,124,1,106,3,131,0,1,0,87, - 0,110,20,4,0,116,4,107,10,114,48,1,0,1,0,1, - 0,89,0,110,10,88,0,124,1,106,5,131,0,1,0,100, - 1,83,0,41,2,97,21,1,0,0,82,101,108,101,97,115, - 101,32,116,104,101,32,103,108,111,98,97,108,32,105,109,112, - 111,114,116,32,108,111,99,107,44,32,97,110,100,32,97,99, - 113,117,105,114,101,115,32,116,104,101,110,32,114,101,108,101, - 97,115,101,32,116,104,101,10,32,32,32,32,109,111,100,117, - 108,101,32,108,111,99,107,32,102,111,114,32,97,32,103,105, - 118,101,110,32,109,111,100,117,108,101,32,110,97,109,101,46, - 10,32,32,32,32,84,104,105,115,32,105,115,32,117,115,101, - 100,32,116,111,32,101,110,115,117,114,101,32,97,32,109,111, - 100,117,108,101,32,105,115,32,99,111,109,112,108,101,116,101, - 108,121,32,105,110,105,116,105,97,108,105,122,101,100,44,32, - 105,110,32,116,104,101,10,32,32,32,32,101,118,101,110,116, - 32,105,116,32,105,115,32,98,101,105,110,103,32,105,109,112, - 111,114,116,101,100,32,98,121,32,97,110,111,116,104,101,114, - 32,116,104,114,101,97,100,46,10,10,32,32,32,32,83,104, - 111,117,108,100,32,111,110,108,121,32,98,101,32,99,97,108, - 108,101,100,32,119,105,116,104,32,116,104,101,32,105,109,112, - 111,114,116,32,108,111,99,107,32,116,97,107,101,110,46,78, - 41,6,114,45,0,0,0,114,46,0,0,0,114,47,0,0, - 0,114,34,0,0,0,114,17,0,0,0,114,35,0,0,0, - 41,2,114,15,0,0,0,114,21,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,19,95,108,111, - 99,107,95,117,110,108,111,99,107,95,109,111,100,117,108,101, - 178,0,0,0,115,14,0,0,0,0,7,8,1,8,1,2, - 1,12,1,14,3,6,2,114,56,0,0,0,99,1,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,79,0,0, - 0,115,10,0,0,0,124,0,124,1,124,2,142,1,83,0, - 41,1,97,46,1,0,0,114,101,109,111,118,101,95,105,109, - 112,111,114,116,108,105,98,95,102,114,97,109,101,115,32,105, - 110,32,105,109,112,111,114,116,46,99,32,119,105,108,108,32, - 97,108,119,97,121,115,32,114,101,109,111,118,101,32,115,101, - 113,117,101,110,99,101,115,10,32,32,32,32,111,102,32,105, - 109,112,111,114,116,108,105,98,32,102,114,97,109,101,115,32, - 116,104,97,116,32,101,110,100,32,119,105,116,104,32,97,32, - 99,97,108,108,32,116,111,32,116,104,105,115,32,102,117,110, - 99,116,105,111,110,10,10,32,32,32,32,85,115,101,32,105, - 116,32,105,110,115,116,101,97,100,32,111,102,32,97,32,110, - 111,114,109,97,108,32,99,97,108,108,32,105,110,32,112,108, - 97,99,101,115,32,119,104,101,114,101,32,105,110,99,108,117, - 100,105,110,103,32,116,104,101,32,105,109,112,111,114,116,108, - 105,98,10,32,32,32,32,102,114,97,109,101,115,32,105,110, - 116,114,111,100,117,99,101,115,32,117,110,119,97,110,116,101, - 100,32,110,111,105,115,101,32,105,110,116,111,32,116,104,101, - 32,116,114,97,99,101,98,97,99,107,32,40,101,46,103,46, - 32,119,104,101,110,32,101,120,101,99,117,116,105,110,103,10, - 32,32,32,32,109,111,100,117,108,101,32,99,111,100,101,41, - 10,32,32,32,32,114,10,0,0,0,41,3,218,1,102,114, - 49,0,0,0,90,4,107,119,100,115,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,218,25,95,99,97,108,108, - 95,119,105,116,104,95,102,114,97,109,101,115,95,114,101,109, - 111,118,101,100,197,0,0,0,115,2,0,0,0,0,8,114, - 58,0,0,0,114,33,0,0,0,41,1,218,9,118,101,114, - 98,111,115,105,116,121,99,1,0,0,0,1,0,0,0,3, - 0,0,0,4,0,0,0,71,0,0,0,115,54,0,0,0, - 116,0,106,1,106,2,124,1,107,5,114,50,124,0,106,3, - 100,6,131,1,115,30,100,3,124,0,23,0,125,0,116,4, - 124,0,106,5,124,2,142,0,116,0,106,6,100,4,141,2, - 1,0,100,5,83,0,41,7,122,61,80,114,105,110,116,32, - 116,104,101,32,109,101,115,115,97,103,101,32,116,111,32,115, - 116,100,101,114,114,32,105,102,32,45,118,47,80,89,84,72, - 79,78,86,69,82,66,79,83,69,32,105,115,32,116,117,114, - 110,101,100,32,111,110,46,250,1,35,250,7,105,109,112,111, - 114,116,32,122,2,35,32,41,1,90,4,102,105,108,101,78, - 41,2,114,60,0,0,0,114,61,0,0,0,41,7,114,14, - 0,0,0,218,5,102,108,97,103,115,218,7,118,101,114,98, - 111,115,101,218,10,115,116,97,114,116,115,119,105,116,104,218, - 5,112,114,105,110,116,114,38,0,0,0,218,6,115,116,100, - 101,114,114,41,3,218,7,109,101,115,115,97,103,101,114,59, - 0,0,0,114,49,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,16,95,118,101,114,98,111,115, - 101,95,109,101,115,115,97,103,101,208,0,0,0,115,8,0, - 0,0,0,2,12,1,10,1,8,1,114,68,0,0,0,99, - 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, - 3,0,0,0,115,26,0,0,0,135,0,102,1,100,1,100, - 2,132,8,125,1,116,0,124,1,136,0,131,2,1,0,124, - 1,83,0,41,3,122,49,68,101,99,111,114,97,116,111,114, - 32,116,111,32,118,101,114,105,102,121,32,116,104,101,32,110, - 97,109,101,100,32,109,111,100,117,108,101,32,105,115,32,98, - 117,105,108,116,45,105,110,46,99,2,0,0,0,0,0,0, - 0,2,0,0,0,4,0,0,0,19,0,0,0,115,38,0, - 0,0,124,1,116,0,106,1,107,7,114,28,116,2,100,1, - 106,3,124,1,131,1,124,1,100,2,141,2,130,1,136,0, - 124,0,124,1,131,2,83,0,41,3,78,122,29,123,33,114, - 125,32,105,115,32,110,111,116,32,97,32,98,117,105,108,116, - 45,105,110,32,109,111,100,117,108,101,41,1,114,15,0,0, - 0,41,4,114,14,0,0,0,218,20,98,117,105,108,116,105, - 110,95,109,111,100,117,108,101,95,110,97,109,101,115,218,11, - 73,109,112,111,114,116,69,114,114,111,114,114,38,0,0,0, - 41,2,114,26,0,0,0,218,8,102,117,108,108,110,97,109, - 101,41,1,218,3,102,120,110,114,10,0,0,0,114,11,0, - 0,0,218,25,95,114,101,113,117,105,114,101,115,95,98,117, - 105,108,116,105,110,95,119,114,97,112,112,101,114,218,0,0, - 0,115,8,0,0,0,0,1,10,1,10,1,8,1,122,52, + 114,45,0,0,0,157,0,0,0,115,28,0,0,0,0,6, + 8,1,2,1,2,1,14,1,14,1,10,2,8,1,8,1, + 10,2,8,1,12,2,20,2,10,2,114,45,0,0,0,99, + 1,0,0,0,0,0,0,0,2,0,0,0,11,0,0,0, + 67,0,0,0,115,54,0,0,0,116,0,124,0,131,1,125, + 1,121,12,124,1,106,1,131,0,1,0,87,0,110,20,4, + 0,116,2,107,10,114,40,1,0,1,0,1,0,89,0,110, + 10,88,0,124,1,106,3,131,0,1,0,100,1,83,0,41, + 2,122,189,65,99,113,117,105,114,101,115,32,116,104,101,110, + 32,114,101,108,101,97,115,101,115,32,116,104,101,32,109,111, + 100,117,108,101,32,108,111,99,107,32,102,111,114,32,97,32, + 103,105,118,101,110,32,109,111,100,117,108,101,32,110,97,109, + 101,46,10,10,32,32,32,32,84,104,105,115,32,105,115,32, + 117,115,101,100,32,116,111,32,101,110,115,117,114,101,32,97, + 32,109,111,100,117,108,101,32,105,115,32,99,111,109,112,108, + 101,116,101,108,121,32,105,110,105,116,105,97,108,105,122,101, + 100,44,32,105,110,32,116,104,101,10,32,32,32,32,101,118, + 101,110,116,32,105,116,32,105,115,32,98,101,105,110,103,32, + 105,109,112,111,114,116,101,100,32,98,121,32,97,110,111,116, + 104,101,114,32,116,104,114,101,97,100,46,10,32,32,32,32, + 78,41,4,114,45,0,0,0,114,34,0,0,0,114,17,0, + 0,0,114,35,0,0,0,41,2,114,15,0,0,0,114,21, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, + 0,0,218,19,95,108,111,99,107,95,117,110,108,111,99,107, + 95,109,111,100,117,108,101,184,0,0,0,115,12,0,0,0, + 0,6,8,1,2,1,12,1,14,3,6,2,114,57,0,0, + 0,99,1,0,0,0,0,0,0,0,3,0,0,0,3,0, + 0,0,79,0,0,0,115,10,0,0,0,124,0,124,1,124, + 2,142,1,83,0,41,1,97,46,1,0,0,114,101,109,111, + 118,101,95,105,109,112,111,114,116,108,105,98,95,102,114,97, + 109,101,115,32,105,110,32,105,109,112,111,114,116,46,99,32, + 119,105,108,108,32,97,108,119,97,121,115,32,114,101,109,111, + 118,101,32,115,101,113,117,101,110,99,101,115,10,32,32,32, + 32,111,102,32,105,109,112,111,114,116,108,105,98,32,102,114, + 97,109,101,115,32,116,104,97,116,32,101,110,100,32,119,105, + 116,104,32,97,32,99,97,108,108,32,116,111,32,116,104,105, + 115,32,102,117,110,99,116,105,111,110,10,10,32,32,32,32, + 85,115,101,32,105,116,32,105,110,115,116,101,97,100,32,111, + 102,32,97,32,110,111,114,109,97,108,32,99,97,108,108,32, + 105,110,32,112,108,97,99,101,115,32,119,104,101,114,101,32, + 105,110,99,108,117,100,105,110,103,32,116,104,101,32,105,109, + 112,111,114,116,108,105,98,10,32,32,32,32,102,114,97,109, + 101,115,32,105,110,116,114,111,100,117,99,101,115,32,117,110, + 119,97,110,116,101,100,32,110,111,105,115,101,32,105,110,116, + 111,32,116,104,101,32,116,114,97,99,101,98,97,99,107,32, + 40,101,46,103,46,32,119,104,101,110,32,101,120,101,99,117, + 116,105,110,103,10,32,32,32,32,109,111,100,117,108,101,32, + 99,111,100,101,41,10,32,32,32,32,114,10,0,0,0,41, + 3,218,1,102,114,47,0,0,0,90,4,107,119,100,115,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,25, + 95,99,97,108,108,95,119,105,116,104,95,102,114,97,109,101, + 115,95,114,101,109,111,118,101,100,201,0,0,0,115,2,0, + 0,0,0,8,114,59,0,0,0,114,33,0,0,0,41,1, + 218,9,118,101,114,98,111,115,105,116,121,99,1,0,0,0, + 1,0,0,0,3,0,0,0,4,0,0,0,71,0,0,0, + 115,54,0,0,0,116,0,106,1,106,2,124,1,107,5,114, + 50,124,0,106,3,100,6,131,1,115,30,100,3,124,0,23, + 0,125,0,116,4,124,0,106,5,124,2,142,0,116,0,106, + 6,100,4,141,2,1,0,100,5,83,0,41,7,122,61,80, + 114,105,110,116,32,116,104,101,32,109,101,115,115,97,103,101, + 32,116,111,32,115,116,100,101,114,114,32,105,102,32,45,118, + 47,80,89,84,72,79,78,86,69,82,66,79,83,69,32,105, + 115,32,116,117,114,110,101,100,32,111,110,46,250,1,35,250, + 7,105,109,112,111,114,116,32,122,2,35,32,41,1,90,4, + 102,105,108,101,78,41,2,114,61,0,0,0,114,62,0,0, + 0,41,7,114,14,0,0,0,218,5,102,108,97,103,115,218, + 7,118,101,114,98,111,115,101,218,10,115,116,97,114,116,115, + 119,105,116,104,218,5,112,114,105,110,116,114,38,0,0,0, + 218,6,115,116,100,101,114,114,41,3,218,7,109,101,115,115, + 97,103,101,114,60,0,0,0,114,47,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,218,16,95,118, + 101,114,98,111,115,101,95,109,101,115,115,97,103,101,212,0, + 0,0,115,8,0,0,0,0,2,12,1,10,1,8,1,114, + 69,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,3,0,0,0,115,26,0,0,0,135,0, + 102,1,100,1,100,2,132,8,125,1,116,0,124,1,136,0, + 131,2,1,0,124,1,83,0,41,3,122,49,68,101,99,111, + 114,97,116,111,114,32,116,111,32,118,101,114,105,102,121,32, + 116,104,101,32,110,97,109,101,100,32,109,111,100,117,108,101, + 32,105,115,32,98,117,105,108,116,45,105,110,46,99,2,0, + 0,0,0,0,0,0,2,0,0,0,4,0,0,0,19,0, + 0,0,115,38,0,0,0,124,1,116,0,106,1,107,7,114, + 28,116,2,100,1,106,3,124,1,131,1,124,1,100,2,141, + 2,130,1,136,0,124,0,124,1,131,2,83,0,41,3,78, + 122,29,123,33,114,125,32,105,115,32,110,111,116,32,97,32, + 98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,41, + 1,114,15,0,0,0,41,4,114,14,0,0,0,218,20,98, + 117,105,108,116,105,110,95,109,111,100,117,108,101,95,110,97, + 109,101,115,218,11,73,109,112,111,114,116,69,114,114,111,114, + 114,38,0,0,0,41,2,114,26,0,0,0,218,8,102,117, + 108,108,110,97,109,101,41,1,218,3,102,120,110,114,10,0, + 0,0,114,11,0,0,0,218,25,95,114,101,113,117,105,114, + 101,115,95,98,117,105,108,116,105,110,95,119,114,97,112,112, + 101,114,222,0,0,0,115,8,0,0,0,0,1,10,1,10, + 1,8,1,122,52,95,114,101,113,117,105,114,101,115,95,98, + 117,105,108,116,105,110,46,60,108,111,99,97,108,115,62,46, 95,114,101,113,117,105,114,101,115,95,98,117,105,108,116,105, - 110,46,60,108,111,99,97,108,115,62,46,95,114,101,113,117, - 105,114,101,115,95,98,117,105,108,116,105,110,95,119,114,97, - 112,112,101,114,41,1,114,12,0,0,0,41,2,114,72,0, - 0,0,114,73,0,0,0,114,10,0,0,0,41,1,114,72, - 0,0,0,114,11,0,0,0,218,17,95,114,101,113,117,105, - 114,101,115,95,98,117,105,108,116,105,110,216,0,0,0,115, - 6,0,0,0,0,2,12,5,10,1,114,74,0,0,0,99, - 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, - 3,0,0,0,115,26,0,0,0,135,0,102,1,100,1,100, - 2,132,8,125,1,116,0,124,1,136,0,131,2,1,0,124, - 1,83,0,41,3,122,47,68,101,99,111,114,97,116,111,114, - 32,116,111,32,118,101,114,105,102,121,32,116,104,101,32,110, - 97,109,101,100,32,109,111,100,117,108,101,32,105,115,32,102, - 114,111,122,101,110,46,99,2,0,0,0,0,0,0,0,2, - 0,0,0,4,0,0,0,19,0,0,0,115,38,0,0,0, - 116,0,106,1,124,1,131,1,115,28,116,2,100,1,106,3, - 124,1,131,1,124,1,100,2,141,2,130,1,136,0,124,0, - 124,1,131,2,83,0,41,3,78,122,27,123,33,114,125,32, - 105,115,32,110,111,116,32,97,32,102,114,111,122,101,110,32, - 109,111,100,117,108,101,41,1,114,15,0,0,0,41,4,114, - 46,0,0,0,218,9,105,115,95,102,114,111,122,101,110,114, - 70,0,0,0,114,38,0,0,0,41,2,114,26,0,0,0, - 114,71,0,0,0,41,1,114,72,0,0,0,114,10,0,0, - 0,114,11,0,0,0,218,24,95,114,101,113,117,105,114,101, - 115,95,102,114,111,122,101,110,95,119,114,97,112,112,101,114, - 229,0,0,0,115,8,0,0,0,0,1,10,1,10,1,8, - 1,122,50,95,114,101,113,117,105,114,101,115,95,102,114,111, - 122,101,110,46,60,108,111,99,97,108,115,62,46,95,114,101, + 110,95,119,114,97,112,112,101,114,41,1,114,12,0,0,0, + 41,2,114,73,0,0,0,114,74,0,0,0,114,10,0,0, + 0,41,1,114,73,0,0,0,114,11,0,0,0,218,17,95, + 114,101,113,117,105,114,101,115,95,98,117,105,108,116,105,110, + 220,0,0,0,115,6,0,0,0,0,2,12,5,10,1,114, + 75,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,3,0,0,0,115,26,0,0,0,135,0, + 102,1,100,1,100,2,132,8,125,1,116,0,124,1,136,0, + 131,2,1,0,124,1,83,0,41,3,122,47,68,101,99,111, + 114,97,116,111,114,32,116,111,32,118,101,114,105,102,121,32, + 116,104,101,32,110,97,109,101,100,32,109,111,100,117,108,101, + 32,105,115,32,102,114,111,122,101,110,46,99,2,0,0,0, + 0,0,0,0,2,0,0,0,4,0,0,0,19,0,0,0, + 115,38,0,0,0,116,0,106,1,124,1,131,1,115,28,116, + 2,100,1,106,3,124,1,131,1,124,1,100,2,141,2,130, + 1,136,0,124,0,124,1,131,2,83,0,41,3,78,122,27, + 123,33,114,125,32,105,115,32,110,111,116,32,97,32,102,114, + 111,122,101,110,32,109,111,100,117,108,101,41,1,114,15,0, + 0,0,41,4,114,52,0,0,0,218,9,105,115,95,102,114, + 111,122,101,110,114,71,0,0,0,114,38,0,0,0,41,2, + 114,26,0,0,0,114,72,0,0,0,41,1,114,73,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,24,95,114,101, 113,117,105,114,101,115,95,102,114,111,122,101,110,95,119,114, - 97,112,112,101,114,41,1,114,12,0,0,0,41,2,114,72, - 0,0,0,114,76,0,0,0,114,10,0,0,0,41,1,114, - 72,0,0,0,114,11,0,0,0,218,16,95,114,101,113,117, - 105,114,101,115,95,102,114,111,122,101,110,227,0,0,0,115, - 6,0,0,0,0,2,12,5,10,1,114,77,0,0,0,99, - 2,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, - 67,0,0,0,115,62,0,0,0,116,0,124,1,124,0,131, - 2,125,2,124,1,116,1,106,2,107,6,114,50,116,1,106, - 2,124,1,25,0,125,3,116,3,124,2,124,3,131,2,1, - 0,116,1,106,2,124,1,25,0,83,0,116,4,124,2,131, - 1,83,0,100,1,83,0,41,2,122,128,76,111,97,100,32, - 116,104,101,32,115,112,101,99,105,102,105,101,100,32,109,111, - 100,117,108,101,32,105,110,116,111,32,115,121,115,46,109,111, - 100,117,108,101,115,32,97,110,100,32,114,101,116,117,114,110, - 32,105,116,46,10,10,32,32,32,32,84,104,105,115,32,109, - 101,116,104,111,100,32,105,115,32,100,101,112,114,101,99,97, - 116,101,100,46,32,32,85,115,101,32,108,111,97,100,101,114, - 46,101,120,101,99,95,109,111,100,117,108,101,32,105,110,115, - 116,101,97,100,46,10,10,32,32,32,32,78,41,5,218,16, - 115,112,101,99,95,102,114,111,109,95,108,111,97,100,101,114, - 114,14,0,0,0,218,7,109,111,100,117,108,101,115,218,5, - 95,101,120,101,99,218,5,95,108,111,97,100,41,4,114,26, - 0,0,0,114,71,0,0,0,218,4,115,112,101,99,218,6, - 109,111,100,117,108,101,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,17,95,108,111,97,100,95,109,111,100, - 117,108,101,95,115,104,105,109,239,0,0,0,115,12,0,0, - 0,0,6,10,1,10,1,10,1,10,1,10,2,114,84,0, - 0,0,99,1,0,0,0,0,0,0,0,5,0,0,0,35, - 0,0,0,67,0,0,0,115,216,0,0,0,116,0,124,0, - 100,1,100,0,131,3,125,1,116,1,124,1,100,2,131,2, - 114,54,121,10,124,1,106,2,124,0,131,1,83,0,4,0, - 116,3,107,10,114,52,1,0,1,0,1,0,89,0,110,2, - 88,0,121,10,124,0,106,4,125,2,87,0,110,20,4,0, - 116,5,107,10,114,84,1,0,1,0,1,0,89,0,110,18, - 88,0,124,2,100,0,107,9,114,102,116,6,124,2,131,1, - 83,0,121,10,124,0,106,7,125,3,87,0,110,24,4,0, - 116,5,107,10,114,136,1,0,1,0,1,0,100,3,125,3, - 89,0,110,2,88,0,121,10,124,0,106,8,125,4,87,0, - 110,50,4,0,116,5,107,10,114,198,1,0,1,0,1,0, - 124,1,100,0,107,8,114,182,100,4,106,9,124,3,131,1, - 83,0,100,5,106,9,124,3,124,1,131,2,83,0,89,0, - 110,14,88,0,100,6,106,9,124,3,124,4,131,2,83,0, - 100,0,83,0,41,7,78,218,10,95,95,108,111,97,100,101, - 114,95,95,218,11,109,111,100,117,108,101,95,114,101,112,114, - 250,1,63,122,13,60,109,111,100,117,108,101,32,123,33,114, - 125,62,122,20,60,109,111,100,117,108,101,32,123,33,114,125, - 32,40,123,33,114,125,41,62,122,23,60,109,111,100,117,108, - 101,32,123,33,114,125,32,102,114,111,109,32,123,33,114,125, - 62,41,10,114,6,0,0,0,114,4,0,0,0,114,86,0, - 0,0,218,9,69,120,99,101,112,116,105,111,110,218,8,95, - 95,115,112,101,99,95,95,218,14,65,116,116,114,105,98,117, - 116,101,69,114,114,111,114,218,22,95,109,111,100,117,108,101, - 95,114,101,112,114,95,102,114,111,109,95,115,112,101,99,114, - 1,0,0,0,218,8,95,95,102,105,108,101,95,95,114,38, - 0,0,0,41,5,114,83,0,0,0,218,6,108,111,97,100, - 101,114,114,82,0,0,0,114,15,0,0,0,218,8,102,105, - 108,101,110,97,109,101,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,12,95,109,111,100,117,108,101,95,114, - 101,112,114,255,0,0,0,115,46,0,0,0,0,2,12,1, - 10,4,2,1,10,1,14,1,6,1,2,1,10,1,14,1, - 6,2,8,1,8,4,2,1,10,1,14,1,10,1,2,1, - 10,1,14,1,8,1,10,2,18,2,114,95,0,0,0,99, - 0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0, - 64,0,0,0,115,36,0,0,0,101,0,90,1,100,0,90, - 2,100,1,100,2,132,0,90,3,100,3,100,4,132,0,90, - 4,100,5,100,6,132,0,90,5,100,7,83,0,41,8,218, - 17,95,105,110,115,116,97,108,108,101,100,95,115,97,102,101, - 108,121,99,2,0,0,0,0,0,0,0,2,0,0,0,2, - 0,0,0,67,0,0,0,115,18,0,0,0,124,1,124,0, - 95,0,124,1,106,1,124,0,95,2,100,0,83,0,41,1, - 78,41,3,218,7,95,109,111,100,117,108,101,114,89,0,0, - 0,218,5,95,115,112,101,99,41,2,114,26,0,0,0,114, - 83,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,27,0,0,0,37,1,0,0,115,4,0,0, - 0,0,1,6,1,122,26,95,105,110,115,116,97,108,108,101, - 100,95,115,97,102,101,108,121,46,95,95,105,110,105,116,95, - 95,99,1,0,0,0,0,0,0,0,1,0,0,0,3,0, - 0,0,67,0,0,0,115,28,0,0,0,100,1,124,0,106, - 0,95,1,124,0,106,2,116,3,106,4,124,0,106,0,106, - 5,60,0,100,0,83,0,41,2,78,84,41,6,114,98,0, - 0,0,218,13,95,105,110,105,116,105,97,108,105,122,105,110, - 103,114,97,0,0,0,114,14,0,0,0,114,79,0,0,0, - 114,15,0,0,0,41,1,114,26,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,48,0,0,0, - 41,1,0,0,115,4,0,0,0,0,4,8,1,122,27,95, - 105,110,115,116,97,108,108,101,100,95,115,97,102,101,108,121, - 46,95,95,101,110,116,101,114,95,95,99,1,0,0,0,0, - 0,0,0,3,0,0,0,17,0,0,0,71,0,0,0,115, - 98,0,0,0,122,82,124,0,106,0,125,2,116,1,100,1, - 100,2,132,0,124,1,68,0,131,1,131,1,114,64,121,14, - 116,2,106,3,124,2,106,4,61,0,87,0,113,80,4,0, - 116,5,107,10,114,60,1,0,1,0,1,0,89,0,113,80, - 88,0,110,16,116,6,100,3,124,2,106,4,124,2,106,7, - 131,3,1,0,87,0,100,0,100,4,124,0,106,0,95,8, - 88,0,100,0,83,0,41,5,78,99,1,0,0,0,0,0, - 0,0,2,0,0,0,3,0,0,0,115,0,0,0,115,22, - 0,0,0,124,0,93,14,125,1,124,1,100,0,107,9,86, - 0,1,0,113,2,100,0,83,0,41,1,78,114,10,0,0, - 0,41,2,90,2,46,48,90,3,97,114,103,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,250,9,60,103,101, - 110,101,120,112,114,62,51,1,0,0,115,2,0,0,0,4, - 0,122,45,95,105,110,115,116,97,108,108,101,100,95,115,97, - 102,101,108,121,46,95,95,101,120,105,116,95,95,46,60,108, - 111,99,97,108,115,62,46,60,103,101,110,101,120,112,114,62, - 122,18,105,109,112,111,114,116,32,123,33,114,125,32,35,32, - 123,33,114,125,70,41,9,114,98,0,0,0,218,3,97,110, - 121,114,14,0,0,0,114,79,0,0,0,114,15,0,0,0, - 114,54,0,0,0,114,68,0,0,0,114,93,0,0,0,114, - 99,0,0,0,41,3,114,26,0,0,0,114,49,0,0,0, - 114,82,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,114,50,0,0,0,48,1,0,0,115,18,0, - 0,0,0,1,2,1,6,1,18,1,2,1,14,1,14,1, - 8,2,20,2,122,26,95,105,110,115,116,97,108,108,101,100, - 95,115,97,102,101,108,121,46,95,95,101,120,105,116,95,95, - 78,41,6,114,1,0,0,0,114,0,0,0,0,114,2,0, - 0,0,114,27,0,0,0,114,48,0,0,0,114,50,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,96,0,0,0,35,1,0,0,115,6, - 0,0,0,8,2,8,4,8,7,114,96,0,0,0,99,0, - 0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,64, - 0,0,0,115,114,0,0,0,101,0,90,1,100,0,90,2, - 100,1,90,3,100,2,100,2,100,2,100,3,156,3,100,4, - 100,5,132,2,90,4,100,6,100,7,132,0,90,5,100,8, - 100,9,132,0,90,6,101,7,100,10,100,11,132,0,131,1, - 90,8,101,8,106,9,100,12,100,11,132,0,131,1,90,8, - 101,7,100,13,100,14,132,0,131,1,90,10,101,7,100,15, - 100,16,132,0,131,1,90,11,101,11,106,9,100,17,100,16, - 132,0,131,1,90,11,100,2,83,0,41,18,218,10,77,111, - 100,117,108,101,83,112,101,99,97,208,5,0,0,84,104,101, - 32,115,112,101,99,105,102,105,99,97,116,105,111,110,32,102, - 111,114,32,97,32,109,111,100,117,108,101,44,32,117,115,101, - 100,32,102,111,114,32,108,111,97,100,105,110,103,46,10,10, - 32,32,32,32,65,32,109,111,100,117,108,101,39,115,32,115, - 112,101,99,32,105,115,32,116,104,101,32,115,111,117,114,99, - 101,32,102,111,114,32,105,110,102,111,114,109,97,116,105,111, - 110,32,97,98,111,117,116,32,116,104,101,32,109,111,100,117, - 108,101,46,32,32,70,111,114,10,32,32,32,32,100,97,116, - 97,32,97,115,115,111,99,105,97,116,101,100,32,119,105,116, - 104,32,116,104,101,32,109,111,100,117,108,101,44,32,105,110, - 99,108,117,100,105,110,103,32,115,111,117,114,99,101,44,32, - 117,115,101,32,116,104,101,32,115,112,101,99,39,115,10,32, - 32,32,32,108,111,97,100,101,114,46,10,10,32,32,32,32, - 96,110,97,109,101,96,32,105,115,32,116,104,101,32,97,98, - 115,111,108,117,116,101,32,110,97,109,101,32,111,102,32,116, - 104,101,32,109,111,100,117,108,101,46,32,32,96,108,111,97, - 100,101,114,96,32,105,115,32,116,104,101,32,108,111,97,100, - 101,114,10,32,32,32,32,116,111,32,117,115,101,32,119,104, - 101,110,32,108,111,97,100,105,110,103,32,116,104,101,32,109, - 111,100,117,108,101,46,32,32,96,112,97,114,101,110,116,96, - 32,105,115,32,116,104,101,32,110,97,109,101,32,111,102,32, - 116,104,101,10,32,32,32,32,112,97,99,107,97,103,101,32, - 116,104,101,32,109,111,100,117,108,101,32,105,115,32,105,110, - 46,32,32,84,104,101,32,112,97,114,101,110,116,32,105,115, - 32,100,101,114,105,118,101,100,32,102,114,111,109,32,116,104, - 101,32,110,97,109,101,46,10,10,32,32,32,32,96,105,115, - 95,112,97,99,107,97,103,101,96,32,100,101,116,101,114,109, - 105,110,101,115,32,105,102,32,116,104,101,32,109,111,100,117, - 108,101,32,105,115,32,99,111,110,115,105,100,101,114,101,100, - 32,97,32,112,97,99,107,97,103,101,32,111,114,10,32,32, - 32,32,110,111,116,46,32,32,79,110,32,109,111,100,117,108, - 101,115,32,116,104,105,115,32,105,115,32,114,101,102,108,101, - 99,116,101,100,32,98,121,32,116,104,101,32,96,95,95,112, - 97,116,104,95,95,96,32,97,116,116,114,105,98,117,116,101, - 46,10,10,32,32,32,32,96,111,114,105,103,105,110,96,32, - 105,115,32,116,104,101,32,115,112,101,99,105,102,105,99,32, - 108,111,99,97,116,105,111,110,32,117,115,101,100,32,98,121, - 32,116,104,101,32,108,111,97,100,101,114,32,102,114,111,109, - 32,119,104,105,99,104,32,116,111,10,32,32,32,32,108,111, - 97,100,32,116,104,101,32,109,111,100,117,108,101,44,32,105, - 102,32,116,104,97,116,32,105,110,102,111,114,109,97,116,105, - 111,110,32,105,115,32,97,118,97,105,108,97,98,108,101,46, - 32,32,87,104,101,110,32,102,105,108,101,110,97,109,101,32, - 105,115,10,32,32,32,32,115,101,116,44,32,111,114,105,103, - 105,110,32,119,105,108,108,32,109,97,116,99,104,46,10,10, - 32,32,32,32,96,104,97,115,95,108,111,99,97,116,105,111, - 110,96,32,105,110,100,105,99,97,116,101,115,32,116,104,97, - 116,32,97,32,115,112,101,99,39,115,32,34,111,114,105,103, - 105,110,34,32,114,101,102,108,101,99,116,115,32,97,32,108, - 111,99,97,116,105,111,110,46,10,32,32,32,32,87,104,101, - 110,32,116,104,105,115,32,105,115,32,84,114,117,101,44,32, - 96,95,95,102,105,108,101,95,95,96,32,97,116,116,114,105, - 98,117,116,101,32,111,102,32,116,104,101,32,109,111,100,117, - 108,101,32,105,115,32,115,101,116,46,10,10,32,32,32,32, - 96,99,97,99,104,101,100,96,32,105,115,32,116,104,101,32, - 108,111,99,97,116,105,111,110,32,111,102,32,116,104,101,32, - 99,97,99,104,101,100,32,98,121,116,101,99,111,100,101,32, - 102,105,108,101,44,32,105,102,32,97,110,121,46,32,32,73, - 116,10,32,32,32,32,99,111,114,114,101,115,112,111,110,100, - 115,32,116,111,32,116,104,101,32,96,95,95,99,97,99,104, - 101,100,95,95,96,32,97,116,116,114,105,98,117,116,101,46, - 10,10,32,32,32,32,96,115,117,98,109,111,100,117,108,101, - 95,115,101,97,114,99,104,95,108,111,99,97,116,105,111,110, - 115,96,32,105,115,32,116,104,101,32,115,101,113,117,101,110, - 99,101,32,111,102,32,112,97,116,104,32,101,110,116,114,105, - 101,115,32,116,111,10,32,32,32,32,115,101,97,114,99,104, - 32,119,104,101,110,32,105,109,112,111,114,116,105,110,103,32, - 115,117,98,109,111,100,117,108,101,115,46,32,32,73,102,32, - 115,101,116,44,32,105,115,95,112,97,99,107,97,103,101,32, - 115,104,111,117,108,100,32,98,101,10,32,32,32,32,84,114, - 117,101,45,45,97,110,100,32,70,97,108,115,101,32,111,116, - 104,101,114,119,105,115,101,46,10,10,32,32,32,32,80,97, - 99,107,97,103,101,115,32,97,114,101,32,115,105,109,112,108, - 121,32,109,111,100,117,108,101,115,32,116,104,97,116,32,40, - 109,97,121,41,32,104,97,118,101,32,115,117,98,109,111,100, - 117,108,101,115,46,32,32,73,102,32,97,32,115,112,101,99, - 10,32,32,32,32,104,97,115,32,97,32,110,111,110,45,78, - 111,110,101,32,118,97,108,117,101,32,105,110,32,96,115,117, - 98,109,111,100,117,108,101,95,115,101,97,114,99,104,95,108, - 111,99,97,116,105,111,110,115,96,44,32,116,104,101,32,105, - 109,112,111,114,116,10,32,32,32,32,115,121,115,116,101,109, - 32,119,105,108,108,32,99,111,110,115,105,100,101,114,32,109, - 111,100,117,108,101,115,32,108,111,97,100,101,100,32,102,114, - 111,109,32,116,104,101,32,115,112,101,99,32,97,115,32,112, - 97,99,107,97,103,101,115,46,10,10,32,32,32,32,79,110, - 108,121,32,102,105,110,100,101,114,115,32,40,115,101,101,32, - 105,109,112,111,114,116,108,105,98,46,97,98,99,46,77,101, - 116,97,80,97,116,104,70,105,110,100,101,114,32,97,110,100, - 10,32,32,32,32,105,109,112,111,114,116,108,105,98,46,97, - 98,99,46,80,97,116,104,69,110,116,114,121,70,105,110,100, - 101,114,41,32,115,104,111,117,108,100,32,109,111,100,105,102, - 121,32,77,111,100,117,108,101,83,112,101,99,32,105,110,115, - 116,97,110,99,101,115,46,10,10,32,32,32,32,78,41,3, - 218,6,111,114,105,103,105,110,218,12,108,111,97,100,101,114, - 95,115,116,97,116,101,218,10,105,115,95,112,97,99,107,97, - 103,101,99,3,0,0,0,3,0,0,0,6,0,0,0,2, - 0,0,0,67,0,0,0,115,54,0,0,0,124,1,124,0, - 95,0,124,2,124,0,95,1,124,3,124,0,95,2,124,4, - 124,0,95,3,124,5,114,32,103,0,110,2,100,0,124,0, - 95,4,100,1,124,0,95,5,100,0,124,0,95,6,100,0, - 83,0,41,2,78,70,41,7,114,15,0,0,0,114,93,0, - 0,0,114,103,0,0,0,114,104,0,0,0,218,26,115,117, - 98,109,111,100,117,108,101,95,115,101,97,114,99,104,95,108, - 111,99,97,116,105,111,110,115,218,13,95,115,101,116,95,102, - 105,108,101,97,116,116,114,218,7,95,99,97,99,104,101,100, - 41,6,114,26,0,0,0,114,15,0,0,0,114,93,0,0, - 0,114,103,0,0,0,114,104,0,0,0,114,105,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 27,0,0,0,99,1,0,0,115,14,0,0,0,0,2,6, - 1,6,1,6,1,6,1,14,3,6,1,122,19,77,111,100, - 117,108,101,83,112,101,99,46,95,95,105,110,105,116,95,95, - 99,1,0,0,0,0,0,0,0,2,0,0,0,4,0,0, - 0,67,0,0,0,115,102,0,0,0,100,1,106,0,124,0, - 106,1,131,1,100,2,106,0,124,0,106,2,131,1,103,2, - 125,1,124,0,106,3,100,0,107,9,114,52,124,1,106,4, - 100,3,106,0,124,0,106,3,131,1,131,1,1,0,124,0, - 106,5,100,0,107,9,114,80,124,1,106,4,100,4,106,0, - 124,0,106,5,131,1,131,1,1,0,100,5,106,0,124,0, - 106,6,106,7,100,6,106,8,124,1,131,1,131,2,83,0, - 41,7,78,122,9,110,97,109,101,61,123,33,114,125,122,11, - 108,111,97,100,101,114,61,123,33,114,125,122,11,111,114,105, - 103,105,110,61,123,33,114,125,122,29,115,117,98,109,111,100, - 117,108,101,95,115,101,97,114,99,104,95,108,111,99,97,116, - 105,111,110,115,61,123,125,122,6,123,125,40,123,125,41,122, - 2,44,32,41,9,114,38,0,0,0,114,15,0,0,0,114, - 93,0,0,0,114,103,0,0,0,218,6,97,112,112,101,110, - 100,114,106,0,0,0,218,9,95,95,99,108,97,115,115,95, - 95,114,1,0,0,0,218,4,106,111,105,110,41,2,114,26, - 0,0,0,114,49,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,114,40,0,0,0,111,1,0,0, - 115,16,0,0,0,0,1,10,1,14,1,10,1,18,1,10, - 1,8,1,10,1,122,19,77,111,100,117,108,101,83,112,101, - 99,46,95,95,114,101,112,114,95,95,99,2,0,0,0,0, - 0,0,0,3,0,0,0,11,0,0,0,67,0,0,0,115, - 102,0,0,0,124,0,106,0,125,2,121,70,124,0,106,1, - 124,1,106,1,107,2,111,76,124,0,106,2,124,1,106,2, - 107,2,111,76,124,0,106,3,124,1,106,3,107,2,111,76, - 124,2,124,1,106,0,107,2,111,76,124,0,106,4,124,1, - 106,4,107,2,111,76,124,0,106,5,124,1,106,5,107,2, - 83,0,4,0,116,6,107,10,114,96,1,0,1,0,1,0, - 100,1,83,0,88,0,100,0,83,0,41,2,78,70,41,7, - 114,106,0,0,0,114,15,0,0,0,114,93,0,0,0,114, - 103,0,0,0,218,6,99,97,99,104,101,100,218,12,104,97, - 115,95,108,111,99,97,116,105,111,110,114,90,0,0,0,41, - 3,114,26,0,0,0,90,5,111,116,104,101,114,90,4,115, - 109,115,108,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,6,95,95,101,113,95,95,121,1,0,0,115,20, - 0,0,0,0,1,6,1,2,1,12,1,12,1,12,1,10, - 1,12,1,12,1,14,1,122,17,77,111,100,117,108,101,83, - 112,101,99,46,95,95,101,113,95,95,99,1,0,0,0,0, - 0,0,0,1,0,0,0,2,0,0,0,67,0,0,0,115, - 58,0,0,0,124,0,106,0,100,0,107,8,114,52,124,0, - 106,1,100,0,107,9,114,52,124,0,106,2,114,52,116,3, - 100,0,107,8,114,38,116,4,130,1,116,3,106,5,124,0, - 106,1,131,1,124,0,95,0,124,0,106,0,83,0,41,1, - 78,41,6,114,108,0,0,0,114,103,0,0,0,114,107,0, - 0,0,218,19,95,98,111,111,116,115,116,114,97,112,95,101, - 120,116,101,114,110,97,108,218,19,78,111,116,73,109,112,108, - 101,109,101,110,116,101,100,69,114,114,111,114,90,11,95,103, - 101,116,95,99,97,99,104,101,100,41,1,114,26,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 112,0,0,0,133,1,0,0,115,12,0,0,0,0,2,10, - 1,16,1,8,1,4,1,14,1,122,17,77,111,100,117,108, - 101,83,112,101,99,46,99,97,99,104,101,100,99,2,0,0, - 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, - 0,115,10,0,0,0,124,1,124,0,95,0,100,0,83,0, - 41,1,78,41,1,114,108,0,0,0,41,2,114,26,0,0, - 0,114,112,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,112,0,0,0,142,1,0,0,115,2, - 0,0,0,0,2,99,1,0,0,0,0,0,0,0,1,0, - 0,0,2,0,0,0,67,0,0,0,115,36,0,0,0,124, - 0,106,0,100,1,107,8,114,26,124,0,106,1,106,2,100, - 2,131,1,100,3,25,0,83,0,124,0,106,1,83,0,100, - 1,83,0,41,4,122,32,84,104,101,32,110,97,109,101,32, - 111,102,32,116,104,101,32,109,111,100,117,108,101,39,115,32, - 112,97,114,101,110,116,46,78,218,1,46,114,19,0,0,0, - 41,3,114,106,0,0,0,114,15,0,0,0,218,10,114,112, - 97,114,116,105,116,105,111,110,41,1,114,26,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,6, - 112,97,114,101,110,116,146,1,0,0,115,6,0,0,0,0, - 3,10,1,16,2,122,17,77,111,100,117,108,101,83,112,101, - 99,46,112,97,114,101,110,116,99,1,0,0,0,0,0,0, - 0,1,0,0,0,1,0,0,0,67,0,0,0,115,6,0, - 0,0,124,0,106,0,83,0,41,1,78,41,1,114,107,0, - 0,0,41,1,114,26,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,113,0,0,0,154,1,0, - 0,115,2,0,0,0,0,2,122,23,77,111,100,117,108,101, - 83,112,101,99,46,104,97,115,95,108,111,99,97,116,105,111, - 110,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, - 0,0,67,0,0,0,115,14,0,0,0,116,0,124,1,131, - 1,124,0,95,1,100,0,83,0,41,1,78,41,2,218,4, - 98,111,111,108,114,107,0,0,0,41,2,114,26,0,0,0, - 218,5,118,97,108,117,101,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,113,0,0,0,158,1,0,0,115, - 2,0,0,0,0,2,41,12,114,1,0,0,0,114,0,0, - 0,0,114,2,0,0,0,114,3,0,0,0,114,27,0,0, - 0,114,40,0,0,0,114,114,0,0,0,218,8,112,114,111, - 112,101,114,116,121,114,112,0,0,0,218,6,115,101,116,116, - 101,114,114,119,0,0,0,114,113,0,0,0,114,10,0,0, + 97,112,112,101,114,233,0,0,0,115,8,0,0,0,0,1, + 10,1,10,1,8,1,122,50,95,114,101,113,117,105,114,101, + 115,95,102,114,111,122,101,110,46,60,108,111,99,97,108,115, + 62,46,95,114,101,113,117,105,114,101,115,95,102,114,111,122, + 101,110,95,119,114,97,112,112,101,114,41,1,114,12,0,0, + 0,41,2,114,73,0,0,0,114,77,0,0,0,114,10,0, + 0,0,41,1,114,73,0,0,0,114,11,0,0,0,218,16, + 95,114,101,113,117,105,114,101,115,95,102,114,111,122,101,110, + 231,0,0,0,115,6,0,0,0,0,2,12,5,10,1,114, + 78,0,0,0,99,2,0,0,0,0,0,0,0,4,0,0, + 0,3,0,0,0,67,0,0,0,115,62,0,0,0,116,0, + 124,1,124,0,131,2,125,2,124,1,116,1,106,2,107,6, + 114,50,116,1,106,2,124,1,25,0,125,3,116,3,124,2, + 124,3,131,2,1,0,116,1,106,2,124,1,25,0,83,0, + 116,4,124,2,131,1,83,0,100,1,83,0,41,2,122,128, + 76,111,97,100,32,116,104,101,32,115,112,101,99,105,102,105, + 101,100,32,109,111,100,117,108,101,32,105,110,116,111,32,115, + 121,115,46,109,111,100,117,108,101,115,32,97,110,100,32,114, + 101,116,117,114,110,32,105,116,46,10,10,32,32,32,32,84, + 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, + 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,108, + 111,97,100,101,114,46,101,120,101,99,95,109,111,100,117,108, + 101,32,105,110,115,116,101,97,100,46,10,10,32,32,32,32, + 78,41,5,218,16,115,112,101,99,95,102,114,111,109,95,108, + 111,97,100,101,114,114,14,0,0,0,218,7,109,111,100,117, + 108,101,115,218,5,95,101,120,101,99,218,5,95,108,111,97, + 100,41,4,114,26,0,0,0,114,72,0,0,0,218,4,115, + 112,101,99,218,6,109,111,100,117,108,101,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,218,17,95,108,111,97, + 100,95,109,111,100,117,108,101,95,115,104,105,109,243,0,0, + 0,115,12,0,0,0,0,6,10,1,10,1,10,1,10,1, + 10,2,114,85,0,0,0,99,1,0,0,0,0,0,0,0, + 5,0,0,0,35,0,0,0,67,0,0,0,115,216,0,0, + 0,116,0,124,0,100,1,100,0,131,3,125,1,116,1,124, + 1,100,2,131,2,114,54,121,10,124,1,106,2,124,0,131, + 1,83,0,4,0,116,3,107,10,114,52,1,0,1,0,1, + 0,89,0,110,2,88,0,121,10,124,0,106,4,125,2,87, + 0,110,20,4,0,116,5,107,10,114,84,1,0,1,0,1, + 0,89,0,110,18,88,0,124,2,100,0,107,9,114,102,116, + 6,124,2,131,1,83,0,121,10,124,0,106,7,125,3,87, + 0,110,24,4,0,116,5,107,10,114,136,1,0,1,0,1, + 0,100,3,125,3,89,0,110,2,88,0,121,10,124,0,106, + 8,125,4,87,0,110,50,4,0,116,5,107,10,114,198,1, + 0,1,0,1,0,124,1,100,0,107,8,114,182,100,4,106, + 9,124,3,131,1,83,0,100,5,106,9,124,3,124,1,131, + 2,83,0,89,0,110,14,88,0,100,6,106,9,124,3,124, + 4,131,2,83,0,100,0,83,0,41,7,78,218,10,95,95, + 108,111,97,100,101,114,95,95,218,11,109,111,100,117,108,101, + 95,114,101,112,114,250,1,63,122,13,60,109,111,100,117,108, + 101,32,123,33,114,125,62,122,20,60,109,111,100,117,108,101, + 32,123,33,114,125,32,40,123,33,114,125,41,62,122,23,60, + 109,111,100,117,108,101,32,123,33,114,125,32,102,114,111,109, + 32,123,33,114,125,62,41,10,114,6,0,0,0,114,4,0, + 0,0,114,87,0,0,0,218,9,69,120,99,101,112,116,105, + 111,110,218,8,95,95,115,112,101,99,95,95,218,14,65,116, + 116,114,105,98,117,116,101,69,114,114,111,114,218,22,95,109, + 111,100,117,108,101,95,114,101,112,114,95,102,114,111,109,95, + 115,112,101,99,114,1,0,0,0,218,8,95,95,102,105,108, + 101,95,95,114,38,0,0,0,41,5,114,84,0,0,0,218, + 6,108,111,97,100,101,114,114,83,0,0,0,114,15,0,0, + 0,218,8,102,105,108,101,110,97,109,101,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,218,12,95,109,111,100, + 117,108,101,95,114,101,112,114,3,1,0,0,115,46,0,0, + 0,0,2,12,1,10,4,2,1,10,1,14,1,6,1,2, + 1,10,1,14,1,6,2,8,1,8,4,2,1,10,1,14, + 1,10,1,2,1,10,1,14,1,8,1,10,2,18,2,114, + 96,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, + 0,2,0,0,0,64,0,0,0,115,36,0,0,0,101,0, + 90,1,100,0,90,2,100,1,100,2,132,0,90,3,100,3, + 100,4,132,0,90,4,100,5,100,6,132,0,90,5,100,7, + 83,0,41,8,218,17,95,105,110,115,116,97,108,108,101,100, + 95,115,97,102,101,108,121,99,2,0,0,0,0,0,0,0, + 2,0,0,0,2,0,0,0,67,0,0,0,115,18,0,0, + 0,124,1,124,0,95,0,124,1,106,1,124,0,95,2,100, + 0,83,0,41,1,78,41,3,218,7,95,109,111,100,117,108, + 101,114,90,0,0,0,218,5,95,115,112,101,99,41,2,114, + 26,0,0,0,114,84,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,27,0,0,0,41,1,0, + 0,115,4,0,0,0,0,1,6,1,122,26,95,105,110,115, + 116,97,108,108,101,100,95,115,97,102,101,108,121,46,95,95, + 105,110,105,116,95,95,99,1,0,0,0,0,0,0,0,1, + 0,0,0,3,0,0,0,67,0,0,0,115,28,0,0,0, + 100,1,124,0,106,0,95,1,124,0,106,2,116,3,106,4, + 124,0,106,0,106,5,60,0,100,0,83,0,41,2,78,84, + 41,6,114,99,0,0,0,218,13,95,105,110,105,116,105,97, + 108,105,122,105,110,103,114,98,0,0,0,114,14,0,0,0, + 114,80,0,0,0,114,15,0,0,0,41,1,114,26,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,102,0,0,0,62,1,0,0,115,20,0,0,0,8,35, - 4,2,4,1,14,11,8,10,8,12,12,9,14,4,12,8, - 12,4,114,102,0,0,0,41,2,114,103,0,0,0,114,105, - 0,0,0,99,2,0,0,0,2,0,0,0,6,0,0,0, - 14,0,0,0,67,0,0,0,115,154,0,0,0,116,0,124, - 1,100,1,131,2,114,74,116,1,100,2,107,8,114,22,116, - 2,130,1,116,1,106,3,125,4,124,3,100,2,107,8,114, - 48,124,4,124,0,124,1,100,3,141,2,83,0,124,3,114, - 56,103,0,110,2,100,2,125,5,124,4,124,0,124,1,124, - 5,100,4,141,3,83,0,124,3,100,2,107,8,114,138,116, - 0,124,1,100,5,131,2,114,134,121,14,124,1,106,4,124, - 0,131,1,125,3,87,0,113,138,4,0,116,5,107,10,114, - 130,1,0,1,0,1,0,100,2,125,3,89,0,113,138,88, - 0,110,4,100,6,125,3,116,6,124,0,124,1,124,2,124, - 3,100,7,141,4,83,0,41,8,122,53,82,101,116,117,114, - 110,32,97,32,109,111,100,117,108,101,32,115,112,101,99,32, - 98,97,115,101,100,32,111,110,32,118,97,114,105,111,117,115, - 32,108,111,97,100,101,114,32,109,101,116,104,111,100,115,46, - 90,12,103,101,116,95,102,105,108,101,110,97,109,101,78,41, - 1,114,93,0,0,0,41,2,114,93,0,0,0,114,106,0, - 0,0,114,105,0,0,0,70,41,2,114,103,0,0,0,114, - 105,0,0,0,41,7,114,4,0,0,0,114,115,0,0,0, - 114,116,0,0,0,218,23,115,112,101,99,95,102,114,111,109, - 95,102,105,108,101,95,108,111,99,97,116,105,111,110,114,105, - 0,0,0,114,70,0,0,0,114,102,0,0,0,41,6,114, - 15,0,0,0,114,93,0,0,0,114,103,0,0,0,114,105, - 0,0,0,114,124,0,0,0,90,6,115,101,97,114,99,104, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 78,0,0,0,163,1,0,0,115,34,0,0,0,0,2,10, - 1,8,1,4,1,6,2,8,1,12,1,12,1,6,1,8, - 2,8,1,10,1,2,1,14,1,14,1,12,3,4,2,114, - 78,0,0,0,99,3,0,0,0,0,0,0,0,8,0,0, - 0,53,0,0,0,67,0,0,0,115,56,1,0,0,121,10, - 124,0,106,0,125,3,87,0,110,20,4,0,116,1,107,10, - 114,30,1,0,1,0,1,0,89,0,110,14,88,0,124,3, - 100,0,107,9,114,44,124,3,83,0,124,0,106,2,125,4, - 124,1,100,0,107,8,114,90,121,10,124,0,106,3,125,1, - 87,0,110,20,4,0,116,1,107,10,114,88,1,0,1,0, - 1,0,89,0,110,2,88,0,121,10,124,0,106,4,125,5, - 87,0,110,24,4,0,116,1,107,10,114,124,1,0,1,0, - 1,0,100,0,125,5,89,0,110,2,88,0,124,2,100,0, - 107,8,114,184,124,5,100,0,107,8,114,180,121,10,124,1, - 106,5,125,2,87,0,113,184,4,0,116,1,107,10,114,176, - 1,0,1,0,1,0,100,0,125,2,89,0,113,184,88,0, - 110,4,124,5,125,2,121,10,124,0,106,6,125,6,87,0, - 110,24,4,0,116,1,107,10,114,218,1,0,1,0,1,0, - 100,0,125,6,89,0,110,2,88,0,121,14,116,7,124,0, - 106,8,131,1,125,7,87,0,110,26,4,0,116,1,107,10, - 144,1,114,4,1,0,1,0,1,0,100,0,125,7,89,0, - 110,2,88,0,116,9,124,4,124,1,124,2,100,1,141,3, - 125,3,124,5,100,0,107,8,144,1,114,34,100,2,110,2, - 100,3,124,3,95,10,124,6,124,3,95,11,124,7,124,3, - 95,12,124,3,83,0,41,4,78,41,1,114,103,0,0,0, - 70,84,41,13,114,89,0,0,0,114,90,0,0,0,114,1, - 0,0,0,114,85,0,0,0,114,92,0,0,0,90,7,95, - 79,82,73,71,73,78,218,10,95,95,99,97,99,104,101,100, - 95,95,218,4,108,105,115,116,218,8,95,95,112,97,116,104, - 95,95,114,102,0,0,0,114,107,0,0,0,114,112,0,0, - 0,114,106,0,0,0,41,8,114,83,0,0,0,114,93,0, - 0,0,114,103,0,0,0,114,82,0,0,0,114,15,0,0, - 0,90,8,108,111,99,97,116,105,111,110,114,112,0,0,0, + 114,46,0,0,0,45,1,0,0,115,4,0,0,0,0,4, + 8,1,122,27,95,105,110,115,116,97,108,108,101,100,95,115, + 97,102,101,108,121,46,95,95,101,110,116,101,114,95,95,99, + 1,0,0,0,0,0,0,0,3,0,0,0,17,0,0,0, + 71,0,0,0,115,98,0,0,0,122,82,124,0,106,0,125, + 2,116,1,100,1,100,2,132,0,124,1,68,0,131,1,131, + 1,114,64,121,14,116,2,106,3,124,2,106,4,61,0,87, + 0,113,80,4,0,116,5,107,10,114,60,1,0,1,0,1, + 0,89,0,113,80,88,0,110,16,116,6,100,3,124,2,106, + 4,124,2,106,7,131,3,1,0,87,0,100,0,100,4,124, + 0,106,0,95,8,88,0,100,0,83,0,41,5,78,99,1, + 0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,115, + 0,0,0,115,22,0,0,0,124,0,93,14,125,1,124,1, + 100,0,107,9,86,0,1,0,113,2,100,0,83,0,41,1, + 78,114,10,0,0,0,41,2,90,2,46,48,90,3,97,114, + 103,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, + 250,9,60,103,101,110,101,120,112,114,62,55,1,0,0,115, + 2,0,0,0,4,0,122,45,95,105,110,115,116,97,108,108, + 101,100,95,115,97,102,101,108,121,46,95,95,101,120,105,116, + 95,95,46,60,108,111,99,97,108,115,62,46,60,103,101,110, + 101,120,112,114,62,122,18,105,109,112,111,114,116,32,123,33, + 114,125,32,35,32,123,33,114,125,70,41,9,114,99,0,0, + 0,218,3,97,110,121,114,14,0,0,0,114,80,0,0,0, + 114,15,0,0,0,114,54,0,0,0,114,69,0,0,0,114, + 94,0,0,0,114,100,0,0,0,41,3,114,26,0,0,0, + 114,47,0,0,0,114,83,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,114,48,0,0,0,52,1, + 0,0,115,18,0,0,0,0,1,2,1,6,1,18,1,2, + 1,14,1,14,1,8,2,20,2,122,26,95,105,110,115,116, + 97,108,108,101,100,95,115,97,102,101,108,121,46,95,95,101, + 120,105,116,95,95,78,41,6,114,1,0,0,0,114,0,0, + 0,0,114,2,0,0,0,114,27,0,0,0,114,46,0,0, + 0,114,48,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,114,97,0,0,0,39, + 1,0,0,115,6,0,0,0,8,2,8,4,8,7,114,97, + 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, + 4,0,0,0,64,0,0,0,115,114,0,0,0,101,0,90, + 1,100,0,90,2,100,1,90,3,100,2,100,2,100,2,100, + 3,156,3,100,4,100,5,132,2,90,4,100,6,100,7,132, + 0,90,5,100,8,100,9,132,0,90,6,101,7,100,10,100, + 11,132,0,131,1,90,8,101,8,106,9,100,12,100,11,132, + 0,131,1,90,8,101,7,100,13,100,14,132,0,131,1,90, + 10,101,7,100,15,100,16,132,0,131,1,90,11,101,11,106, + 9,100,17,100,16,132,0,131,1,90,11,100,2,83,0,41, + 18,218,10,77,111,100,117,108,101,83,112,101,99,97,208,5, + 0,0,84,104,101,32,115,112,101,99,105,102,105,99,97,116, + 105,111,110,32,102,111,114,32,97,32,109,111,100,117,108,101, + 44,32,117,115,101,100,32,102,111,114,32,108,111,97,100,105, + 110,103,46,10,10,32,32,32,32,65,32,109,111,100,117,108, + 101,39,115,32,115,112,101,99,32,105,115,32,116,104,101,32, + 115,111,117,114,99,101,32,102,111,114,32,105,110,102,111,114, + 109,97,116,105,111,110,32,97,98,111,117,116,32,116,104,101, + 32,109,111,100,117,108,101,46,32,32,70,111,114,10,32,32, + 32,32,100,97,116,97,32,97,115,115,111,99,105,97,116,101, + 100,32,119,105,116,104,32,116,104,101,32,109,111,100,117,108, + 101,44,32,105,110,99,108,117,100,105,110,103,32,115,111,117, + 114,99,101,44,32,117,115,101,32,116,104,101,32,115,112,101, + 99,39,115,10,32,32,32,32,108,111,97,100,101,114,46,10, + 10,32,32,32,32,96,110,97,109,101,96,32,105,115,32,116, + 104,101,32,97,98,115,111,108,117,116,101,32,110,97,109,101, + 32,111,102,32,116,104,101,32,109,111,100,117,108,101,46,32, + 32,96,108,111,97,100,101,114,96,32,105,115,32,116,104,101, + 32,108,111,97,100,101,114,10,32,32,32,32,116,111,32,117, + 115,101,32,119,104,101,110,32,108,111,97,100,105,110,103,32, + 116,104,101,32,109,111,100,117,108,101,46,32,32,96,112,97, + 114,101,110,116,96,32,105,115,32,116,104,101,32,110,97,109, + 101,32,111,102,32,116,104,101,10,32,32,32,32,112,97,99, + 107,97,103,101,32,116,104,101,32,109,111,100,117,108,101,32, + 105,115,32,105,110,46,32,32,84,104,101,32,112,97,114,101, + 110,116,32,105,115,32,100,101,114,105,118,101,100,32,102,114, + 111,109,32,116,104,101,32,110,97,109,101,46,10,10,32,32, + 32,32,96,105,115,95,112,97,99,107,97,103,101,96,32,100, + 101,116,101,114,109,105,110,101,115,32,105,102,32,116,104,101, + 32,109,111,100,117,108,101,32,105,115,32,99,111,110,115,105, + 100,101,114,101,100,32,97,32,112,97,99,107,97,103,101,32, + 111,114,10,32,32,32,32,110,111,116,46,32,32,79,110,32, + 109,111,100,117,108,101,115,32,116,104,105,115,32,105,115,32, + 114,101,102,108,101,99,116,101,100,32,98,121,32,116,104,101, + 32,96,95,95,112,97,116,104,95,95,96,32,97,116,116,114, + 105,98,117,116,101,46,10,10,32,32,32,32,96,111,114,105, + 103,105,110,96,32,105,115,32,116,104,101,32,115,112,101,99, + 105,102,105,99,32,108,111,99,97,116,105,111,110,32,117,115, + 101,100,32,98,121,32,116,104,101,32,108,111,97,100,101,114, + 32,102,114,111,109,32,119,104,105,99,104,32,116,111,10,32, + 32,32,32,108,111,97,100,32,116,104,101,32,109,111,100,117, + 108,101,44,32,105,102,32,116,104,97,116,32,105,110,102,111, + 114,109,97,116,105,111,110,32,105,115,32,97,118,97,105,108, + 97,98,108,101,46,32,32,87,104,101,110,32,102,105,108,101, + 110,97,109,101,32,105,115,10,32,32,32,32,115,101,116,44, + 32,111,114,105,103,105,110,32,119,105,108,108,32,109,97,116, + 99,104,46,10,10,32,32,32,32,96,104,97,115,95,108,111, + 99,97,116,105,111,110,96,32,105,110,100,105,99,97,116,101, + 115,32,116,104,97,116,32,97,32,115,112,101,99,39,115,32, + 34,111,114,105,103,105,110,34,32,114,101,102,108,101,99,116, + 115,32,97,32,108,111,99,97,116,105,111,110,46,10,32,32, + 32,32,87,104,101,110,32,116,104,105,115,32,105,115,32,84, + 114,117,101,44,32,96,95,95,102,105,108,101,95,95,96,32, + 97,116,116,114,105,98,117,116,101,32,111,102,32,116,104,101, + 32,109,111,100,117,108,101,32,105,115,32,115,101,116,46,10, + 10,32,32,32,32,96,99,97,99,104,101,100,96,32,105,115, + 32,116,104,101,32,108,111,99,97,116,105,111,110,32,111,102, + 32,116,104,101,32,99,97,99,104,101,100,32,98,121,116,101, + 99,111,100,101,32,102,105,108,101,44,32,105,102,32,97,110, + 121,46,32,32,73,116,10,32,32,32,32,99,111,114,114,101, + 115,112,111,110,100,115,32,116,111,32,116,104,101,32,96,95, + 95,99,97,99,104,101,100,95,95,96,32,97,116,116,114,105, + 98,117,116,101,46,10,10,32,32,32,32,96,115,117,98,109, + 111,100,117,108,101,95,115,101,97,114,99,104,95,108,111,99, + 97,116,105,111,110,115,96,32,105,115,32,116,104,101,32,115, + 101,113,117,101,110,99,101,32,111,102,32,112,97,116,104,32, + 101,110,116,114,105,101,115,32,116,111,10,32,32,32,32,115, + 101,97,114,99,104,32,119,104,101,110,32,105,109,112,111,114, + 116,105,110,103,32,115,117,98,109,111,100,117,108,101,115,46, + 32,32,73,102,32,115,101,116,44,32,105,115,95,112,97,99, + 107,97,103,101,32,115,104,111,117,108,100,32,98,101,10,32, + 32,32,32,84,114,117,101,45,45,97,110,100,32,70,97,108, + 115,101,32,111,116,104,101,114,119,105,115,101,46,10,10,32, + 32,32,32,80,97,99,107,97,103,101,115,32,97,114,101,32, + 115,105,109,112,108,121,32,109,111,100,117,108,101,115,32,116, + 104,97,116,32,40,109,97,121,41,32,104,97,118,101,32,115, + 117,98,109,111,100,117,108,101,115,46,32,32,73,102,32,97, + 32,115,112,101,99,10,32,32,32,32,104,97,115,32,97,32, + 110,111,110,45,78,111,110,101,32,118,97,108,117,101,32,105, + 110,32,96,115,117,98,109,111,100,117,108,101,95,115,101,97, + 114,99,104,95,108,111,99,97,116,105,111,110,115,96,44,32, + 116,104,101,32,105,109,112,111,114,116,10,32,32,32,32,115, + 121,115,116,101,109,32,119,105,108,108,32,99,111,110,115,105, + 100,101,114,32,109,111,100,117,108,101,115,32,108,111,97,100, + 101,100,32,102,114,111,109,32,116,104,101,32,115,112,101,99, + 32,97,115,32,112,97,99,107,97,103,101,115,46,10,10,32, + 32,32,32,79,110,108,121,32,102,105,110,100,101,114,115,32, + 40,115,101,101,32,105,109,112,111,114,116,108,105,98,46,97, + 98,99,46,77,101,116,97,80,97,116,104,70,105,110,100,101, + 114,32,97,110,100,10,32,32,32,32,105,109,112,111,114,116, + 108,105,98,46,97,98,99,46,80,97,116,104,69,110,116,114, + 121,70,105,110,100,101,114,41,32,115,104,111,117,108,100,32, + 109,111,100,105,102,121,32,77,111,100,117,108,101,83,112,101, + 99,32,105,110,115,116,97,110,99,101,115,46,10,10,32,32, + 32,32,78,41,3,218,6,111,114,105,103,105,110,218,12,108, + 111,97,100,101,114,95,115,116,97,116,101,218,10,105,115,95, + 112,97,99,107,97,103,101,99,3,0,0,0,3,0,0,0, + 6,0,0,0,2,0,0,0,67,0,0,0,115,54,0,0, + 0,124,1,124,0,95,0,124,2,124,0,95,1,124,3,124, + 0,95,2,124,4,124,0,95,3,124,5,114,32,103,0,110, + 2,100,0,124,0,95,4,100,1,124,0,95,5,100,0,124, + 0,95,6,100,0,83,0,41,2,78,70,41,7,114,15,0, + 0,0,114,94,0,0,0,114,104,0,0,0,114,105,0,0, + 0,218,26,115,117,98,109,111,100,117,108,101,95,115,101,97, + 114,99,104,95,108,111,99,97,116,105,111,110,115,218,13,95, + 115,101,116,95,102,105,108,101,97,116,116,114,218,7,95,99, + 97,99,104,101,100,41,6,114,26,0,0,0,114,15,0,0, + 0,114,94,0,0,0,114,104,0,0,0,114,105,0,0,0, 114,106,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,17,95,115,112,101,99,95,102,114,111,109, - 95,109,111,100,117,108,101,192,1,0,0,115,72,0,0,0, - 0,2,2,1,10,1,14,1,6,2,8,1,4,2,6,1, - 8,1,2,1,10,1,14,2,6,1,2,1,10,1,14,1, - 10,1,8,1,8,1,2,1,10,1,14,1,12,2,4,1, - 2,1,10,1,14,1,10,1,2,1,14,1,16,1,10,2, - 14,1,20,1,6,1,6,1,114,128,0,0,0,70,41,1, - 218,8,111,118,101,114,114,105,100,101,99,2,0,0,0,1, - 0,0,0,5,0,0,0,59,0,0,0,67,0,0,0,115, - 212,1,0,0,124,2,115,20,116,0,124,1,100,1,100,0, - 131,3,100,0,107,8,114,54,121,12,124,0,106,1,124,1, - 95,2,87,0,110,20,4,0,116,3,107,10,114,52,1,0, - 1,0,1,0,89,0,110,2,88,0,124,2,115,74,116,0, - 124,1,100,2,100,0,131,3,100,0,107,8,114,166,124,0, - 106,4,125,3,124,3,100,0,107,8,114,134,124,0,106,5, - 100,0,107,9,114,134,116,6,100,0,107,8,114,110,116,7, - 130,1,116,6,106,8,125,4,124,4,106,9,124,4,131,1, - 125,3,124,0,106,5,124,3,95,10,121,10,124,3,124,1, - 95,11,87,0,110,20,4,0,116,3,107,10,114,164,1,0, - 1,0,1,0,89,0,110,2,88,0,124,2,115,186,116,0, - 124,1,100,3,100,0,131,3,100,0,107,8,114,220,121,12, - 124,0,106,12,124,1,95,13,87,0,110,20,4,0,116,3, - 107,10,114,218,1,0,1,0,1,0,89,0,110,2,88,0, - 121,10,124,0,124,1,95,14,87,0,110,20,4,0,116,3, - 107,10,114,250,1,0,1,0,1,0,89,0,110,2,88,0, - 124,2,144,1,115,20,116,0,124,1,100,4,100,0,131,3, - 100,0,107,8,144,1,114,68,124,0,106,5,100,0,107,9, - 144,1,114,68,121,12,124,0,106,5,124,1,95,15,87,0, - 110,22,4,0,116,3,107,10,144,1,114,66,1,0,1,0, - 1,0,89,0,110,2,88,0,124,0,106,16,144,1,114,208, - 124,2,144,1,115,100,116,0,124,1,100,5,100,0,131,3, - 100,0,107,8,144,1,114,136,121,12,124,0,106,17,124,1, - 95,18,87,0,110,22,4,0,116,3,107,10,144,1,114,134, - 1,0,1,0,1,0,89,0,110,2,88,0,124,2,144,1, - 115,160,116,0,124,1,100,6,100,0,131,3,100,0,107,8, - 144,1,114,208,124,0,106,19,100,0,107,9,144,1,114,208, - 121,12,124,0,106,19,124,1,95,20,87,0,110,22,4,0, - 116,3,107,10,144,1,114,206,1,0,1,0,1,0,89,0, - 110,2,88,0,124,1,83,0,41,7,78,114,1,0,0,0, - 114,85,0,0,0,218,11,95,95,112,97,99,107,97,103,101, - 95,95,114,127,0,0,0,114,92,0,0,0,114,125,0,0, - 0,41,21,114,6,0,0,0,114,15,0,0,0,114,1,0, - 0,0,114,90,0,0,0,114,93,0,0,0,114,106,0,0, - 0,114,115,0,0,0,114,116,0,0,0,218,16,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,218,7,95, - 95,110,101,119,95,95,90,5,95,112,97,116,104,114,85,0, - 0,0,114,119,0,0,0,114,130,0,0,0,114,89,0,0, - 0,114,127,0,0,0,114,113,0,0,0,114,103,0,0,0, - 114,92,0,0,0,114,112,0,0,0,114,125,0,0,0,41, - 5,114,82,0,0,0,114,83,0,0,0,114,129,0,0,0, - 114,93,0,0,0,114,131,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,218,18,95,105,110,105,116, - 95,109,111,100,117,108,101,95,97,116,116,114,115,237,1,0, - 0,115,92,0,0,0,0,4,20,1,2,1,12,1,14,1, - 6,2,20,1,6,1,8,2,10,1,8,1,4,1,6,2, - 10,1,8,1,2,1,10,1,14,1,6,2,20,1,2,1, - 12,1,14,1,6,2,2,1,10,1,14,1,6,2,24,1, - 12,1,2,1,12,1,16,1,6,2,8,1,24,1,2,1, - 12,1,16,1,6,2,24,1,12,1,2,1,12,1,16,1, - 6,1,114,133,0,0,0,99,1,0,0,0,0,0,0,0, - 2,0,0,0,3,0,0,0,67,0,0,0,115,82,0,0, - 0,100,1,125,1,116,0,124,0,106,1,100,2,131,2,114, - 30,124,0,106,1,106,2,124,0,131,1,125,1,110,20,116, - 0,124,0,106,1,100,3,131,2,114,50,116,3,100,4,131, - 1,130,1,124,1,100,1,107,8,114,68,116,4,124,0,106, - 5,131,1,125,1,116,6,124,0,124,1,131,2,1,0,124, - 1,83,0,41,5,122,43,67,114,101,97,116,101,32,97,32, - 109,111,100,117,108,101,32,98,97,115,101,100,32,111,110,32, - 116,104,101,32,112,114,111,118,105,100,101,100,32,115,112,101, - 99,46,78,218,13,99,114,101,97,116,101,95,109,111,100,117, - 108,101,218,11,101,120,101,99,95,109,111,100,117,108,101,122, - 66,108,111,97,100,101,114,115,32,116,104,97,116,32,100,101, - 102,105,110,101,32,101,120,101,99,95,109,111,100,117,108,101, - 40,41,32,109,117,115,116,32,97,108,115,111,32,100,101,102, - 105,110,101,32,99,114,101,97,116,101,95,109,111,100,117,108, - 101,40,41,41,7,114,4,0,0,0,114,93,0,0,0,114, - 134,0,0,0,114,70,0,0,0,114,16,0,0,0,114,15, - 0,0,0,114,133,0,0,0,41,2,114,82,0,0,0,114, - 83,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,16,109,111,100,117,108,101,95,102,114,111,109, - 95,115,112,101,99,41,2,0,0,115,18,0,0,0,0,3, - 4,1,12,3,14,1,12,1,8,2,8,1,10,1,10,1, - 114,136,0,0,0,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,106,0,0,0,124, - 0,106,0,100,1,107,8,114,14,100,2,110,4,124,0,106, - 0,125,1,124,0,106,1,100,1,107,8,114,66,124,0,106, - 2,100,1,107,8,114,50,100,3,106,3,124,1,131,1,83, - 0,100,4,106,3,124,1,124,0,106,2,131,2,83,0,110, - 36,124,0,106,4,114,86,100,5,106,3,124,1,124,0,106, - 1,131,2,83,0,100,6,106,3,124,0,106,0,124,0,106, - 1,131,2,83,0,100,1,83,0,41,7,122,38,82,101,116, - 117,114,110,32,116,104,101,32,114,101,112,114,32,116,111,32, - 117,115,101,32,102,111,114,32,116,104,101,32,109,111,100,117, - 108,101,46,78,114,87,0,0,0,122,13,60,109,111,100,117, - 108,101,32,123,33,114,125,62,122,20,60,109,111,100,117,108, - 101,32,123,33,114,125,32,40,123,33,114,125,41,62,122,23, - 60,109,111,100,117,108,101,32,123,33,114,125,32,102,114,111, - 109,32,123,33,114,125,62,122,18,60,109,111,100,117,108,101, - 32,123,33,114,125,32,40,123,125,41,62,41,5,114,15,0, - 0,0,114,103,0,0,0,114,93,0,0,0,114,38,0,0, - 0,114,113,0,0,0,41,2,114,82,0,0,0,114,15,0, - 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,91,0,0,0,58,2,0,0,115,16,0,0,0,0, - 3,20,1,10,1,10,1,10,2,16,2,6,1,14,2,114, - 91,0,0,0,99,2,0,0,0,0,0,0,0,4,0,0, - 0,12,0,0,0,67,0,0,0,115,186,0,0,0,124,0, - 106,0,125,2,116,1,106,2,131,0,1,0,116,3,124,2, - 131,1,143,148,1,0,116,4,106,5,106,6,124,2,131,1, - 124,1,107,9,114,62,100,1,106,7,124,2,131,1,125,3, - 116,8,124,3,124,2,100,2,141,2,130,1,124,0,106,9, - 100,3,107,8,114,114,124,0,106,10,100,3,107,8,114,96, - 116,8,100,4,124,0,106,0,100,2,141,2,130,1,116,11, - 124,0,124,1,100,5,100,6,141,3,1,0,124,1,83,0, - 116,11,124,0,124,1,100,5,100,6,141,3,1,0,116,12, - 124,0,106,9,100,7,131,2,115,154,124,0,106,9,106,13, - 124,2,131,1,1,0,110,12,124,0,106,9,106,14,124,1, - 131,1,1,0,87,0,100,3,81,0,82,0,88,0,116,4, - 106,5,124,2,25,0,83,0,41,8,122,70,69,120,101,99, - 117,116,101,32,116,104,101,32,115,112,101,99,39,115,32,115, - 112,101,99,105,102,105,101,100,32,109,111,100,117,108,101,32, - 105,110,32,97,110,32,101,120,105,115,116,105,110,103,32,109, - 111,100,117,108,101,39,115,32,110,97,109,101,115,112,97,99, - 101,46,122,30,109,111,100,117,108,101,32,123,33,114,125,32, - 110,111,116,32,105,110,32,115,121,115,46,109,111,100,117,108, - 101,115,41,1,114,15,0,0,0,78,122,14,109,105,115,115, - 105,110,103,32,108,111,97,100,101,114,84,41,1,114,129,0, - 0,0,114,135,0,0,0,41,15,114,15,0,0,0,114,46, - 0,0,0,218,12,97,99,113,117,105,114,101,95,108,111,99, - 107,114,42,0,0,0,114,14,0,0,0,114,79,0,0,0, - 114,30,0,0,0,114,38,0,0,0,114,70,0,0,0,114, - 93,0,0,0,114,106,0,0,0,114,133,0,0,0,114,4, - 0,0,0,218,11,108,111,97,100,95,109,111,100,117,108,101, - 114,135,0,0,0,41,4,114,82,0,0,0,114,83,0,0, - 0,114,15,0,0,0,218,3,109,115,103,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,114,80,0,0,0,75, - 2,0,0,115,32,0,0,0,0,2,6,1,8,1,10,1, - 16,1,10,1,12,1,10,1,10,1,14,2,14,1,4,1, - 14,1,12,4,14,2,22,1,114,80,0,0,0,99,1,0, - 0,0,0,0,0,0,2,0,0,0,27,0,0,0,67,0, - 0,0,115,206,0,0,0,124,0,106,0,106,1,124,0,106, - 2,131,1,1,0,116,3,106,4,124,0,106,2,25,0,125, - 1,116,5,124,1,100,1,100,0,131,3,100,0,107,8,114, - 76,121,12,124,0,106,0,124,1,95,6,87,0,110,20,4, - 0,116,7,107,10,114,74,1,0,1,0,1,0,89,0,110, - 2,88,0,116,5,124,1,100,2,100,0,131,3,100,0,107, - 8,114,154,121,40,124,1,106,8,124,1,95,9,116,10,124, - 1,100,3,131,2,115,130,124,0,106,2,106,11,100,4,131, - 1,100,5,25,0,124,1,95,9,87,0,110,20,4,0,116, - 7,107,10,114,152,1,0,1,0,1,0,89,0,110,2,88, - 0,116,5,124,1,100,6,100,0,131,3,100,0,107,8,114, - 202,121,10,124,0,124,1,95,12,87,0,110,20,4,0,116, - 7,107,10,114,200,1,0,1,0,1,0,89,0,110,2,88, - 0,124,1,83,0,41,7,78,114,85,0,0,0,114,130,0, - 0,0,114,127,0,0,0,114,117,0,0,0,114,19,0,0, - 0,114,89,0,0,0,41,13,114,93,0,0,0,114,138,0, - 0,0,114,15,0,0,0,114,14,0,0,0,114,79,0,0, - 0,114,6,0,0,0,114,85,0,0,0,114,90,0,0,0, - 114,1,0,0,0,114,130,0,0,0,114,4,0,0,0,114, - 118,0,0,0,114,89,0,0,0,41,2,114,82,0,0,0, - 114,83,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,25,95,108,111,97,100,95,98,97,99,107, - 119,97,114,100,95,99,111,109,112,97,116,105,98,108,101,100, - 2,0,0,115,40,0,0,0,0,4,14,2,12,1,16,1, - 2,1,12,1,14,1,6,1,16,1,2,4,8,1,10,1, - 22,1,14,1,6,1,16,1,2,1,10,1,14,1,6,1, - 114,140,0,0,0,99,1,0,0,0,0,0,0,0,2,0, - 0,0,11,0,0,0,67,0,0,0,115,118,0,0,0,124, - 0,106,0,100,0,107,9,114,30,116,1,124,0,106,0,100, - 1,131,2,115,30,116,2,124,0,131,1,83,0,116,3,124, - 0,131,1,125,1,116,4,124,1,131,1,143,54,1,0,124, - 0,106,0,100,0,107,8,114,84,124,0,106,5,100,0,107, - 8,114,96,116,6,100,2,124,0,106,7,100,3,141,2,130, - 1,110,12,124,0,106,0,106,8,124,1,131,1,1,0,87, - 0,100,0,81,0,82,0,88,0,116,9,106,10,124,0,106, - 7,25,0,83,0,41,4,78,114,135,0,0,0,122,14,109, - 105,115,115,105,110,103,32,108,111,97,100,101,114,41,1,114, - 15,0,0,0,41,11,114,93,0,0,0,114,4,0,0,0, - 114,140,0,0,0,114,136,0,0,0,114,96,0,0,0,114, - 106,0,0,0,114,70,0,0,0,114,15,0,0,0,114,135, - 0,0,0,114,14,0,0,0,114,79,0,0,0,41,2,114, - 82,0,0,0,114,83,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,218,14,95,108,111,97,100,95, - 117,110,108,111,99,107,101,100,129,2,0,0,115,20,0,0, - 0,0,2,10,2,12,1,8,2,8,1,10,1,10,1,10, - 1,16,3,22,5,114,141,0,0,0,99,1,0,0,0,0, - 0,0,0,1,0,0,0,9,0,0,0,67,0,0,0,115, - 38,0,0,0,116,0,106,1,131,0,1,0,116,2,124,0, - 106,3,131,1,143,10,1,0,116,4,124,0,131,1,83,0, + 11,0,0,0,114,27,0,0,0,103,1,0,0,115,14,0, + 0,0,0,2,6,1,6,1,6,1,6,1,14,3,6,1, + 122,19,77,111,100,117,108,101,83,112,101,99,46,95,95,105, + 110,105,116,95,95,99,1,0,0,0,0,0,0,0,2,0, + 0,0,4,0,0,0,67,0,0,0,115,102,0,0,0,100, + 1,106,0,124,0,106,1,131,1,100,2,106,0,124,0,106, + 2,131,1,103,2,125,1,124,0,106,3,100,0,107,9,114, + 52,124,1,106,4,100,3,106,0,124,0,106,3,131,1,131, + 1,1,0,124,0,106,5,100,0,107,9,114,80,124,1,106, + 4,100,4,106,0,124,0,106,5,131,1,131,1,1,0,100, + 5,106,0,124,0,106,6,106,7,100,6,106,8,124,1,131, + 1,131,2,83,0,41,7,78,122,9,110,97,109,101,61,123, + 33,114,125,122,11,108,111,97,100,101,114,61,123,33,114,125, + 122,11,111,114,105,103,105,110,61,123,33,114,125,122,29,115, + 117,98,109,111,100,117,108,101,95,115,101,97,114,99,104,95, + 108,111,99,97,116,105,111,110,115,61,123,125,122,6,123,125, + 40,123,125,41,122,2,44,32,41,9,114,38,0,0,0,114, + 15,0,0,0,114,94,0,0,0,114,104,0,0,0,218,6, + 97,112,112,101,110,100,114,107,0,0,0,218,9,95,95,99, + 108,97,115,115,95,95,114,1,0,0,0,218,4,106,111,105, + 110,41,2,114,26,0,0,0,114,47,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,114,40,0,0, + 0,115,1,0,0,115,16,0,0,0,0,1,10,1,14,1, + 10,1,18,1,10,1,8,1,10,1,122,19,77,111,100,117, + 108,101,83,112,101,99,46,95,95,114,101,112,114,95,95,99, + 2,0,0,0,0,0,0,0,3,0,0,0,11,0,0,0, + 67,0,0,0,115,102,0,0,0,124,0,106,0,125,2,121, + 70,124,0,106,1,124,1,106,1,107,2,111,76,124,0,106, + 2,124,1,106,2,107,2,111,76,124,0,106,3,124,1,106, + 3,107,2,111,76,124,2,124,1,106,0,107,2,111,76,124, + 0,106,4,124,1,106,4,107,2,111,76,124,0,106,5,124, + 1,106,5,107,2,83,0,4,0,116,6,107,10,114,96,1, + 0,1,0,1,0,100,1,83,0,88,0,100,0,83,0,41, + 2,78,70,41,7,114,107,0,0,0,114,15,0,0,0,114, + 94,0,0,0,114,104,0,0,0,218,6,99,97,99,104,101, + 100,218,12,104,97,115,95,108,111,99,97,116,105,111,110,114, + 91,0,0,0,41,3,114,26,0,0,0,90,5,111,116,104, + 101,114,90,4,115,109,115,108,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,6,95,95,101,113,95,95,125, + 1,0,0,115,20,0,0,0,0,1,6,1,2,1,12,1, + 12,1,12,1,10,1,12,1,12,1,14,1,122,17,77,111, + 100,117,108,101,83,112,101,99,46,95,95,101,113,95,95,99, + 1,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0, + 67,0,0,0,115,58,0,0,0,124,0,106,0,100,0,107, + 8,114,52,124,0,106,1,100,0,107,9,114,52,124,0,106, + 2,114,52,116,3,100,0,107,8,114,38,116,4,130,1,116, + 3,106,5,124,0,106,1,131,1,124,0,95,0,124,0,106, + 0,83,0,41,1,78,41,6,114,109,0,0,0,114,104,0, + 0,0,114,108,0,0,0,218,19,95,98,111,111,116,115,116, + 114,97,112,95,101,120,116,101,114,110,97,108,218,19,78,111, + 116,73,109,112,108,101,109,101,110,116,101,100,69,114,114,111, + 114,90,11,95,103,101,116,95,99,97,99,104,101,100,41,1, + 114,26,0,0,0,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,113,0,0,0,137,1,0,0,115,12,0, + 0,0,0,2,10,1,16,1,8,1,4,1,14,1,122,17, + 77,111,100,117,108,101,83,112,101,99,46,99,97,99,104,101, + 100,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, + 0,0,67,0,0,0,115,10,0,0,0,124,1,124,0,95, + 0,100,0,83,0,41,1,78,41,1,114,109,0,0,0,41, + 2,114,26,0,0,0,114,113,0,0,0,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,114,113,0,0,0,146, + 1,0,0,115,2,0,0,0,0,2,99,1,0,0,0,0, + 0,0,0,1,0,0,0,2,0,0,0,67,0,0,0,115, + 36,0,0,0,124,0,106,0,100,1,107,8,114,26,124,0, + 106,1,106,2,100,2,131,1,100,3,25,0,83,0,124,0, + 106,1,83,0,100,1,83,0,41,4,122,32,84,104,101,32, + 110,97,109,101,32,111,102,32,116,104,101,32,109,111,100,117, + 108,101,39,115,32,112,97,114,101,110,116,46,78,218,1,46, + 114,19,0,0,0,41,3,114,107,0,0,0,114,15,0,0, + 0,218,10,114,112,97,114,116,105,116,105,111,110,41,1,114, + 26,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,6,112,97,114,101,110,116,150,1,0,0,115, + 6,0,0,0,0,3,10,1,16,2,122,17,77,111,100,117, + 108,101,83,112,101,99,46,112,97,114,101,110,116,99,1,0, + 0,0,0,0,0,0,1,0,0,0,1,0,0,0,67,0, + 0,0,115,6,0,0,0,124,0,106,0,83,0,41,1,78, + 41,1,114,108,0,0,0,41,1,114,26,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,114,0, + 0,0,158,1,0,0,115,2,0,0,0,0,2,122,23,77, + 111,100,117,108,101,83,112,101,99,46,104,97,115,95,108,111, + 99,97,116,105,111,110,99,2,0,0,0,0,0,0,0,2, + 0,0,0,2,0,0,0,67,0,0,0,115,14,0,0,0, + 116,0,124,1,131,1,124,0,95,1,100,0,83,0,41,1, + 78,41,2,218,4,98,111,111,108,114,108,0,0,0,41,2, + 114,26,0,0,0,218,5,118,97,108,117,101,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,114,0,0,0, + 162,1,0,0,115,2,0,0,0,0,2,41,12,114,1,0, + 0,0,114,0,0,0,0,114,2,0,0,0,114,3,0,0, + 0,114,27,0,0,0,114,40,0,0,0,114,115,0,0,0, + 218,8,112,114,111,112,101,114,116,121,114,113,0,0,0,218, + 6,115,101,116,116,101,114,114,120,0,0,0,114,114,0,0, + 0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,103,0,0,0,66,1,0,0,115,20, + 0,0,0,8,35,4,2,4,1,14,11,8,10,8,12,12, + 9,14,4,12,8,12,4,114,103,0,0,0,41,2,114,104, + 0,0,0,114,106,0,0,0,99,2,0,0,0,2,0,0, + 0,6,0,0,0,14,0,0,0,67,0,0,0,115,154,0, + 0,0,116,0,124,1,100,1,131,2,114,74,116,1,100,2, + 107,8,114,22,116,2,130,1,116,1,106,3,125,4,124,3, + 100,2,107,8,114,48,124,4,124,0,124,1,100,3,141,2, + 83,0,124,3,114,56,103,0,110,2,100,2,125,5,124,4, + 124,0,124,1,124,5,100,4,141,3,83,0,124,3,100,2, + 107,8,114,138,116,0,124,1,100,5,131,2,114,134,121,14, + 124,1,106,4,124,0,131,1,125,3,87,0,113,138,4,0, + 116,5,107,10,114,130,1,0,1,0,1,0,100,2,125,3, + 89,0,113,138,88,0,110,4,100,6,125,3,116,6,124,0, + 124,1,124,2,124,3,100,7,141,4,83,0,41,8,122,53, + 82,101,116,117,114,110,32,97,32,109,111,100,117,108,101,32, + 115,112,101,99,32,98,97,115,101,100,32,111,110,32,118,97, + 114,105,111,117,115,32,108,111,97,100,101,114,32,109,101,116, + 104,111,100,115,46,90,12,103,101,116,95,102,105,108,101,110, + 97,109,101,78,41,1,114,94,0,0,0,41,2,114,94,0, + 0,0,114,107,0,0,0,114,106,0,0,0,70,41,2,114, + 104,0,0,0,114,106,0,0,0,41,7,114,4,0,0,0, + 114,116,0,0,0,114,117,0,0,0,218,23,115,112,101,99, + 95,102,114,111,109,95,102,105,108,101,95,108,111,99,97,116, + 105,111,110,114,106,0,0,0,114,71,0,0,0,114,103,0, + 0,0,41,6,114,15,0,0,0,114,94,0,0,0,114,104, + 0,0,0,114,106,0,0,0,114,125,0,0,0,90,6,115, + 101,97,114,99,104,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,79,0,0,0,167,1,0,0,115,34,0, + 0,0,0,2,10,1,8,1,4,1,6,2,8,1,12,1, + 12,1,6,1,8,2,8,1,10,1,2,1,14,1,14,1, + 12,3,4,2,114,79,0,0,0,99,3,0,0,0,0,0, + 0,0,8,0,0,0,53,0,0,0,67,0,0,0,115,56, + 1,0,0,121,10,124,0,106,0,125,3,87,0,110,20,4, + 0,116,1,107,10,114,30,1,0,1,0,1,0,89,0,110, + 14,88,0,124,3,100,0,107,9,114,44,124,3,83,0,124, + 0,106,2,125,4,124,1,100,0,107,8,114,90,121,10,124, + 0,106,3,125,1,87,0,110,20,4,0,116,1,107,10,114, + 88,1,0,1,0,1,0,89,0,110,2,88,0,121,10,124, + 0,106,4,125,5,87,0,110,24,4,0,116,1,107,10,114, + 124,1,0,1,0,1,0,100,0,125,5,89,0,110,2,88, + 0,124,2,100,0,107,8,114,184,124,5,100,0,107,8,114, + 180,121,10,124,1,106,5,125,2,87,0,113,184,4,0,116, + 1,107,10,114,176,1,0,1,0,1,0,100,0,125,2,89, + 0,113,184,88,0,110,4,124,5,125,2,121,10,124,0,106, + 6,125,6,87,0,110,24,4,0,116,1,107,10,114,218,1, + 0,1,0,1,0,100,0,125,6,89,0,110,2,88,0,121, + 14,116,7,124,0,106,8,131,1,125,7,87,0,110,26,4, + 0,116,1,107,10,144,1,114,4,1,0,1,0,1,0,100, + 0,125,7,89,0,110,2,88,0,116,9,124,4,124,1,124, + 2,100,1,141,3,125,3,124,5,100,0,107,8,144,1,114, + 34,100,2,110,2,100,3,124,3,95,10,124,6,124,3,95, + 11,124,7,124,3,95,12,124,3,83,0,41,4,78,41,1, + 114,104,0,0,0,70,84,41,13,114,90,0,0,0,114,91, + 0,0,0,114,1,0,0,0,114,86,0,0,0,114,93,0, + 0,0,90,7,95,79,82,73,71,73,78,218,10,95,95,99, + 97,99,104,101,100,95,95,218,4,108,105,115,116,218,8,95, + 95,112,97,116,104,95,95,114,103,0,0,0,114,108,0,0, + 0,114,113,0,0,0,114,107,0,0,0,41,8,114,84,0, + 0,0,114,94,0,0,0,114,104,0,0,0,114,83,0,0, + 0,114,15,0,0,0,90,8,108,111,99,97,116,105,111,110, + 114,113,0,0,0,114,107,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,17,95,115,112,101,99, + 95,102,114,111,109,95,109,111,100,117,108,101,196,1,0,0, + 115,72,0,0,0,0,2,2,1,10,1,14,1,6,2,8, + 1,4,2,6,1,8,1,2,1,10,1,14,2,6,1,2, + 1,10,1,14,1,10,1,8,1,8,1,2,1,10,1,14, + 1,12,2,4,1,2,1,10,1,14,1,10,1,2,1,14, + 1,16,1,10,2,14,1,20,1,6,1,6,1,114,129,0, + 0,0,70,41,1,218,8,111,118,101,114,114,105,100,101,99, + 2,0,0,0,1,0,0,0,5,0,0,0,59,0,0,0, + 67,0,0,0,115,212,1,0,0,124,2,115,20,116,0,124, + 1,100,1,100,0,131,3,100,0,107,8,114,54,121,12,124, + 0,106,1,124,1,95,2,87,0,110,20,4,0,116,3,107, + 10,114,52,1,0,1,0,1,0,89,0,110,2,88,0,124, + 2,115,74,116,0,124,1,100,2,100,0,131,3,100,0,107, + 8,114,166,124,0,106,4,125,3,124,3,100,0,107,8,114, + 134,124,0,106,5,100,0,107,9,114,134,116,6,100,0,107, + 8,114,110,116,7,130,1,116,6,106,8,125,4,124,4,106, + 9,124,4,131,1,125,3,124,0,106,5,124,3,95,10,121, + 10,124,3,124,1,95,11,87,0,110,20,4,0,116,3,107, + 10,114,164,1,0,1,0,1,0,89,0,110,2,88,0,124, + 2,115,186,116,0,124,1,100,3,100,0,131,3,100,0,107, + 8,114,220,121,12,124,0,106,12,124,1,95,13,87,0,110, + 20,4,0,116,3,107,10,114,218,1,0,1,0,1,0,89, + 0,110,2,88,0,121,10,124,0,124,1,95,14,87,0,110, + 20,4,0,116,3,107,10,114,250,1,0,1,0,1,0,89, + 0,110,2,88,0,124,2,144,1,115,20,116,0,124,1,100, + 4,100,0,131,3,100,0,107,8,144,1,114,68,124,0,106, + 5,100,0,107,9,144,1,114,68,121,12,124,0,106,5,124, + 1,95,15,87,0,110,22,4,0,116,3,107,10,144,1,114, + 66,1,0,1,0,1,0,89,0,110,2,88,0,124,0,106, + 16,144,1,114,208,124,2,144,1,115,100,116,0,124,1,100, + 5,100,0,131,3,100,0,107,8,144,1,114,136,121,12,124, + 0,106,17,124,1,95,18,87,0,110,22,4,0,116,3,107, + 10,144,1,114,134,1,0,1,0,1,0,89,0,110,2,88, + 0,124,2,144,1,115,160,116,0,124,1,100,6,100,0,131, + 3,100,0,107,8,144,1,114,208,124,0,106,19,100,0,107, + 9,144,1,114,208,121,12,124,0,106,19,124,1,95,20,87, + 0,110,22,4,0,116,3,107,10,144,1,114,206,1,0,1, + 0,1,0,89,0,110,2,88,0,124,1,83,0,41,7,78, + 114,1,0,0,0,114,86,0,0,0,218,11,95,95,112,97, + 99,107,97,103,101,95,95,114,128,0,0,0,114,93,0,0, + 0,114,126,0,0,0,41,21,114,6,0,0,0,114,15,0, + 0,0,114,1,0,0,0,114,91,0,0,0,114,94,0,0, + 0,114,107,0,0,0,114,116,0,0,0,114,117,0,0,0, + 218,16,95,78,97,109,101,115,112,97,99,101,76,111,97,100, + 101,114,218,7,95,95,110,101,119,95,95,90,5,95,112,97, + 116,104,114,86,0,0,0,114,120,0,0,0,114,131,0,0, + 0,114,90,0,0,0,114,128,0,0,0,114,114,0,0,0, + 114,104,0,0,0,114,93,0,0,0,114,113,0,0,0,114, + 126,0,0,0,41,5,114,83,0,0,0,114,84,0,0,0, + 114,130,0,0,0,114,94,0,0,0,114,132,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,18, + 95,105,110,105,116,95,109,111,100,117,108,101,95,97,116,116, + 114,115,241,1,0,0,115,92,0,0,0,0,4,20,1,2, + 1,12,1,14,1,6,2,20,1,6,1,8,2,10,1,8, + 1,4,1,6,2,10,1,8,1,2,1,10,1,14,1,6, + 2,20,1,2,1,12,1,14,1,6,2,2,1,10,1,14, + 1,6,2,24,1,12,1,2,1,12,1,16,1,6,2,8, + 1,24,1,2,1,12,1,16,1,6,2,24,1,12,1,2, + 1,12,1,16,1,6,1,114,134,0,0,0,99,1,0,0, + 0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,0, + 0,115,82,0,0,0,100,1,125,1,116,0,124,0,106,1, + 100,2,131,2,114,30,124,0,106,1,106,2,124,0,131,1, + 125,1,110,20,116,0,124,0,106,1,100,3,131,2,114,50, + 116,3,100,4,131,1,130,1,124,1,100,1,107,8,114,68, + 116,4,124,0,106,5,131,1,125,1,116,6,124,0,124,1, + 131,2,1,0,124,1,83,0,41,5,122,43,67,114,101,97, + 116,101,32,97,32,109,111,100,117,108,101,32,98,97,115,101, + 100,32,111,110,32,116,104,101,32,112,114,111,118,105,100,101, + 100,32,115,112,101,99,46,78,218,13,99,114,101,97,116,101, + 95,109,111,100,117,108,101,218,11,101,120,101,99,95,109,111, + 100,117,108,101,122,66,108,111,97,100,101,114,115,32,116,104, + 97,116,32,100,101,102,105,110,101,32,101,120,101,99,95,109, + 111,100,117,108,101,40,41,32,109,117,115,116,32,97,108,115, + 111,32,100,101,102,105,110,101,32,99,114,101,97,116,101,95, + 109,111,100,117,108,101,40,41,41,7,114,4,0,0,0,114, + 94,0,0,0,114,135,0,0,0,114,71,0,0,0,114,16, + 0,0,0,114,15,0,0,0,114,134,0,0,0,41,2,114, + 83,0,0,0,114,84,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,16,109,111,100,117,108,101, + 95,102,114,111,109,95,115,112,101,99,45,2,0,0,115,18, + 0,0,0,0,3,4,1,12,3,14,1,12,1,8,2,8, + 1,10,1,10,1,114,137,0,0,0,99,1,0,0,0,0, + 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, + 106,0,0,0,124,0,106,0,100,1,107,8,114,14,100,2, + 110,4,124,0,106,0,125,1,124,0,106,1,100,1,107,8, + 114,66,124,0,106,2,100,1,107,8,114,50,100,3,106,3, + 124,1,131,1,83,0,100,4,106,3,124,1,124,0,106,2, + 131,2,83,0,110,36,124,0,106,4,114,86,100,5,106,3, + 124,1,124,0,106,1,131,2,83,0,100,6,106,3,124,0, + 106,0,124,0,106,1,131,2,83,0,100,1,83,0,41,7, + 122,38,82,101,116,117,114,110,32,116,104,101,32,114,101,112, + 114,32,116,111,32,117,115,101,32,102,111,114,32,116,104,101, + 32,109,111,100,117,108,101,46,78,114,88,0,0,0,122,13, + 60,109,111,100,117,108,101,32,123,33,114,125,62,122,20,60, + 109,111,100,117,108,101,32,123,33,114,125,32,40,123,33,114, + 125,41,62,122,23,60,109,111,100,117,108,101,32,123,33,114, + 125,32,102,114,111,109,32,123,33,114,125,62,122,18,60,109, + 111,100,117,108,101,32,123,33,114,125,32,40,123,125,41,62, + 41,5,114,15,0,0,0,114,104,0,0,0,114,94,0,0, + 0,114,38,0,0,0,114,114,0,0,0,41,2,114,83,0, + 0,0,114,15,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,92,0,0,0,62,2,0,0,115, + 16,0,0,0,0,3,20,1,10,1,10,1,10,2,16,2, + 6,1,14,2,114,92,0,0,0,99,2,0,0,0,0,0, + 0,0,4,0,0,0,12,0,0,0,67,0,0,0,115,178, + 0,0,0,124,0,106,0,125,2,116,1,124,2,131,1,143, + 148,1,0,116,2,106,3,106,4,124,2,131,1,124,1,107, + 9,114,54,100,1,106,5,124,2,131,1,125,3,116,6,124, + 3,124,2,100,2,141,2,130,1,124,0,106,7,100,3,107, + 8,114,106,124,0,106,8,100,3,107,8,114,88,116,6,100, + 4,124,0,106,0,100,2,141,2,130,1,116,9,124,0,124, + 1,100,5,100,6,141,3,1,0,124,1,83,0,116,9,124, + 0,124,1,100,5,100,6,141,3,1,0,116,10,124,0,106, + 7,100,7,131,2,115,146,124,0,106,7,106,11,124,2,131, + 1,1,0,110,12,124,0,106,7,106,12,124,1,131,1,1, + 0,87,0,100,3,81,0,82,0,88,0,116,2,106,3,124, + 2,25,0,83,0,41,8,122,70,69,120,101,99,117,116,101, + 32,116,104,101,32,115,112,101,99,39,115,32,115,112,101,99, + 105,102,105,101,100,32,109,111,100,117,108,101,32,105,110,32, + 97,110,32,101,120,105,115,116,105,110,103,32,109,111,100,117, + 108,101,39,115,32,110,97,109,101,115,112,97,99,101,46,122, + 30,109,111,100,117,108,101,32,123,33,114,125,32,110,111,116, + 32,105,110,32,115,121,115,46,109,111,100,117,108,101,115,41, + 1,114,15,0,0,0,78,122,14,109,105,115,115,105,110,103, + 32,108,111,97,100,101,114,84,41,1,114,130,0,0,0,114, + 136,0,0,0,41,13,114,15,0,0,0,114,42,0,0,0, + 114,14,0,0,0,114,80,0,0,0,114,30,0,0,0,114, + 38,0,0,0,114,71,0,0,0,114,94,0,0,0,114,107, + 0,0,0,114,134,0,0,0,114,4,0,0,0,218,11,108, + 111,97,100,95,109,111,100,117,108,101,114,136,0,0,0,41, + 4,114,83,0,0,0,114,84,0,0,0,114,15,0,0,0, + 218,3,109,115,103,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,81,0,0,0,79,2,0,0,115,30,0, + 0,0,0,2,6,1,10,1,16,1,10,1,12,1,10,1, + 10,1,14,2,14,1,4,1,14,1,12,4,14,2,22,1, + 114,81,0,0,0,99,1,0,0,0,0,0,0,0,2,0, + 0,0,27,0,0,0,67,0,0,0,115,206,0,0,0,124, + 0,106,0,106,1,124,0,106,2,131,1,1,0,116,3,106, + 4,124,0,106,2,25,0,125,1,116,5,124,1,100,1,100, + 0,131,3,100,0,107,8,114,76,121,12,124,0,106,0,124, + 1,95,6,87,0,110,20,4,0,116,7,107,10,114,74,1, + 0,1,0,1,0,89,0,110,2,88,0,116,5,124,1,100, + 2,100,0,131,3,100,0,107,8,114,154,121,40,124,1,106, + 8,124,1,95,9,116,10,124,1,100,3,131,2,115,130,124, + 0,106,2,106,11,100,4,131,1,100,5,25,0,124,1,95, + 9,87,0,110,20,4,0,116,7,107,10,114,152,1,0,1, + 0,1,0,89,0,110,2,88,0,116,5,124,1,100,6,100, + 0,131,3,100,0,107,8,114,202,121,10,124,0,124,1,95, + 12,87,0,110,20,4,0,116,7,107,10,114,200,1,0,1, + 0,1,0,89,0,110,2,88,0,124,1,83,0,41,7,78, + 114,86,0,0,0,114,131,0,0,0,114,128,0,0,0,114, + 118,0,0,0,114,19,0,0,0,114,90,0,0,0,41,13, + 114,94,0,0,0,114,138,0,0,0,114,15,0,0,0,114, + 14,0,0,0,114,80,0,0,0,114,6,0,0,0,114,86, + 0,0,0,114,91,0,0,0,114,1,0,0,0,114,131,0, + 0,0,114,4,0,0,0,114,119,0,0,0,114,90,0,0, + 0,41,2,114,83,0,0,0,114,84,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,218,25,95,108, + 111,97,100,95,98,97,99,107,119,97,114,100,95,99,111,109, + 112,97,116,105,98,108,101,103,2,0,0,115,40,0,0,0, + 0,4,14,2,12,1,16,1,2,1,12,1,14,1,6,1, + 16,1,2,4,8,1,10,1,22,1,14,1,6,1,16,1, + 2,1,10,1,14,1,6,1,114,140,0,0,0,99,1,0, + 0,0,0,0,0,0,2,0,0,0,11,0,0,0,67,0, + 0,0,115,118,0,0,0,124,0,106,0,100,0,107,9,114, + 30,116,1,124,0,106,0,100,1,131,2,115,30,116,2,124, + 0,131,1,83,0,116,3,124,0,131,1,125,1,116,4,124, + 1,131,1,143,54,1,0,124,0,106,0,100,0,107,8,114, + 84,124,0,106,5,100,0,107,8,114,96,116,6,100,2,124, + 0,106,7,100,3,141,2,130,1,110,12,124,0,106,0,106, + 8,124,1,131,1,1,0,87,0,100,0,81,0,82,0,88, + 0,116,9,106,10,124,0,106,7,25,0,83,0,41,4,78, + 114,136,0,0,0,122,14,109,105,115,115,105,110,103,32,108, + 111,97,100,101,114,41,1,114,15,0,0,0,41,11,114,94, + 0,0,0,114,4,0,0,0,114,140,0,0,0,114,137,0, + 0,0,114,97,0,0,0,114,107,0,0,0,114,71,0,0, + 0,114,15,0,0,0,114,136,0,0,0,114,14,0,0,0, + 114,80,0,0,0,41,2,114,83,0,0,0,114,84,0,0, + 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, + 218,14,95,108,111,97,100,95,117,110,108,111,99,107,101,100, + 132,2,0,0,115,20,0,0,0,0,2,10,2,12,1,8, + 2,8,1,10,1,10,1,10,1,16,3,22,5,114,141,0, + 0,0,99,1,0,0,0,0,0,0,0,1,0,0,0,9, + 0,0,0,67,0,0,0,115,30,0,0,0,116,0,124,0, + 106,1,131,1,143,10,1,0,116,2,124,0,131,1,83,0, 81,0,82,0,88,0,100,1,83,0,41,2,122,191,82,101, 116,117,114,110,32,97,32,110,101,119,32,109,111,100,117,108, 101,32,111,98,106,101,99,116,44,32,108,111,97,100,101,100, @@ -1053,768 +1047,766 @@ const unsigned char _Py_M__importlib[] = { 110,32,115,121,115,46,109,111,100,117,108,101,115,44,32,116, 104,97,116,32,101,120,105,115,116,105,110,103,32,109,111,100, 117,108,101,32,103,101,116,115,10,32,32,32,32,99,108,111, - 98,98,101,114,101,100,46,10,10,32,32,32,32,78,41,5, - 114,46,0,0,0,114,137,0,0,0,114,42,0,0,0,114, - 15,0,0,0,114,141,0,0,0,41,1,114,82,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 81,0,0,0,152,2,0,0,115,6,0,0,0,0,9,8, - 1,12,1,114,81,0,0,0,99,0,0,0,0,0,0,0, - 0,0,0,0,0,4,0,0,0,64,0,0,0,115,136,0, + 98,98,101,114,101,100,46,10,10,32,32,32,32,78,41,3, + 114,42,0,0,0,114,15,0,0,0,114,141,0,0,0,41, + 1,114,83,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,82,0,0,0,155,2,0,0,115,4, + 0,0,0,0,9,12,1,114,82,0,0,0,99,0,0,0, + 0,0,0,0,0,0,0,0,0,4,0,0,0,64,0,0, + 0,115,136,0,0,0,101,0,90,1,100,0,90,2,100,1, + 90,3,101,4,100,2,100,3,132,0,131,1,90,5,101,6, + 100,19,100,5,100,6,132,1,131,1,90,7,101,6,100,20, + 100,7,100,8,132,1,131,1,90,8,101,6,100,9,100,10, + 132,0,131,1,90,9,101,6,100,11,100,12,132,0,131,1, + 90,10,101,6,101,11,100,13,100,14,132,0,131,1,131,1, + 90,12,101,6,101,11,100,15,100,16,132,0,131,1,131,1, + 90,13,101,6,101,11,100,17,100,18,132,0,131,1,131,1, + 90,14,101,6,101,15,131,1,90,16,100,4,83,0,41,21, + 218,15,66,117,105,108,116,105,110,73,109,112,111,114,116,101, + 114,122,144,77,101,116,97,32,112,97,116,104,32,105,109,112, + 111,114,116,32,102,111,114,32,98,117,105,108,116,45,105,110, + 32,109,111,100,117,108,101,115,46,10,10,32,32,32,32,65, + 108,108,32,109,101,116,104,111,100,115,32,97,114,101,32,101, + 105,116,104,101,114,32,99,108,97,115,115,32,111,114,32,115, + 116,97,116,105,99,32,109,101,116,104,111,100,115,32,116,111, + 32,97,118,111,105,100,32,116,104,101,32,110,101,101,100,32, + 116,111,10,32,32,32,32,105,110,115,116,97,110,116,105,97, + 116,101,32,116,104,101,32,99,108,97,115,115,46,10,10,32, + 32,32,32,99,1,0,0,0,0,0,0,0,1,0,0,0, + 2,0,0,0,67,0,0,0,115,12,0,0,0,100,1,106, + 0,124,0,106,1,131,1,83,0,41,2,122,115,82,101,116, + 117,114,110,32,114,101,112,114,32,102,111,114,32,116,104,101, + 32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32, + 32,32,84,104,101,32,109,101,116,104,111,100,32,105,115,32, + 100,101,112,114,101,99,97,116,101,100,46,32,32,84,104,101, + 32,105,109,112,111,114,116,32,109,97,99,104,105,110,101,114, + 121,32,100,111,101,115,32,116,104,101,32,106,111,98,32,105, + 116,115,101,108,102,46,10,10,32,32,32,32,32,32,32,32, + 122,24,60,109,111,100,117,108,101,32,123,33,114,125,32,40, + 98,117,105,108,116,45,105,110,41,62,41,2,114,38,0,0, + 0,114,1,0,0,0,41,1,114,84,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,114,87,0,0, + 0,179,2,0,0,115,2,0,0,0,0,7,122,27,66,117, + 105,108,116,105,110,73,109,112,111,114,116,101,114,46,109,111, + 100,117,108,101,95,114,101,112,114,78,99,4,0,0,0,0, + 0,0,0,4,0,0,0,5,0,0,0,67,0,0,0,115, + 44,0,0,0,124,2,100,0,107,9,114,12,100,0,83,0, + 116,0,106,1,124,1,131,1,114,36,116,2,124,1,124,0, + 100,1,100,2,141,3,83,0,100,0,83,0,100,0,83,0, + 41,3,78,122,8,98,117,105,108,116,45,105,110,41,1,114, + 104,0,0,0,41,3,114,52,0,0,0,90,10,105,115,95, + 98,117,105,108,116,105,110,114,79,0,0,0,41,4,218,3, + 99,108,115,114,72,0,0,0,218,4,112,97,116,104,218,6, + 116,97,114,103,101,116,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,9,102,105,110,100,95,115,112,101,99, + 188,2,0,0,115,10,0,0,0,0,2,8,1,4,1,10, + 1,14,2,122,25,66,117,105,108,116,105,110,73,109,112,111, + 114,116,101,114,46,102,105,110,100,95,115,112,101,99,99,3, + 0,0,0,0,0,0,0,4,0,0,0,3,0,0,0,67, + 0,0,0,115,30,0,0,0,124,0,106,0,124,1,124,2, + 131,2,125,3,124,3,100,1,107,9,114,26,124,3,106,1, + 83,0,100,1,83,0,41,2,122,175,70,105,110,100,32,116, + 104,101,32,98,117,105,108,116,45,105,110,32,109,111,100,117, + 108,101,46,10,10,32,32,32,32,32,32,32,32,73,102,32, + 39,112,97,116,104,39,32,105,115,32,101,118,101,114,32,115, + 112,101,99,105,102,105,101,100,32,116,104,101,110,32,116,104, + 101,32,115,101,97,114,99,104,32,105,115,32,99,111,110,115, + 105,100,101,114,101,100,32,97,32,102,97,105,108,117,114,101, + 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, + 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, + 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, + 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, + 10,32,32,32,32,32,32,32,32,78,41,2,114,146,0,0, + 0,114,94,0,0,0,41,4,114,143,0,0,0,114,72,0, + 0,0,114,144,0,0,0,114,83,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,11,102,105,110, + 100,95,109,111,100,117,108,101,197,2,0,0,115,4,0,0, + 0,0,9,12,1,122,27,66,117,105,108,116,105,110,73,109, + 112,111,114,116,101,114,46,102,105,110,100,95,109,111,100,117, + 108,101,99,2,0,0,0,0,0,0,0,2,0,0,0,4, + 0,0,0,67,0,0,0,115,46,0,0,0,124,1,106,0, + 116,1,106,2,107,7,114,34,116,3,100,1,106,4,124,1, + 106,0,131,1,124,1,106,0,100,2,141,2,130,1,116,5, + 116,6,106,7,124,1,131,2,83,0,41,3,122,24,67,114, + 101,97,116,101,32,97,32,98,117,105,108,116,45,105,110,32, + 109,111,100,117,108,101,122,29,123,33,114,125,32,105,115,32, + 110,111,116,32,97,32,98,117,105,108,116,45,105,110,32,109, + 111,100,117,108,101,41,1,114,15,0,0,0,41,8,114,15, + 0,0,0,114,14,0,0,0,114,70,0,0,0,114,71,0, + 0,0,114,38,0,0,0,114,59,0,0,0,114,52,0,0, + 0,90,14,99,114,101,97,116,101,95,98,117,105,108,116,105, + 110,41,2,114,26,0,0,0,114,83,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,114,135,0,0, + 0,209,2,0,0,115,8,0,0,0,0,3,12,1,12,1, + 10,1,122,29,66,117,105,108,116,105,110,73,109,112,111,114, + 116,101,114,46,99,114,101,97,116,101,95,109,111,100,117,108, + 101,99,2,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,67,0,0,0,115,16,0,0,0,116,0,116,1,106, + 2,124,1,131,2,1,0,100,1,83,0,41,2,122,22,69, + 120,101,99,32,97,32,98,117,105,108,116,45,105,110,32,109, + 111,100,117,108,101,78,41,3,114,59,0,0,0,114,52,0, + 0,0,90,12,101,120,101,99,95,98,117,105,108,116,105,110, + 41,2,114,26,0,0,0,114,84,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,136,0,0,0, + 217,2,0,0,115,2,0,0,0,0,3,122,27,66,117,105, + 108,116,105,110,73,109,112,111,114,116,101,114,46,101,120,101, + 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, + 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, + 0,0,100,1,83,0,41,2,122,57,82,101,116,117,114,110, + 32,78,111,110,101,32,97,115,32,98,117,105,108,116,45,105, + 110,32,109,111,100,117,108,101,115,32,100,111,32,110,111,116, + 32,104,97,118,101,32,99,111,100,101,32,111,98,106,101,99, + 116,115,46,78,114,10,0,0,0,41,2,114,143,0,0,0, + 114,72,0,0,0,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,218,8,103,101,116,95,99,111,100,101,222,2, + 0,0,115,2,0,0,0,0,4,122,24,66,117,105,108,116, + 105,110,73,109,112,111,114,116,101,114,46,103,101,116,95,99, + 111,100,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,83, + 0,41,2,122,56,82,101,116,117,114,110,32,78,111,110,101, + 32,97,115,32,98,117,105,108,116,45,105,110,32,109,111,100, + 117,108,101,115,32,100,111,32,110,111,116,32,104,97,118,101, + 32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,10, + 0,0,0,41,2,114,143,0,0,0,114,72,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,10, + 103,101,116,95,115,111,117,114,99,101,228,2,0,0,115,2, + 0,0,0,0,4,122,26,66,117,105,108,116,105,110,73,109, + 112,111,114,116,101,114,46,103,101,116,95,115,111,117,114,99, + 101,99,2,0,0,0,0,0,0,0,2,0,0,0,1,0, + 0,0,67,0,0,0,115,4,0,0,0,100,1,83,0,41, + 2,122,52,82,101,116,117,114,110,32,70,97,108,115,101,32, + 97,115,32,98,117,105,108,116,45,105,110,32,109,111,100,117, + 108,101,115,32,97,114,101,32,110,101,118,101,114,32,112,97, + 99,107,97,103,101,115,46,70,114,10,0,0,0,41,2,114, + 143,0,0,0,114,72,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,106,0,0,0,234,2,0, + 0,115,2,0,0,0,0,4,122,26,66,117,105,108,116,105, + 110,73,109,112,111,114,116,101,114,46,105,115,95,112,97,99, + 107,97,103,101,41,2,78,78,41,1,78,41,17,114,1,0, + 0,0,114,0,0,0,0,114,2,0,0,0,114,3,0,0, + 0,218,12,115,116,97,116,105,99,109,101,116,104,111,100,114, + 87,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, + 100,114,146,0,0,0,114,147,0,0,0,114,135,0,0,0, + 114,136,0,0,0,114,75,0,0,0,114,148,0,0,0,114, + 149,0,0,0,114,106,0,0,0,114,85,0,0,0,114,138, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,114,142,0,0,0,170,2,0,0, + 115,30,0,0,0,8,7,4,2,12,9,2,1,12,8,2, + 1,12,11,12,8,12,5,2,1,14,5,2,1,14,5,2, + 1,14,5,114,142,0,0,0,99,0,0,0,0,0,0,0, + 0,0,0,0,0,4,0,0,0,64,0,0,0,115,140,0, 0,0,101,0,90,1,100,0,90,2,100,1,90,3,101,4, - 100,2,100,3,132,0,131,1,90,5,101,6,100,19,100,5, - 100,6,132,1,131,1,90,7,101,6,100,20,100,7,100,8, + 100,2,100,3,132,0,131,1,90,5,101,6,100,21,100,5, + 100,6,132,1,131,1,90,7,101,6,100,22,100,7,100,8, 132,1,131,1,90,8,101,6,100,9,100,10,132,0,131,1, - 90,9,101,6,100,11,100,12,132,0,131,1,90,10,101,6, - 101,11,100,13,100,14,132,0,131,1,131,1,90,12,101,6, - 101,11,100,15,100,16,132,0,131,1,131,1,90,13,101,6, - 101,11,100,17,100,18,132,0,131,1,131,1,90,14,101,6, - 101,15,131,1,90,16,100,4,83,0,41,21,218,15,66,117, - 105,108,116,105,110,73,109,112,111,114,116,101,114,122,144,77, - 101,116,97,32,112,97,116,104,32,105,109,112,111,114,116,32, - 102,111,114,32,98,117,105,108,116,45,105,110,32,109,111,100, - 117,108,101,115,46,10,10,32,32,32,32,65,108,108,32,109, - 101,116,104,111,100,115,32,97,114,101,32,101,105,116,104,101, - 114,32,99,108,97,115,115,32,111,114,32,115,116,97,116,105, - 99,32,109,101,116,104,111,100,115,32,116,111,32,97,118,111, - 105,100,32,116,104,101,32,110,101,101,100,32,116,111,10,32, - 32,32,32,105,110,115,116,97,110,116,105,97,116,101,32,116, - 104,101,32,99,108,97,115,115,46,10,10,32,32,32,32,99, - 1,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0, - 67,0,0,0,115,12,0,0,0,100,1,106,0,124,0,106, - 1,131,1,83,0,41,2,122,115,82,101,116,117,114,110,32, - 114,101,112,114,32,102,111,114,32,116,104,101,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, - 101,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,84,104,101,32,105,109,112, - 111,114,116,32,109,97,99,104,105,110,101,114,121,32,100,111, - 101,115,32,116,104,101,32,106,111,98,32,105,116,115,101,108, - 102,46,10,10,32,32,32,32,32,32,32,32,122,24,60,109, - 111,100,117,108,101,32,123,33,114,125,32,40,98,117,105,108, - 116,45,105,110,41,62,41,2,114,38,0,0,0,114,1,0, - 0,0,41,1,114,83,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,86,0,0,0,177,2,0, - 0,115,2,0,0,0,0,7,122,27,66,117,105,108,116,105, - 110,73,109,112,111,114,116,101,114,46,109,111,100,117,108,101, - 95,114,101,112,114,78,99,4,0,0,0,0,0,0,0,4, - 0,0,0,5,0,0,0,67,0,0,0,115,44,0,0,0, - 124,2,100,0,107,9,114,12,100,0,83,0,116,0,106,1, - 124,1,131,1,114,36,116,2,124,1,124,0,100,1,100,2, - 141,3,83,0,100,0,83,0,100,0,83,0,41,3,78,122, - 8,98,117,105,108,116,45,105,110,41,1,114,103,0,0,0, - 41,3,114,46,0,0,0,90,10,105,115,95,98,117,105,108, - 116,105,110,114,78,0,0,0,41,4,218,3,99,108,115,114, - 71,0,0,0,218,4,112,97,116,104,218,6,116,97,114,103, - 101,116,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,9,102,105,110,100,95,115,112,101,99,186,2,0,0, - 115,10,0,0,0,0,2,8,1,4,1,10,1,14,2,122, - 25,66,117,105,108,116,105,110,73,109,112,111,114,116,101,114, - 46,102,105,110,100,95,115,112,101,99,99,3,0,0,0,0, - 0,0,0,4,0,0,0,3,0,0,0,67,0,0,0,115, - 30,0,0,0,124,0,106,0,124,1,124,2,131,2,125,3, - 124,3,100,1,107,9,114,26,124,3,106,1,83,0,100,1, - 83,0,41,2,122,175,70,105,110,100,32,116,104,101,32,98, - 117,105,108,116,45,105,110,32,109,111,100,117,108,101,46,10, - 10,32,32,32,32,32,32,32,32,73,102,32,39,112,97,116, - 104,39,32,105,115,32,101,118,101,114,32,115,112,101,99,105, - 102,105,101,100,32,116,104,101,110,32,116,104,101,32,115,101, - 97,114,99,104,32,105,115,32,99,111,110,115,105,100,101,114, - 101,100,32,97,32,102,97,105,108,117,114,101,46,10,10,32, - 32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,104, - 111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,100, - 46,32,32,85,115,101,32,102,105,110,100,95,115,112,101,99, - 40,41,32,105,110,115,116,101,97,100,46,10,10,32,32,32, - 32,32,32,32,32,78,41,2,114,146,0,0,0,114,93,0, - 0,0,41,4,114,143,0,0,0,114,71,0,0,0,114,144, - 0,0,0,114,82,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,11,102,105,110,100,95,109,111, - 100,117,108,101,195,2,0,0,115,4,0,0,0,0,9,12, - 1,122,27,66,117,105,108,116,105,110,73,109,112,111,114,116, - 101,114,46,102,105,110,100,95,109,111,100,117,108,101,99,2, - 0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,67, - 0,0,0,115,46,0,0,0,124,1,106,0,116,1,106,2, - 107,7,114,34,116,3,100,1,106,4,124,1,106,0,131,1, - 124,1,106,0,100,2,141,2,130,1,116,5,116,6,106,7, - 124,1,131,2,83,0,41,3,122,24,67,114,101,97,116,101, - 32,97,32,98,117,105,108,116,45,105,110,32,109,111,100,117, - 108,101,122,29,123,33,114,125,32,105,115,32,110,111,116,32, - 97,32,98,117,105,108,116,45,105,110,32,109,111,100,117,108, - 101,41,1,114,15,0,0,0,41,8,114,15,0,0,0,114, - 14,0,0,0,114,69,0,0,0,114,70,0,0,0,114,38, - 0,0,0,114,58,0,0,0,114,46,0,0,0,90,14,99, - 114,101,97,116,101,95,98,117,105,108,116,105,110,41,2,114, - 26,0,0,0,114,82,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,134,0,0,0,207,2,0, - 0,115,8,0,0,0,0,3,12,1,12,1,10,1,122,29, - 66,117,105,108,116,105,110,73,109,112,111,114,116,101,114,46, - 99,114,101,97,116,101,95,109,111,100,117,108,101,99,2,0, - 0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0, - 0,0,115,16,0,0,0,116,0,116,1,106,2,124,1,131, - 2,1,0,100,1,83,0,41,2,122,22,69,120,101,99,32, - 97,32,98,117,105,108,116,45,105,110,32,109,111,100,117,108, - 101,78,41,3,114,58,0,0,0,114,46,0,0,0,90,12, - 101,120,101,99,95,98,117,105,108,116,105,110,41,2,114,26, - 0,0,0,114,83,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,114,135,0,0,0,215,2,0,0, - 115,2,0,0,0,0,3,122,27,66,117,105,108,116,105,110, - 73,109,112,111,114,116,101,114,46,101,120,101,99,95,109,111, - 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 83,0,41,2,122,57,82,101,116,117,114,110,32,78,111,110, - 101,32,97,115,32,98,117,105,108,116,45,105,110,32,109,111, + 90,9,101,4,100,11,100,12,132,0,131,1,90,10,101,6, + 100,13,100,14,132,0,131,1,90,11,101,6,101,12,100,15, + 100,16,132,0,131,1,131,1,90,13,101,6,101,12,100,17, + 100,18,132,0,131,1,131,1,90,14,101,6,101,12,100,19, + 100,20,132,0,131,1,131,1,90,15,100,4,83,0,41,23, + 218,14,70,114,111,122,101,110,73,109,112,111,114,116,101,114, + 122,142,77,101,116,97,32,112,97,116,104,32,105,109,112,111, + 114,116,32,102,111,114,32,102,114,111,122,101,110,32,109,111, + 100,117,108,101,115,46,10,10,32,32,32,32,65,108,108,32, + 109,101,116,104,111,100,115,32,97,114,101,32,101,105,116,104, + 101,114,32,99,108,97,115,115,32,111,114,32,115,116,97,116, + 105,99,32,109,101,116,104,111,100,115,32,116,111,32,97,118, + 111,105,100,32,116,104,101,32,110,101,101,100,32,116,111,10, + 32,32,32,32,105,110,115,116,97,110,116,105,97,116,101,32, + 116,104,101,32,99,108,97,115,115,46,10,10,32,32,32,32, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,12,0,0,0,100,1,106,0,124,0, + 106,1,131,1,83,0,41,2,122,115,82,101,116,117,114,110, + 32,114,101,112,114,32,102,111,114,32,116,104,101,32,109,111, + 100,117,108,101,46,10,10,32,32,32,32,32,32,32,32,84, + 104,101,32,109,101,116,104,111,100,32,105,115,32,100,101,112, + 114,101,99,97,116,101,100,46,32,32,84,104,101,32,105,109, + 112,111,114,116,32,109,97,99,104,105,110,101,114,121,32,100, + 111,101,115,32,116,104,101,32,106,111,98,32,105,116,115,101, + 108,102,46,10,10,32,32,32,32,32,32,32,32,122,22,60, + 109,111,100,117,108,101,32,123,33,114,125,32,40,102,114,111, + 122,101,110,41,62,41,2,114,38,0,0,0,114,1,0,0, + 0,41,1,218,1,109,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,87,0,0,0,252,2,0,0,115,2, + 0,0,0,0,7,122,26,70,114,111,122,101,110,73,109,112, + 111,114,116,101,114,46,109,111,100,117,108,101,95,114,101,112, + 114,78,99,4,0,0,0,0,0,0,0,4,0,0,0,5, + 0,0,0,67,0,0,0,115,32,0,0,0,116,0,106,1, + 124,1,131,1,114,24,116,2,124,1,124,0,100,1,100,2, + 141,3,83,0,100,0,83,0,100,0,83,0,41,3,78,90, + 6,102,114,111,122,101,110,41,1,114,104,0,0,0,41,3, + 114,52,0,0,0,114,76,0,0,0,114,79,0,0,0,41, + 4,114,143,0,0,0,114,72,0,0,0,114,144,0,0,0, + 114,145,0,0,0,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,146,0,0,0,5,3,0,0,115,6,0, + 0,0,0,2,10,1,14,2,122,24,70,114,111,122,101,110, + 73,109,112,111,114,116,101,114,46,102,105,110,100,95,115,112, + 101,99,99,3,0,0,0,0,0,0,0,3,0,0,0,2, + 0,0,0,67,0,0,0,115,18,0,0,0,116,0,106,1, + 124,1,131,1,114,14,124,0,83,0,100,1,83,0,41,2, + 122,93,70,105,110,100,32,97,32,102,114,111,122,101,110,32, + 109,111,100,117,108,101,46,10,10,32,32,32,32,32,32,32, + 32,84,104,105,115,32,109,101,116,104,111,100,32,105,115,32, + 100,101,112,114,101,99,97,116,101,100,46,32,32,85,115,101, + 32,102,105,110,100,95,115,112,101,99,40,41,32,105,110,115, + 116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,78, + 41,2,114,52,0,0,0,114,76,0,0,0,41,3,114,143, + 0,0,0,114,72,0,0,0,114,144,0,0,0,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,114,147,0,0, + 0,12,3,0,0,115,2,0,0,0,0,7,122,26,70,114, + 111,122,101,110,73,109,112,111,114,116,101,114,46,102,105,110, + 100,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, + 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, + 0,0,100,1,83,0,41,2,122,42,85,115,101,32,100,101, + 102,97,117,108,116,32,115,101,109,97,110,116,105,99,115,32, + 102,111,114,32,109,111,100,117,108,101,32,99,114,101,97,116, + 105,111,110,46,78,114,10,0,0,0,41,2,114,143,0,0, + 0,114,83,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,135,0,0,0,21,3,0,0,115,0, + 0,0,0,122,28,70,114,111,122,101,110,73,109,112,111,114, + 116,101,114,46,99,114,101,97,116,101,95,109,111,100,117,108, + 101,99,1,0,0,0,0,0,0,0,3,0,0,0,4,0, + 0,0,67,0,0,0,115,64,0,0,0,124,0,106,0,106, + 1,125,1,116,2,106,3,124,1,131,1,115,36,116,4,100, + 1,106,5,124,1,131,1,124,1,100,2,141,2,130,1,116, + 6,116,2,106,7,124,1,131,2,125,2,116,8,124,2,124, + 0,106,9,131,2,1,0,100,0,83,0,41,3,78,122,27, + 123,33,114,125,32,105,115,32,110,111,116,32,97,32,102,114, + 111,122,101,110,32,109,111,100,117,108,101,41,1,114,15,0, + 0,0,41,10,114,90,0,0,0,114,15,0,0,0,114,52, + 0,0,0,114,76,0,0,0,114,71,0,0,0,114,38,0, + 0,0,114,59,0,0,0,218,17,103,101,116,95,102,114,111, + 122,101,110,95,111,98,106,101,99,116,218,4,101,120,101,99, + 114,7,0,0,0,41,3,114,84,0,0,0,114,15,0,0, + 0,218,4,99,111,100,101,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,136,0,0,0,25,3,0,0,115, + 12,0,0,0,0,2,8,1,10,1,10,1,8,1,12,1, + 122,26,70,114,111,122,101,110,73,109,112,111,114,116,101,114, + 46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,0, + 0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,0, + 0,115,10,0,0,0,116,0,124,0,124,1,131,2,83,0, + 41,1,122,95,76,111,97,100,32,97,32,102,114,111,122,101, + 110,32,109,111,100,117,108,101,46,10,10,32,32,32,32,32, + 32,32,32,84,104,105,115,32,109,101,116,104,111,100,32,105, + 115,32,100,101,112,114,101,99,97,116,101,100,46,32,32,85, + 115,101,32,101,120,101,99,95,109,111,100,117,108,101,40,41, + 32,105,110,115,116,101,97,100,46,10,10,32,32,32,32,32, + 32,32,32,41,1,114,85,0,0,0,41,2,114,143,0,0, + 0,114,72,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,138,0,0,0,34,3,0,0,115,2, + 0,0,0,0,7,122,26,70,114,111,122,101,110,73,109,112, + 111,114,116,101,114,46,108,111,97,100,95,109,111,100,117,108, + 101,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, + 0,0,67,0,0,0,115,10,0,0,0,116,0,106,1,124, + 1,131,1,83,0,41,1,122,45,82,101,116,117,114,110,32, + 116,104,101,32,99,111,100,101,32,111,98,106,101,99,116,32, + 102,111,114,32,116,104,101,32,102,114,111,122,101,110,32,109, + 111,100,117,108,101,46,41,2,114,52,0,0,0,114,154,0, + 0,0,41,2,114,143,0,0,0,114,72,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,148,0, + 0,0,43,3,0,0,115,2,0,0,0,0,4,122,23,70, + 114,111,122,101,110,73,109,112,111,114,116,101,114,46,103,101, + 116,95,99,111,100,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, + 100,1,83,0,41,2,122,54,82,101,116,117,114,110,32,78, + 111,110,101,32,97,115,32,102,114,111,122,101,110,32,109,111, 100,117,108,101,115,32,100,111,32,110,111,116,32,104,97,118, - 101,32,99,111,100,101,32,111,98,106,101,99,116,115,46,78, - 114,10,0,0,0,41,2,114,143,0,0,0,114,71,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,8,103,101,116,95,99,111,100,101,220,2,0,0,115,2, - 0,0,0,0,4,122,24,66,117,105,108,116,105,110,73,109, - 112,111,114,116,101,114,46,103,101,116,95,99,111,100,101,99, - 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, - 67,0,0,0,115,4,0,0,0,100,1,83,0,41,2,122, - 56,82,101,116,117,114,110,32,78,111,110,101,32,97,115,32, - 98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,115, - 32,100,111,32,110,111,116,32,104,97,118,101,32,115,111,117, - 114,99,101,32,99,111,100,101,46,78,114,10,0,0,0,41, - 2,114,143,0,0,0,114,71,0,0,0,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,218,10,103,101,116,95, - 115,111,117,114,99,101,226,2,0,0,115,2,0,0,0,0, - 4,122,26,66,117,105,108,116,105,110,73,109,112,111,114,116, - 101,114,46,103,101,116,95,115,111,117,114,99,101,99,2,0, - 0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,0, - 0,0,115,4,0,0,0,100,1,83,0,41,2,122,52,82, - 101,116,117,114,110,32,70,97,108,115,101,32,97,115,32,98, - 117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,32, - 97,114,101,32,110,101,118,101,114,32,112,97,99,107,97,103, - 101,115,46,70,114,10,0,0,0,41,2,114,143,0,0,0, - 114,71,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,114,105,0,0,0,232,2,0,0,115,2,0, - 0,0,0,4,122,26,66,117,105,108,116,105,110,73,109,112, + 101,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, + 10,0,0,0,41,2,114,143,0,0,0,114,72,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, + 149,0,0,0,49,3,0,0,115,2,0,0,0,0,4,122, + 25,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, + 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, + 10,0,0,0,116,0,106,1,124,1,131,1,83,0,41,1, + 122,46,82,101,116,117,114,110,32,84,114,117,101,32,105,102, + 32,116,104,101,32,102,114,111,122,101,110,32,109,111,100,117, + 108,101,32,105,115,32,97,32,112,97,99,107,97,103,101,46, + 41,2,114,52,0,0,0,90,17,105,115,95,102,114,111,122, + 101,110,95,112,97,99,107,97,103,101,41,2,114,143,0,0, + 0,114,72,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,106,0,0,0,55,3,0,0,115,2, + 0,0,0,0,4,122,25,70,114,111,122,101,110,73,109,112, 111,114,116,101,114,46,105,115,95,112,97,99,107,97,103,101, - 41,2,78,78,41,1,78,41,17,114,1,0,0,0,114,0, - 0,0,0,114,2,0,0,0,114,3,0,0,0,218,12,115, - 116,97,116,105,99,109,101,116,104,111,100,114,86,0,0,0, - 218,11,99,108,97,115,115,109,101,116,104,111,100,114,146,0, - 0,0,114,147,0,0,0,114,134,0,0,0,114,135,0,0, - 0,114,74,0,0,0,114,148,0,0,0,114,149,0,0,0, - 114,105,0,0,0,114,84,0,0,0,114,138,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,142,0,0,0,168,2,0,0,115,30,0,0, - 0,8,7,4,2,12,9,2,1,12,8,2,1,12,11,12, - 8,12,5,2,1,14,5,2,1,14,5,2,1,14,5,114, - 142,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, - 0,4,0,0,0,64,0,0,0,115,140,0,0,0,101,0, - 90,1,100,0,90,2,100,1,90,3,101,4,100,2,100,3, - 132,0,131,1,90,5,101,6,100,21,100,5,100,6,132,1, - 131,1,90,7,101,6,100,22,100,7,100,8,132,1,131,1, - 90,8,101,6,100,9,100,10,132,0,131,1,90,9,101,4, - 100,11,100,12,132,0,131,1,90,10,101,6,100,13,100,14, - 132,0,131,1,90,11,101,6,101,12,100,15,100,16,132,0, - 131,1,131,1,90,13,101,6,101,12,100,17,100,18,132,0, - 131,1,131,1,90,14,101,6,101,12,100,19,100,20,132,0, - 131,1,131,1,90,15,100,4,83,0,41,23,218,14,70,114, - 111,122,101,110,73,109,112,111,114,116,101,114,122,142,77,101, - 116,97,32,112,97,116,104,32,105,109,112,111,114,116,32,102, - 111,114,32,102,114,111,122,101,110,32,109,111,100,117,108,101, - 115,46,10,10,32,32,32,32,65,108,108,32,109,101,116,104, - 111,100,115,32,97,114,101,32,101,105,116,104,101,114,32,99, - 108,97,115,115,32,111,114,32,115,116,97,116,105,99,32,109, - 101,116,104,111,100,115,32,116,111,32,97,118,111,105,100,32, - 116,104,101,32,110,101,101,100,32,116,111,10,32,32,32,32, - 105,110,115,116,97,110,116,105,97,116,101,32,116,104,101,32, - 99,108,97,115,115,46,10,10,32,32,32,32,99,1,0,0, - 0,0,0,0,0,1,0,0,0,2,0,0,0,67,0,0, - 0,115,12,0,0,0,100,1,106,0,124,0,106,1,131,1, - 83,0,41,2,122,115,82,101,116,117,114,110,32,114,101,112, - 114,32,102,111,114,32,116,104,101,32,109,111,100,117,108,101, - 46,10,10,32,32,32,32,32,32,32,32,84,104,101,32,109, - 101,116,104,111,100,32,105,115,32,100,101,112,114,101,99,97, - 116,101,100,46,32,32,84,104,101,32,105,109,112,111,114,116, - 32,109,97,99,104,105,110,101,114,121,32,100,111,101,115,32, - 116,104,101,32,106,111,98,32,105,116,115,101,108,102,46,10, - 10,32,32,32,32,32,32,32,32,122,22,60,109,111,100,117, - 108,101,32,123,33,114,125,32,40,102,114,111,122,101,110,41, - 62,41,2,114,38,0,0,0,114,1,0,0,0,41,1,218, - 1,109,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,86,0,0,0,250,2,0,0,115,2,0,0,0,0, - 7,122,26,70,114,111,122,101,110,73,109,112,111,114,116,101, - 114,46,109,111,100,117,108,101,95,114,101,112,114,78,99,4, - 0,0,0,0,0,0,0,4,0,0,0,5,0,0,0,67, - 0,0,0,115,32,0,0,0,116,0,106,1,124,1,131,1, - 114,24,116,2,124,1,124,0,100,1,100,2,141,3,83,0, - 100,0,83,0,100,0,83,0,41,3,78,90,6,102,114,111, - 122,101,110,41,1,114,103,0,0,0,41,3,114,46,0,0, - 0,114,75,0,0,0,114,78,0,0,0,41,4,114,143,0, - 0,0,114,71,0,0,0,114,144,0,0,0,114,145,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,146,0,0,0,3,3,0,0,115,6,0,0,0,0,2, - 10,1,14,2,122,24,70,114,111,122,101,110,73,109,112,111, - 114,116,101,114,46,102,105,110,100,95,115,112,101,99,99,3, - 0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,67, - 0,0,0,115,18,0,0,0,116,0,106,1,124,1,131,1, - 114,14,124,0,83,0,100,1,83,0,41,2,122,93,70,105, - 110,100,32,97,32,102,114,111,122,101,110,32,109,111,100,117, - 108,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105, - 115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,85,115,101,32,102,105,110, - 100,95,115,112,101,99,40,41,32,105,110,115,116,101,97,100, - 46,10,10,32,32,32,32,32,32,32,32,78,41,2,114,46, - 0,0,0,114,75,0,0,0,41,3,114,143,0,0,0,114, - 71,0,0,0,114,144,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,147,0,0,0,10,3,0, - 0,115,2,0,0,0,0,7,122,26,70,114,111,122,101,110, - 73,109,112,111,114,116,101,114,46,102,105,110,100,95,109,111, - 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 83,0,41,2,122,42,85,115,101,32,100,101,102,97,117,108, - 116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,32, - 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, - 78,114,10,0,0,0,41,2,114,143,0,0,0,114,82,0, - 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,134,0,0,0,19,3,0,0,115,0,0,0,0,122, - 28,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, - 99,114,101,97,116,101,95,109,111,100,117,108,101,99,1,0, - 0,0,0,0,0,0,3,0,0,0,4,0,0,0,67,0, - 0,0,115,64,0,0,0,124,0,106,0,106,1,125,1,116, - 2,106,3,124,1,131,1,115,36,116,4,100,1,106,5,124, - 1,131,1,124,1,100,2,141,2,130,1,116,6,116,2,106, - 7,124,1,131,2,125,2,116,8,124,2,124,0,106,9,131, - 2,1,0,100,0,83,0,41,3,78,122,27,123,33,114,125, - 32,105,115,32,110,111,116,32,97,32,102,114,111,122,101,110, - 32,109,111,100,117,108,101,41,1,114,15,0,0,0,41,10, - 114,89,0,0,0,114,15,0,0,0,114,46,0,0,0,114, - 75,0,0,0,114,70,0,0,0,114,38,0,0,0,114,58, - 0,0,0,218,17,103,101,116,95,102,114,111,122,101,110,95, - 111,98,106,101,99,116,218,4,101,120,101,99,114,7,0,0, - 0,41,3,114,83,0,0,0,114,15,0,0,0,218,4,99, - 111,100,101,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,114,135,0,0,0,23,3,0,0,115,12,0,0,0, - 0,2,8,1,10,1,10,1,8,1,12,1,122,26,70,114, - 111,122,101,110,73,109,112,111,114,116,101,114,46,101,120,101, - 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, - 0,2,0,0,0,3,0,0,0,67,0,0,0,115,10,0, - 0,0,116,0,124,0,124,1,131,2,83,0,41,1,122,95, - 76,111,97,100,32,97,32,102,114,111,122,101,110,32,109,111, - 100,117,108,101,46,10,10,32,32,32,32,32,32,32,32,84, - 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, - 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,101, - 120,101,99,95,109,111,100,117,108,101,40,41,32,105,110,115, - 116,101,97,100,46,10,10,32,32,32,32,32,32,32,32,41, - 1,114,84,0,0,0,41,2,114,143,0,0,0,114,71,0, - 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,138,0,0,0,32,3,0,0,115,2,0,0,0,0, - 7,122,26,70,114,111,122,101,110,73,109,112,111,114,116,101, - 114,46,108,111,97,100,95,109,111,100,117,108,101,99,2,0, - 0,0,0,0,0,0,2,0,0,0,2,0,0,0,67,0, - 0,0,115,10,0,0,0,116,0,106,1,124,1,131,1,83, - 0,41,1,122,45,82,101,116,117,114,110,32,116,104,101,32, - 99,111,100,101,32,111,98,106,101,99,116,32,102,111,114,32, - 116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108, - 101,46,41,2,114,46,0,0,0,114,154,0,0,0,41,2, - 114,143,0,0,0,114,71,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,148,0,0,0,41,3, - 0,0,115,2,0,0,0,0,4,122,23,70,114,111,122,101, - 110,73,109,112,111,114,116,101,114,46,103,101,116,95,99,111, - 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, - 0,0,0,67,0,0,0,115,4,0,0,0,100,1,83,0, - 41,2,122,54,82,101,116,117,114,110,32,78,111,110,101,32, - 97,115,32,102,114,111,122,101,110,32,109,111,100,117,108,101, - 115,32,100,111,32,110,111,116,32,104,97,118,101,32,115,111, - 117,114,99,101,32,99,111,100,101,46,78,114,10,0,0,0, - 41,2,114,143,0,0,0,114,71,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,149,0,0,0, - 47,3,0,0,115,2,0,0,0,0,4,122,25,70,114,111, - 122,101,110,73,109,112,111,114,116,101,114,46,103,101,116,95, - 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, - 0,0,0,2,0,0,0,67,0,0,0,115,10,0,0,0, - 116,0,106,1,124,1,131,1,83,0,41,1,122,46,82,101, - 116,117,114,110,32,84,114,117,101,32,105,102,32,116,104,101, - 32,102,114,111,122,101,110,32,109,111,100,117,108,101,32,105, - 115,32,97,32,112,97,99,107,97,103,101,46,41,2,114,46, - 0,0,0,90,17,105,115,95,102,114,111,122,101,110,95,112, - 97,99,107,97,103,101,41,2,114,143,0,0,0,114,71,0, + 41,2,78,78,41,1,78,41,16,114,1,0,0,0,114,0, + 0,0,0,114,2,0,0,0,114,3,0,0,0,114,150,0, + 0,0,114,87,0,0,0,114,151,0,0,0,114,146,0,0, + 0,114,147,0,0,0,114,135,0,0,0,114,136,0,0,0, + 114,138,0,0,0,114,78,0,0,0,114,148,0,0,0,114, + 149,0,0,0,114,106,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,152,0, + 0,0,243,2,0,0,115,30,0,0,0,8,7,4,2,12, + 9,2,1,12,6,2,1,12,8,12,4,12,9,12,9,2, + 1,14,5,2,1,14,5,2,1,114,152,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, + 0,0,0,115,32,0,0,0,101,0,90,1,100,0,90,2, + 100,1,90,3,100,2,100,3,132,0,90,4,100,4,100,5, + 132,0,90,5,100,6,83,0,41,7,218,18,95,73,109,112, + 111,114,116,76,111,99,107,67,111,110,116,101,120,116,122,36, + 67,111,110,116,101,120,116,32,109,97,110,97,103,101,114,32, + 102,111,114,32,116,104,101,32,105,109,112,111,114,116,32,108, + 111,99,107,46,99,1,0,0,0,0,0,0,0,1,0,0, + 0,1,0,0,0,67,0,0,0,115,12,0,0,0,116,0, + 106,1,131,0,1,0,100,1,83,0,41,2,122,24,65,99, + 113,117,105,114,101,32,116,104,101,32,105,109,112,111,114,116, + 32,108,111,99,107,46,78,41,2,114,52,0,0,0,114,53, + 0,0,0,41,1,114,26,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,114,46,0,0,0,68,3, + 0,0,115,2,0,0,0,0,2,122,28,95,73,109,112,111, + 114,116,76,111,99,107,67,111,110,116,101,120,116,46,95,95, + 101,110,116,101,114,95,95,99,4,0,0,0,0,0,0,0, + 4,0,0,0,1,0,0,0,67,0,0,0,115,12,0,0, + 0,116,0,106,1,131,0,1,0,100,1,83,0,41,2,122, + 60,82,101,108,101,97,115,101,32,116,104,101,32,105,109,112, + 111,114,116,32,108,111,99,107,32,114,101,103,97,114,100,108, + 101,115,115,32,111,102,32,97,110,121,32,114,97,105,115,101, + 100,32,101,120,99,101,112,116,105,111,110,115,46,78,41,2, + 114,52,0,0,0,114,56,0,0,0,41,4,114,26,0,0, + 0,90,8,101,120,99,95,116,121,112,101,90,9,101,120,99, + 95,118,97,108,117,101,90,13,101,120,99,95,116,114,97,99, + 101,98,97,99,107,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,114,48,0,0,0,72,3,0,0,115,2,0, + 0,0,0,2,122,27,95,73,109,112,111,114,116,76,111,99, + 107,67,111,110,116,101,120,116,46,95,95,101,120,105,116,95, + 95,78,41,6,114,1,0,0,0,114,0,0,0,0,114,2, + 0,0,0,114,3,0,0,0,114,46,0,0,0,114,48,0, + 0,0,114,10,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,157,0,0,0,64,3,0,0,115, + 6,0,0,0,8,2,4,2,8,4,114,157,0,0,0,99, + 3,0,0,0,0,0,0,0,5,0,0,0,4,0,0,0, + 67,0,0,0,115,64,0,0,0,124,1,106,0,100,1,124, + 2,100,2,24,0,131,2,125,3,116,1,124,3,131,1,124, + 2,107,0,114,36,116,2,100,3,131,1,130,1,124,3,100, + 4,25,0,125,4,124,0,114,60,100,5,106,3,124,4,124, + 0,131,2,83,0,124,4,83,0,41,6,122,50,82,101,115, + 111,108,118,101,32,97,32,114,101,108,97,116,105,118,101,32, + 109,111,100,117,108,101,32,110,97,109,101,32,116,111,32,97, + 110,32,97,98,115,111,108,117,116,101,32,111,110,101,46,114, + 118,0,0,0,114,33,0,0,0,122,50,97,116,116,101,109, + 112,116,101,100,32,114,101,108,97,116,105,118,101,32,105,109, + 112,111,114,116,32,98,101,121,111,110,100,32,116,111,112,45, + 108,101,118,101,108,32,112,97,99,107,97,103,101,114,19,0, + 0,0,122,5,123,125,46,123,125,41,4,218,6,114,115,112, + 108,105,116,218,3,108,101,110,218,10,86,97,108,117,101,69, + 114,114,111,114,114,38,0,0,0,41,5,114,15,0,0,0, + 218,7,112,97,99,107,97,103,101,218,5,108,101,118,101,108, + 90,4,98,105,116,115,90,4,98,97,115,101,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,13,95,114,101, + 115,111,108,118,101,95,110,97,109,101,77,3,0,0,115,10, + 0,0,0,0,2,16,1,12,1,8,1,8,1,114,163,0, + 0,0,99,3,0,0,0,0,0,0,0,4,0,0,0,3, + 0,0,0,67,0,0,0,115,34,0,0,0,124,0,106,0, + 124,1,124,2,131,2,125,3,124,3,100,0,107,8,114,24, + 100,0,83,0,116,1,124,1,124,3,131,2,83,0,41,1, + 78,41,2,114,147,0,0,0,114,79,0,0,0,41,4,218, + 6,102,105,110,100,101,114,114,15,0,0,0,114,144,0,0, + 0,114,94,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,17,95,102,105,110,100,95,115,112,101, + 99,95,108,101,103,97,99,121,86,3,0,0,115,8,0,0, + 0,0,3,12,1,8,1,4,1,114,165,0,0,0,99,3, + 0,0,0,0,0,0,0,10,0,0,0,27,0,0,0,67, + 0,0,0,115,242,0,0,0,116,0,106,1,125,3,124,3, + 100,1,107,8,114,22,116,2,100,2,131,1,130,1,124,3, + 115,38,116,3,106,4,100,3,116,5,131,2,1,0,124,0, + 116,0,106,6,107,6,125,4,120,188,124,3,68,0,93,176, + 125,5,116,7,131,0,143,72,1,0,121,10,124,5,106,8, + 125,6,87,0,110,42,4,0,116,9,107,10,114,118,1,0, + 1,0,1,0,116,10,124,5,124,0,124,1,131,3,125,7, + 124,7,100,1,107,8,114,114,119,54,89,0,110,14,88,0, + 124,6,124,0,124,1,124,2,131,3,125,7,87,0,100,1, + 81,0,82,0,88,0,124,7,100,1,107,9,114,54,124,4, + 12,0,114,226,124,0,116,0,106,6,107,6,114,226,116,0, + 106,6,124,0,25,0,125,8,121,10,124,8,106,11,125,9, + 87,0,110,20,4,0,116,9,107,10,114,206,1,0,1,0, + 1,0,124,7,83,0,88,0,124,9,100,1,107,8,114,220, + 124,7,83,0,124,9,83,0,113,54,124,7,83,0,113,54, + 87,0,100,1,83,0,100,1,83,0,41,4,122,21,70,105, + 110,100,32,97,32,109,111,100,117,108,101,39,115,32,115,112, + 101,99,46,78,122,53,115,121,115,46,109,101,116,97,95,112, + 97,116,104,32,105,115,32,78,111,110,101,44,32,80,121,116, + 104,111,110,32,105,115,32,108,105,107,101,108,121,32,115,104, + 117,116,116,105,110,103,32,100,111,119,110,122,22,115,121,115, + 46,109,101,116,97,95,112,97,116,104,32,105,115,32,101,109, + 112,116,121,41,12,114,14,0,0,0,218,9,109,101,116,97, + 95,112,97,116,104,114,71,0,0,0,218,9,95,119,97,114, + 110,105,110,103,115,218,4,119,97,114,110,218,13,73,109,112, + 111,114,116,87,97,114,110,105,110,103,114,80,0,0,0,114, + 157,0,0,0,114,146,0,0,0,114,91,0,0,0,114,165, + 0,0,0,114,90,0,0,0,41,10,114,15,0,0,0,114, + 144,0,0,0,114,145,0,0,0,114,166,0,0,0,90,9, + 105,115,95,114,101,108,111,97,100,114,164,0,0,0,114,146, + 0,0,0,114,83,0,0,0,114,84,0,0,0,114,90,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,114,105,0,0,0,53,3,0,0,115,2,0,0,0,0, - 4,122,25,70,114,111,122,101,110,73,109,112,111,114,116,101, - 114,46,105,115,95,112,97,99,107,97,103,101,41,2,78,78, - 41,1,78,41,16,114,1,0,0,0,114,0,0,0,0,114, - 2,0,0,0,114,3,0,0,0,114,150,0,0,0,114,86, - 0,0,0,114,151,0,0,0,114,146,0,0,0,114,147,0, - 0,0,114,134,0,0,0,114,135,0,0,0,114,138,0,0, - 0,114,77,0,0,0,114,148,0,0,0,114,149,0,0,0, - 114,105,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,152,0,0,0,241,2, - 0,0,115,30,0,0,0,8,7,4,2,12,9,2,1,12, - 6,2,1,12,8,12,4,12,9,12,9,2,1,14,5,2, - 1,14,5,2,1,114,152,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, - 32,0,0,0,101,0,90,1,100,0,90,2,100,1,90,3, - 100,2,100,3,132,0,90,4,100,4,100,5,132,0,90,5, - 100,6,83,0,41,7,218,18,95,73,109,112,111,114,116,76, - 111,99,107,67,111,110,116,101,120,116,122,36,67,111,110,116, - 101,120,116,32,109,97,110,97,103,101,114,32,102,111,114,32, - 116,104,101,32,105,109,112,111,114,116,32,108,111,99,107,46, - 99,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0, - 0,67,0,0,0,115,12,0,0,0,116,0,106,1,131,0, - 1,0,100,1,83,0,41,2,122,24,65,99,113,117,105,114, - 101,32,116,104,101,32,105,109,112,111,114,116,32,108,111,99, - 107,46,78,41,2,114,46,0,0,0,114,137,0,0,0,41, - 1,114,26,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,48,0,0,0,66,3,0,0,115,2, - 0,0,0,0,2,122,28,95,73,109,112,111,114,116,76,111, - 99,107,67,111,110,116,101,120,116,46,95,95,101,110,116,101, - 114,95,95,99,4,0,0,0,0,0,0,0,4,0,0,0, - 1,0,0,0,67,0,0,0,115,12,0,0,0,116,0,106, - 1,131,0,1,0,100,1,83,0,41,2,122,60,82,101,108, - 101,97,115,101,32,116,104,101,32,105,109,112,111,114,116,32, - 108,111,99,107,32,114,101,103,97,114,100,108,101,115,115,32, - 111,102,32,97,110,121,32,114,97,105,115,101,100,32,101,120, - 99,101,112,116,105,111,110,115,46,78,41,2,114,46,0,0, - 0,114,47,0,0,0,41,4,114,26,0,0,0,90,8,101, - 120,99,95,116,121,112,101,90,9,101,120,99,95,118,97,108, - 117,101,90,13,101,120,99,95,116,114,97,99,101,98,97,99, - 107,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,50,0,0,0,70,3,0,0,115,2,0,0,0,0,2, - 122,27,95,73,109,112,111,114,116,76,111,99,107,67,111,110, - 116,101,120,116,46,95,95,101,120,105,116,95,95,78,41,6, - 114,1,0,0,0,114,0,0,0,0,114,2,0,0,0,114, - 3,0,0,0,114,48,0,0,0,114,50,0,0,0,114,10, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,114,157,0,0,0,62,3,0,0,115,6,0,0,0, - 8,2,4,2,8,4,114,157,0,0,0,99,3,0,0,0, - 0,0,0,0,5,0,0,0,4,0,0,0,67,0,0,0, - 115,64,0,0,0,124,1,106,0,100,1,124,2,100,2,24, - 0,131,2,125,3,116,1,124,3,131,1,124,2,107,0,114, - 36,116,2,100,3,131,1,130,1,124,3,100,4,25,0,125, - 4,124,0,114,60,100,5,106,3,124,4,124,0,131,2,83, - 0,124,4,83,0,41,6,122,50,82,101,115,111,108,118,101, - 32,97,32,114,101,108,97,116,105,118,101,32,109,111,100,117, - 108,101,32,110,97,109,101,32,116,111,32,97,110,32,97,98, - 115,111,108,117,116,101,32,111,110,101,46,114,117,0,0,0, - 114,33,0,0,0,122,50,97,116,116,101,109,112,116,101,100, - 32,114,101,108,97,116,105,118,101,32,105,109,112,111,114,116, - 32,98,101,121,111,110,100,32,116,111,112,45,108,101,118,101, - 108,32,112,97,99,107,97,103,101,114,19,0,0,0,122,5, - 123,125,46,123,125,41,4,218,6,114,115,112,108,105,116,218, - 3,108,101,110,218,10,86,97,108,117,101,69,114,114,111,114, - 114,38,0,0,0,41,5,114,15,0,0,0,218,7,112,97, - 99,107,97,103,101,218,5,108,101,118,101,108,90,4,98,105, - 116,115,90,4,98,97,115,101,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,13,95,114,101,115,111,108,118, - 101,95,110,97,109,101,75,3,0,0,115,10,0,0,0,0, - 2,16,1,12,1,8,1,8,1,114,163,0,0,0,99,3, - 0,0,0,0,0,0,0,4,0,0,0,3,0,0,0,67, - 0,0,0,115,34,0,0,0,124,0,106,0,124,1,124,2, - 131,2,125,3,124,3,100,0,107,8,114,24,100,0,83,0, - 116,1,124,1,124,3,131,2,83,0,41,1,78,41,2,114, - 147,0,0,0,114,78,0,0,0,41,4,218,6,102,105,110, - 100,101,114,114,15,0,0,0,114,144,0,0,0,114,93,0, + 0,218,10,95,102,105,110,100,95,115,112,101,99,95,3,0, + 0,115,54,0,0,0,0,2,6,1,8,2,8,3,4,1, + 12,5,10,1,10,1,8,1,2,1,10,1,14,1,12,1, + 8,1,8,2,22,1,8,2,16,1,10,1,2,1,10,1, + 14,4,6,2,8,1,4,2,6,2,8,2,114,170,0,0, + 0,99,3,0,0,0,0,0,0,0,4,0,0,0,4,0, + 0,0,67,0,0,0,115,140,0,0,0,116,0,124,0,116, + 1,131,2,115,28,116,2,100,1,106,3,116,4,124,0,131, + 1,131,1,131,1,130,1,124,2,100,2,107,0,114,44,116, + 5,100,3,131,1,130,1,124,2,100,2,107,4,114,114,116, + 0,124,1,116,1,131,2,115,72,116,2,100,4,131,1,130, + 1,110,42,124,1,115,86,116,6,100,5,131,1,130,1,110, + 28,124,1,116,7,106,8,107,7,114,114,100,6,125,3,116, + 9,124,3,106,3,124,1,131,1,131,1,130,1,124,0,12, + 0,114,136,124,2,100,2,107,2,114,136,116,5,100,7,131, + 1,130,1,100,8,83,0,41,9,122,28,86,101,114,105,102, + 121,32,97,114,103,117,109,101,110,116,115,32,97,114,101,32, + 34,115,97,110,101,34,46,122,31,109,111,100,117,108,101,32, + 110,97,109,101,32,109,117,115,116,32,98,101,32,115,116,114, + 44,32,110,111,116,32,123,125,114,19,0,0,0,122,18,108, + 101,118,101,108,32,109,117,115,116,32,98,101,32,62,61,32, + 48,122,31,95,95,112,97,99,107,97,103,101,95,95,32,110, + 111,116,32,115,101,116,32,116,111,32,97,32,115,116,114,105, + 110,103,122,54,97,116,116,101,109,112,116,101,100,32,114,101, + 108,97,116,105,118,101,32,105,109,112,111,114,116,32,119,105, + 116,104,32,110,111,32,107,110,111,119,110,32,112,97,114,101, + 110,116,32,112,97,99,107,97,103,101,122,61,80,97,114,101, + 110,116,32,109,111,100,117,108,101,32,123,33,114,125,32,110, + 111,116,32,108,111,97,100,101,100,44,32,99,97,110,110,111, + 116,32,112,101,114,102,111,114,109,32,114,101,108,97,116,105, + 118,101,32,105,109,112,111,114,116,122,17,69,109,112,116,121, + 32,109,111,100,117,108,101,32,110,97,109,101,78,41,10,218, + 10,105,115,105,110,115,116,97,110,99,101,218,3,115,116,114, + 218,9,84,121,112,101,69,114,114,111,114,114,38,0,0,0, + 114,13,0,0,0,114,160,0,0,0,114,71,0,0,0,114, + 14,0,0,0,114,80,0,0,0,218,11,83,121,115,116,101, + 109,69,114,114,111,114,41,4,114,15,0,0,0,114,161,0, + 0,0,114,162,0,0,0,114,139,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,13,95,115,97, + 110,105,116,121,95,99,104,101,99,107,142,3,0,0,115,28, + 0,0,0,0,2,10,1,18,1,8,1,8,1,8,1,10, + 1,10,1,4,1,10,2,10,1,4,2,14,1,14,1,114, + 175,0,0,0,122,16,78,111,32,109,111,100,117,108,101,32, + 110,97,109,101,100,32,122,4,123,33,114,125,99,2,0,0, + 0,0,0,0,0,8,0,0,0,12,0,0,0,67,0,0, + 0,115,220,0,0,0,100,0,125,2,124,0,106,0,100,1, + 131,1,100,2,25,0,125,3,124,3,114,134,124,3,116,1, + 106,2,107,7,114,42,116,3,124,1,124,3,131,2,1,0, + 124,0,116,1,106,2,107,6,114,62,116,1,106,2,124,0, + 25,0,83,0,116,1,106,2,124,3,25,0,125,4,121,10, + 124,4,106,4,125,2,87,0,110,50,4,0,116,5,107,10, + 114,132,1,0,1,0,1,0,116,6,100,3,23,0,106,7, + 124,0,124,3,131,2,125,5,116,8,124,5,124,0,100,4, + 141,2,100,0,130,2,89,0,110,2,88,0,116,9,124,0, + 124,2,131,2,125,6,124,6,100,0,107,8,114,172,116,8, + 116,6,106,7,124,0,131,1,124,0,100,4,141,2,130,1, + 110,8,116,10,124,6,131,1,125,7,124,3,114,216,116,1, + 106,2,124,3,25,0,125,4,116,11,124,4,124,0,106,0, + 100,1,131,1,100,5,25,0,124,7,131,3,1,0,124,7, + 83,0,41,6,78,114,118,0,0,0,114,19,0,0,0,122, + 23,59,32,123,33,114,125,32,105,115,32,110,111,116,32,97, + 32,112,97,99,107,97,103,101,41,1,114,15,0,0,0,233, + 2,0,0,0,41,12,114,119,0,0,0,114,14,0,0,0, + 114,80,0,0,0,114,59,0,0,0,114,128,0,0,0,114, + 91,0,0,0,218,8,95,69,82,82,95,77,83,71,114,38, + 0,0,0,218,19,77,111,100,117,108,101,78,111,116,70,111, + 117,110,100,69,114,114,111,114,114,170,0,0,0,114,141,0, + 0,0,114,5,0,0,0,41,8,114,15,0,0,0,218,7, + 105,109,112,111,114,116,95,114,144,0,0,0,114,120,0,0, + 0,90,13,112,97,114,101,110,116,95,109,111,100,117,108,101, + 114,139,0,0,0,114,83,0,0,0,114,84,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,23, + 95,102,105,110,100,95,97,110,100,95,108,111,97,100,95,117, + 110,108,111,99,107,101,100,165,3,0,0,115,42,0,0,0, + 0,1,4,1,14,1,4,1,10,1,10,2,10,1,10,1, + 10,1,2,1,10,1,14,1,16,1,20,1,10,1,8,1, + 20,2,8,1,4,2,10,1,22,1,114,180,0,0,0,99, + 2,0,0,0,0,0,0,0,4,0,0,0,11,0,0,0, + 67,0,0,0,115,92,0,0,0,116,0,124,0,131,1,143, + 26,1,0,124,0,116,1,106,2,107,7,114,30,116,3,124, + 0,124,1,131,2,83,0,87,0,100,1,81,0,82,0,88, + 0,116,1,106,2,124,0,25,0,125,2,124,2,100,1,107, + 8,114,80,100,2,106,4,124,0,131,1,125,3,116,5,124, + 3,124,0,100,3,141,2,130,1,116,6,124,0,131,1,1, + 0,124,2,83,0,41,4,122,25,70,105,110,100,32,97,110, + 100,32,108,111,97,100,32,116,104,101,32,109,111,100,117,108, + 101,46,78,122,40,105,109,112,111,114,116,32,111,102,32,123, + 125,32,104,97,108,116,101,100,59,32,78,111,110,101,32,105, + 110,32,115,121,115,46,109,111,100,117,108,101,115,41,1,114, + 15,0,0,0,41,7,114,42,0,0,0,114,14,0,0,0, + 114,80,0,0,0,114,180,0,0,0,114,38,0,0,0,114, + 178,0,0,0,114,57,0,0,0,41,4,114,15,0,0,0, + 114,179,0,0,0,114,84,0,0,0,114,68,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,14, + 95,102,105,110,100,95,97,110,100,95,108,111,97,100,192,3, + 0,0,115,20,0,0,0,0,2,10,1,10,1,20,2,10, + 1,8,1,4,1,6,1,12,2,8,1,114,181,0,0,0, + 114,19,0,0,0,99,3,0,0,0,0,0,0,0,3,0, + 0,0,4,0,0,0,67,0,0,0,115,42,0,0,0,116, + 0,124,0,124,1,124,2,131,3,1,0,124,2,100,1,107, + 4,114,32,116,1,124,0,124,1,124,2,131,3,125,0,116, + 2,124,0,116,3,131,2,83,0,41,2,97,50,1,0,0, + 73,109,112,111,114,116,32,97,110,100,32,114,101,116,117,114, + 110,32,116,104,101,32,109,111,100,117,108,101,32,98,97,115, + 101,100,32,111,110,32,105,116,115,32,110,97,109,101,44,32, + 116,104,101,32,112,97,99,107,97,103,101,32,116,104,101,32, + 99,97,108,108,32,105,115,10,32,32,32,32,98,101,105,110, + 103,32,109,97,100,101,32,102,114,111,109,44,32,97,110,100, + 32,116,104,101,32,108,101,118,101,108,32,97,100,106,117,115, + 116,109,101,110,116,46,10,10,32,32,32,32,84,104,105,115, + 32,102,117,110,99,116,105,111,110,32,114,101,112,114,101,115, + 101,110,116,115,32,116,104,101,32,103,114,101,97,116,101,115, + 116,32,99,111,109,109,111,110,32,100,101,110,111,109,105,110, + 97,116,111,114,32,111,102,32,102,117,110,99,116,105,111,110, + 97,108,105,116,121,10,32,32,32,32,98,101,116,119,101,101, + 110,32,105,109,112,111,114,116,95,109,111,100,117,108,101,32, + 97,110,100,32,95,95,105,109,112,111,114,116,95,95,46,32, + 84,104,105,115,32,105,110,99,108,117,100,101,115,32,115,101, + 116,116,105,110,103,32,95,95,112,97,99,107,97,103,101,95, + 95,32,105,102,10,32,32,32,32,116,104,101,32,108,111,97, + 100,101,114,32,100,105,100,32,110,111,116,46,10,10,32,32, + 32,32,114,19,0,0,0,41,4,114,175,0,0,0,114,163, + 0,0,0,114,181,0,0,0,218,11,95,103,99,100,95,105, + 109,112,111,114,116,41,3,114,15,0,0,0,114,161,0,0, + 0,114,162,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,114,182,0,0,0,208,3,0,0,115,8, + 0,0,0,0,9,12,1,8,1,12,1,114,182,0,0,0, + 99,3,0,0,0,0,0,0,0,6,0,0,0,17,0,0, + 0,67,0,0,0,115,164,0,0,0,116,0,124,0,100,1, + 131,2,114,160,100,2,124,1,107,6,114,58,116,1,124,1, + 131,1,125,1,124,1,106,2,100,2,131,1,1,0,116,0, + 124,0,100,3,131,2,114,58,124,1,106,3,124,0,106,4, + 131,1,1,0,120,100,124,1,68,0,93,92,125,3,116,0, + 124,0,124,3,131,2,115,64,100,4,106,5,124,0,106,6, + 124,3,131,2,125,4,121,14,116,7,124,2,124,4,131,2, + 1,0,87,0,113,64,4,0,116,8,107,10,114,154,1,0, + 125,5,1,0,122,20,124,5,106,9,124,4,107,2,114,136, + 119,64,130,0,87,0,89,0,100,5,100,5,125,5,126,5, + 88,0,113,64,88,0,113,64,87,0,124,0,83,0,41,6, + 122,238,70,105,103,117,114,101,32,111,117,116,32,119,104,97, + 116,32,95,95,105,109,112,111,114,116,95,95,32,115,104,111, + 117,108,100,32,114,101,116,117,114,110,46,10,10,32,32,32, + 32,84,104,101,32,105,109,112,111,114,116,95,32,112,97,114, + 97,109,101,116,101,114,32,105,115,32,97,32,99,97,108,108, + 97,98,108,101,32,119,104,105,99,104,32,116,97,107,101,115, + 32,116,104,101,32,110,97,109,101,32,111,102,32,109,111,100, + 117,108,101,32,116,111,10,32,32,32,32,105,109,112,111,114, + 116,46,32,73,116,32,105,115,32,114,101,113,117,105,114,101, + 100,32,116,111,32,100,101,99,111,117,112,108,101,32,116,104, + 101,32,102,117,110,99,116,105,111,110,32,102,114,111,109,32, + 97,115,115,117,109,105,110,103,32,105,109,112,111,114,116,108, + 105,98,39,115,10,32,32,32,32,105,109,112,111,114,116,32, + 105,109,112,108,101,109,101,110,116,97,116,105,111,110,32,105, + 115,32,100,101,115,105,114,101,100,46,10,10,32,32,32,32, + 114,128,0,0,0,250,1,42,218,7,95,95,97,108,108,95, + 95,122,5,123,125,46,123,125,78,41,10,114,4,0,0,0, + 114,127,0,0,0,218,6,114,101,109,111,118,101,218,6,101, + 120,116,101,110,100,114,184,0,0,0,114,38,0,0,0,114, + 1,0,0,0,114,59,0,0,0,114,178,0,0,0,114,15, + 0,0,0,41,6,114,84,0,0,0,218,8,102,114,111,109, + 108,105,115,116,114,179,0,0,0,218,1,120,90,9,102,114, + 111,109,95,110,97,109,101,90,3,101,120,99,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,16,95,104,97, + 110,100,108,101,95,102,114,111,109,108,105,115,116,223,3,0, + 0,115,32,0,0,0,0,10,10,1,8,1,8,1,10,1, + 10,1,12,1,10,1,10,1,14,1,2,1,14,1,16,4, + 10,1,2,1,24,1,114,189,0,0,0,99,1,0,0,0, + 0,0,0,0,3,0,0,0,6,0,0,0,67,0,0,0, + 115,146,0,0,0,124,0,106,0,100,1,131,1,125,1,124, + 0,106,0,100,2,131,1,125,2,124,1,100,3,107,9,114, + 82,124,2,100,3,107,9,114,78,124,1,124,2,106,1,107, + 3,114,78,116,2,106,3,100,4,124,1,155,2,100,5,124, + 2,106,1,155,2,100,6,157,5,116,4,100,7,100,8,141, + 3,1,0,124,1,83,0,124,2,100,3,107,9,114,96,124, + 2,106,1,83,0,116,2,106,3,100,9,116,4,100,7,100, + 8,141,3,1,0,124,0,100,10,25,0,125,1,100,11,124, + 0,107,7,114,142,124,1,106,5,100,12,131,1,100,13,25, + 0,125,1,124,1,83,0,41,14,122,167,67,97,108,99,117, + 108,97,116,101,32,119,104,97,116,32,95,95,112,97,99,107, + 97,103,101,95,95,32,115,104,111,117,108,100,32,98,101,46, + 10,10,32,32,32,32,95,95,112,97,99,107,97,103,101,95, + 95,32,105,115,32,110,111,116,32,103,117,97,114,97,110,116, + 101,101,100,32,116,111,32,98,101,32,100,101,102,105,110,101, + 100,32,111,114,32,99,111,117,108,100,32,98,101,32,115,101, + 116,32,116,111,32,78,111,110,101,10,32,32,32,32,116,111, + 32,114,101,112,114,101,115,101,110,116,32,116,104,97,116,32, + 105,116,115,32,112,114,111,112,101,114,32,118,97,108,117,101, + 32,105,115,32,117,110,107,110,111,119,110,46,10,10,32,32, + 32,32,114,131,0,0,0,114,90,0,0,0,78,122,32,95, + 95,112,97,99,107,97,103,101,95,95,32,33,61,32,95,95, + 115,112,101,99,95,95,46,112,97,114,101,110,116,32,40,122, + 4,32,33,61,32,250,1,41,233,3,0,0,0,41,1,90, + 10,115,116,97,99,107,108,101,118,101,108,122,89,99,97,110, + 39,116,32,114,101,115,111,108,118,101,32,112,97,99,107,97, + 103,101,32,102,114,111,109,32,95,95,115,112,101,99,95,95, + 32,111,114,32,95,95,112,97,99,107,97,103,101,95,95,44, + 32,102,97,108,108,105,110,103,32,98,97,99,107,32,111,110, + 32,95,95,110,97,109,101,95,95,32,97,110,100,32,95,95, + 112,97,116,104,95,95,114,1,0,0,0,114,128,0,0,0, + 114,118,0,0,0,114,19,0,0,0,41,6,114,30,0,0, + 0,114,120,0,0,0,114,167,0,0,0,114,168,0,0,0, + 114,169,0,0,0,114,119,0,0,0,41,3,218,7,103,108, + 111,98,97,108,115,114,161,0,0,0,114,83,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,17, + 95,99,97,108,99,95,95,95,112,97,99,107,97,103,101,95, + 95,254,3,0,0,115,30,0,0,0,0,7,10,1,10,1, + 8,1,18,1,22,2,10,1,4,1,8,1,6,2,6,2, + 10,1,8,1,8,1,14,1,114,193,0,0,0,99,5,0, + 0,0,0,0,0,0,9,0,0,0,5,0,0,0,67,0, + 0,0,115,166,0,0,0,124,4,100,1,107,2,114,18,116, + 0,124,0,131,1,125,5,110,36,124,1,100,2,107,9,114, + 30,124,1,110,2,105,0,125,6,116,1,124,6,131,1,125, + 7,116,0,124,0,124,7,124,4,131,3,125,5,124,3,115, + 150,124,4,100,1,107,2,114,84,116,0,124,0,106,2,100, + 3,131,1,100,1,25,0,131,1,83,0,124,0,115,92,124, + 5,83,0,116,3,124,0,131,1,116,3,124,0,106,2,100, + 3,131,1,100,1,25,0,131,1,24,0,125,8,116,4,106, + 5,124,5,106,6,100,2,116,3,124,5,106,6,131,1,124, + 8,24,0,133,2,25,0,25,0,83,0,110,12,116,7,124, + 5,124,3,116,0,131,3,83,0,100,2,83,0,41,4,97, + 215,1,0,0,73,109,112,111,114,116,32,97,32,109,111,100, + 117,108,101,46,10,10,32,32,32,32,84,104,101,32,39,103, + 108,111,98,97,108,115,39,32,97,114,103,117,109,101,110,116, + 32,105,115,32,117,115,101,100,32,116,111,32,105,110,102,101, + 114,32,119,104,101,114,101,32,116,104,101,32,105,109,112,111, + 114,116,32,105,115,32,111,99,99,117,114,114,105,110,103,32, + 102,114,111,109,10,32,32,32,32,116,111,32,104,97,110,100, + 108,101,32,114,101,108,97,116,105,118,101,32,105,109,112,111, + 114,116,115,46,32,84,104,101,32,39,108,111,99,97,108,115, + 39,32,97,114,103,117,109,101,110,116,32,105,115,32,105,103, + 110,111,114,101,100,46,32,84,104,101,10,32,32,32,32,39, + 102,114,111,109,108,105,115,116,39,32,97,114,103,117,109,101, + 110,116,32,115,112,101,99,105,102,105,101,115,32,119,104,97, + 116,32,115,104,111,117,108,100,32,101,120,105,115,116,32,97, + 115,32,97,116,116,114,105,98,117,116,101,115,32,111,110,32, + 116,104,101,32,109,111,100,117,108,101,10,32,32,32,32,98, + 101,105,110,103,32,105,109,112,111,114,116,101,100,32,40,101, + 46,103,46,32,96,96,102,114,111,109,32,109,111,100,117,108, + 101,32,105,109,112,111,114,116,32,60,102,114,111,109,108,105, + 115,116,62,96,96,41,46,32,32,84,104,101,32,39,108,101, + 118,101,108,39,10,32,32,32,32,97,114,103,117,109,101,110, + 116,32,114,101,112,114,101,115,101,110,116,115,32,116,104,101, + 32,112,97,99,107,97,103,101,32,108,111,99,97,116,105,111, + 110,32,116,111,32,105,109,112,111,114,116,32,102,114,111,109, + 32,105,110,32,97,32,114,101,108,97,116,105,118,101,10,32, + 32,32,32,105,109,112,111,114,116,32,40,101,46,103,46,32, + 96,96,102,114,111,109,32,46,46,112,107,103,32,105,109,112, + 111,114,116,32,109,111,100,96,96,32,119,111,117,108,100,32, + 104,97,118,101,32,97,32,39,108,101,118,101,108,39,32,111, + 102,32,50,41,46,10,10,32,32,32,32,114,19,0,0,0, + 78,114,118,0,0,0,41,8,114,182,0,0,0,114,193,0, + 0,0,218,9,112,97,114,116,105,116,105,111,110,114,159,0, + 0,0,114,14,0,0,0,114,80,0,0,0,114,1,0,0, + 0,114,189,0,0,0,41,9,114,15,0,0,0,114,192,0, + 0,0,218,6,108,111,99,97,108,115,114,187,0,0,0,114, + 162,0,0,0,114,84,0,0,0,90,8,103,108,111,98,97, + 108,115,95,114,161,0,0,0,90,7,99,117,116,95,111,102, + 102,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, + 218,10,95,95,105,109,112,111,114,116,95,95,25,4,0,0, + 115,26,0,0,0,0,11,8,1,10,2,16,1,8,1,12, + 1,4,3,8,1,18,1,4,1,4,4,26,3,32,2,114, + 196,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,67,0,0,0,115,38,0,0,0,116,0, + 106,1,124,0,131,1,125,1,124,1,100,0,107,8,114,30, + 116,2,100,1,124,0,23,0,131,1,130,1,116,3,124,1, + 131,1,83,0,41,2,78,122,25,110,111,32,98,117,105,108, + 116,45,105,110,32,109,111,100,117,108,101,32,110,97,109,101, + 100,32,41,4,114,142,0,0,0,114,146,0,0,0,114,71, + 0,0,0,114,141,0,0,0,41,2,114,15,0,0,0,114, + 83,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,18,95,98,117,105,108,116,105,110,95,102,114, + 111,109,95,110,97,109,101,60,4,0,0,115,8,0,0,0, + 0,1,10,1,8,1,12,1,114,197,0,0,0,99,2,0, + 0,0,0,0,0,0,12,0,0,0,12,0,0,0,67,0, + 0,0,115,244,0,0,0,124,1,97,0,124,0,97,1,116, + 2,116,1,131,1,125,2,120,86,116,1,106,3,106,4,131, + 0,68,0,93,72,92,2,125,3,125,4,116,5,124,4,124, + 2,131,2,114,28,124,3,116,1,106,6,107,6,114,62,116, + 7,125,5,110,18,116,0,106,8,124,3,131,1,114,28,116, + 9,125,5,110,2,113,28,116,10,124,4,124,5,131,2,125, + 6,116,11,124,6,124,4,131,2,1,0,113,28,87,0,116, + 1,106,3,116,12,25,0,125,7,120,54,100,5,68,0,93, + 46,125,8,124,8,116,1,106,3,107,7,114,144,116,13,124, + 8,131,1,125,9,110,10,116,1,106,3,124,8,25,0,125, + 9,116,14,124,7,124,8,124,9,131,3,1,0,113,120,87, + 0,121,12,116,13,100,2,131,1,125,10,87,0,110,24,4, + 0,116,15,107,10,114,206,1,0,1,0,1,0,100,3,125, + 10,89,0,110,2,88,0,116,14,124,7,100,2,124,10,131, + 3,1,0,116,13,100,4,131,1,125,11,116,14,124,7,100, + 4,124,11,131,3,1,0,100,3,83,0,41,6,122,250,83, + 101,116,117,112,32,105,109,112,111,114,116,108,105,98,32,98, + 121,32,105,109,112,111,114,116,105,110,103,32,110,101,101,100, + 101,100,32,98,117,105,108,116,45,105,110,32,109,111,100,117, + 108,101,115,32,97,110,100,32,105,110,106,101,99,116,105,110, + 103,32,116,104,101,109,10,32,32,32,32,105,110,116,111,32, + 116,104,101,32,103,108,111,98,97,108,32,110,97,109,101,115, + 112,97,99,101,46,10,10,32,32,32,32,65,115,32,115,121, + 115,32,105,115,32,110,101,101,100,101,100,32,102,111,114,32, + 115,121,115,46,109,111,100,117,108,101,115,32,97,99,99,101, + 115,115,32,97,110,100,32,95,105,109,112,32,105,115,32,110, + 101,101,100,101,100,32,116,111,32,108,111,97,100,32,98,117, + 105,108,116,45,105,110,10,32,32,32,32,109,111,100,117,108, + 101,115,44,32,116,104,111,115,101,32,116,119,111,32,109,111, + 100,117,108,101,115,32,109,117,115,116,32,98,101,32,101,120, + 112,108,105,99,105,116,108,121,32,112,97,115,115,101,100,32, + 105,110,46,10,10,32,32,32,32,114,167,0,0,0,114,20, + 0,0,0,78,114,55,0,0,0,41,1,114,167,0,0,0, + 41,16,114,52,0,0,0,114,14,0,0,0,114,13,0,0, + 0,114,80,0,0,0,218,5,105,116,101,109,115,114,171,0, + 0,0,114,70,0,0,0,114,142,0,0,0,114,76,0,0, + 0,114,152,0,0,0,114,129,0,0,0,114,134,0,0,0, + 114,1,0,0,0,114,197,0,0,0,114,5,0,0,0,114, + 71,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, + 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, + 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, + 0,114,84,0,0,0,114,94,0,0,0,114,83,0,0,0, + 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, + 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, + 108,116,105,110,95,109,111,100,117,108,101,90,13,116,104,114, + 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, + 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, + 112,67,4,0,0,115,50,0,0,0,0,9,4,1,4,3, + 8,1,20,1,10,1,10,1,6,1,10,1,6,2,2,1, + 10,1,14,3,10,1,10,1,10,1,10,2,10,1,16,3, + 2,1,12,1,14,2,10,1,12,3,8,1,114,201,0,0, + 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, + 0,0,67,0,0,0,115,66,0,0,0,116,0,124,0,124, + 1,131,2,1,0,116,1,106,2,106,3,116,4,131,1,1, + 0,116,1,106,2,106,3,116,5,131,1,1,0,100,1,100, + 2,108,6,125,2,124,2,97,7,124,2,106,8,116,1,106, + 9,116,10,25,0,131,1,1,0,100,2,83,0,41,3,122, + 50,73,110,115,116,97,108,108,32,105,109,112,111,114,116,108, + 105,98,32,97,115,32,116,104,101,32,105,109,112,108,101,109, + 101,110,116,97,116,105,111,110,32,111,102,32,105,109,112,111, + 114,116,46,114,19,0,0,0,78,41,11,114,201,0,0,0, + 114,14,0,0,0,114,166,0,0,0,114,110,0,0,0,114, + 142,0,0,0,114,152,0,0,0,218,26,95,102,114,111,122, + 101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,116, + 101,114,110,97,108,114,116,0,0,0,218,8,95,105,110,115, + 116,97,108,108,114,80,0,0,0,114,1,0,0,0,41,3, + 114,199,0,0,0,114,200,0,0,0,114,202,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,203, + 0,0,0,114,4,0,0,115,12,0,0,0,0,2,10,2, + 12,1,12,3,8,1,4,1,114,203,0,0,0,41,2,78, + 78,41,1,78,41,2,78,114,19,0,0,0,41,50,114,3, + 0,0,0,114,116,0,0,0,114,12,0,0,0,114,16,0, + 0,0,114,49,0,0,0,114,29,0,0,0,114,36,0,0, + 0,114,17,0,0,0,114,18,0,0,0,114,41,0,0,0, + 114,42,0,0,0,114,45,0,0,0,114,57,0,0,0,114, + 59,0,0,0,114,69,0,0,0,114,75,0,0,0,114,78, + 0,0,0,114,85,0,0,0,114,96,0,0,0,114,97,0, + 0,0,114,103,0,0,0,114,79,0,0,0,218,6,111,98, + 106,101,99,116,90,9,95,80,79,80,85,76,65,84,69,114, + 129,0,0,0,114,134,0,0,0,114,137,0,0,0,114,92, + 0,0,0,114,81,0,0,0,114,140,0,0,0,114,141,0, + 0,0,114,82,0,0,0,114,142,0,0,0,114,152,0,0, + 0,114,157,0,0,0,114,163,0,0,0,114,165,0,0,0, + 114,170,0,0,0,114,175,0,0,0,90,15,95,69,82,82, + 95,77,83,71,95,80,82,69,70,73,88,114,177,0,0,0, + 114,180,0,0,0,114,181,0,0,0,114,182,0,0,0,114, + 189,0,0,0,114,193,0,0,0,114,196,0,0,0,114,197, + 0,0,0,114,201,0,0,0,114,203,0,0,0,114,10,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,17,95,102,105,110,100,95,115,112,101,99,95,108,101, - 103,97,99,121,84,3,0,0,115,8,0,0,0,0,3,12, - 1,8,1,4,1,114,165,0,0,0,99,3,0,0,0,0, - 0,0,0,10,0,0,0,27,0,0,0,67,0,0,0,115, - 242,0,0,0,116,0,106,1,125,3,124,3,100,1,107,8, - 114,22,116,2,100,2,131,1,130,1,124,3,115,38,116,3, - 106,4,100,3,116,5,131,2,1,0,124,0,116,0,106,6, - 107,6,125,4,120,188,124,3,68,0,93,176,125,5,116,7, - 131,0,143,72,1,0,121,10,124,5,106,8,125,6,87,0, - 110,42,4,0,116,9,107,10,114,118,1,0,1,0,1,0, - 116,10,124,5,124,0,124,1,131,3,125,7,124,7,100,1, - 107,8,114,114,119,54,89,0,110,14,88,0,124,6,124,0, - 124,1,124,2,131,3,125,7,87,0,100,1,81,0,82,0, - 88,0,124,7,100,1,107,9,114,54,124,4,12,0,114,226, - 124,0,116,0,106,6,107,6,114,226,116,0,106,6,124,0, - 25,0,125,8,121,10,124,8,106,11,125,9,87,0,110,20, - 4,0,116,9,107,10,114,206,1,0,1,0,1,0,124,7, - 83,0,88,0,124,9,100,1,107,8,114,220,124,7,83,0, - 124,9,83,0,113,54,124,7,83,0,113,54,87,0,100,1, - 83,0,100,1,83,0,41,4,122,21,70,105,110,100,32,97, - 32,109,111,100,117,108,101,39,115,32,115,112,101,99,46,78, - 122,53,115,121,115,46,109,101,116,97,95,112,97,116,104,32, - 105,115,32,78,111,110,101,44,32,80,121,116,104,111,110,32, - 105,115,32,108,105,107,101,108,121,32,115,104,117,116,116,105, - 110,103,32,100,111,119,110,122,22,115,121,115,46,109,101,116, - 97,95,112,97,116,104,32,105,115,32,101,109,112,116,121,41, - 12,114,14,0,0,0,218,9,109,101,116,97,95,112,97,116, - 104,114,70,0,0,0,218,9,95,119,97,114,110,105,110,103, - 115,218,4,119,97,114,110,218,13,73,109,112,111,114,116,87, - 97,114,110,105,110,103,114,79,0,0,0,114,157,0,0,0, - 114,146,0,0,0,114,90,0,0,0,114,165,0,0,0,114, - 89,0,0,0,41,10,114,15,0,0,0,114,144,0,0,0, - 114,145,0,0,0,114,166,0,0,0,90,9,105,115,95,114, - 101,108,111,97,100,114,164,0,0,0,114,146,0,0,0,114, - 82,0,0,0,114,83,0,0,0,114,89,0,0,0,114,10, - 0,0,0,114,10,0,0,0,114,11,0,0,0,218,10,95, - 102,105,110,100,95,115,112,101,99,93,3,0,0,115,54,0, - 0,0,0,2,6,1,8,2,8,3,4,1,12,5,10,1, - 10,1,8,1,2,1,10,1,14,1,12,1,8,1,8,2, - 22,1,8,2,16,1,10,1,2,1,10,1,14,4,6,2, - 8,1,4,2,6,2,8,2,114,170,0,0,0,99,3,0, - 0,0,0,0,0,0,4,0,0,0,4,0,0,0,67,0, - 0,0,115,140,0,0,0,116,0,124,0,116,1,131,2,115, - 28,116,2,100,1,106,3,116,4,124,0,131,1,131,1,131, - 1,130,1,124,2,100,2,107,0,114,44,116,5,100,3,131, - 1,130,1,124,2,100,2,107,4,114,114,116,0,124,1,116, - 1,131,2,115,72,116,2,100,4,131,1,130,1,110,42,124, - 1,115,86,116,6,100,5,131,1,130,1,110,28,124,1,116, - 7,106,8,107,7,114,114,100,6,125,3,116,9,124,3,106, - 3,124,1,131,1,131,1,130,1,124,0,12,0,114,136,124, - 2,100,2,107,2,114,136,116,5,100,7,131,1,130,1,100, - 8,83,0,41,9,122,28,86,101,114,105,102,121,32,97,114, - 103,117,109,101,110,116,115,32,97,114,101,32,34,115,97,110, - 101,34,46,122,31,109,111,100,117,108,101,32,110,97,109,101, - 32,109,117,115,116,32,98,101,32,115,116,114,44,32,110,111, - 116,32,123,125,114,19,0,0,0,122,18,108,101,118,101,108, - 32,109,117,115,116,32,98,101,32,62,61,32,48,122,31,95, - 95,112,97,99,107,97,103,101,95,95,32,110,111,116,32,115, - 101,116,32,116,111,32,97,32,115,116,114,105,110,103,122,54, - 97,116,116,101,109,112,116,101,100,32,114,101,108,97,116,105, - 118,101,32,105,109,112,111,114,116,32,119,105,116,104,32,110, - 111,32,107,110,111,119,110,32,112,97,114,101,110,116,32,112, - 97,99,107,97,103,101,122,61,80,97,114,101,110,116,32,109, - 111,100,117,108,101,32,123,33,114,125,32,110,111,116,32,108, - 111,97,100,101,100,44,32,99,97,110,110,111,116,32,112,101, - 114,102,111,114,109,32,114,101,108,97,116,105,118,101,32,105, - 109,112,111,114,116,122,17,69,109,112,116,121,32,109,111,100, - 117,108,101,32,110,97,109,101,78,41,10,218,10,105,115,105, - 110,115,116,97,110,99,101,218,3,115,116,114,218,9,84,121, - 112,101,69,114,114,111,114,114,38,0,0,0,114,13,0,0, - 0,114,160,0,0,0,114,70,0,0,0,114,14,0,0,0, - 114,79,0,0,0,218,11,83,121,115,116,101,109,69,114,114, - 111,114,41,4,114,15,0,0,0,114,161,0,0,0,114,162, - 0,0,0,114,139,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,13,95,115,97,110,105,116,121, - 95,99,104,101,99,107,140,3,0,0,115,28,0,0,0,0, - 2,10,1,18,1,8,1,8,1,8,1,10,1,10,1,4, - 1,10,2,10,1,4,2,14,1,14,1,114,175,0,0,0, - 122,16,78,111,32,109,111,100,117,108,101,32,110,97,109,101, - 100,32,122,4,123,33,114,125,99,2,0,0,0,0,0,0, - 0,8,0,0,0,12,0,0,0,67,0,0,0,115,220,0, - 0,0,100,0,125,2,124,0,106,0,100,1,131,1,100,2, - 25,0,125,3,124,3,114,134,124,3,116,1,106,2,107,7, - 114,42,116,3,124,1,124,3,131,2,1,0,124,0,116,1, - 106,2,107,6,114,62,116,1,106,2,124,0,25,0,83,0, - 116,1,106,2,124,3,25,0,125,4,121,10,124,4,106,4, - 125,2,87,0,110,50,4,0,116,5,107,10,114,132,1,0, - 1,0,1,0,116,6,100,3,23,0,106,7,124,0,124,3, - 131,2,125,5,116,8,124,5,124,0,100,4,141,2,100,0, - 130,2,89,0,110,2,88,0,116,9,124,0,124,2,131,2, - 125,6,124,6,100,0,107,8,114,172,116,8,116,6,106,7, - 124,0,131,1,124,0,100,4,141,2,130,1,110,8,116,10, - 124,6,131,1,125,7,124,3,114,216,116,1,106,2,124,3, - 25,0,125,4,116,11,124,4,124,0,106,0,100,1,131,1, - 100,5,25,0,124,7,131,3,1,0,124,7,83,0,41,6, - 78,114,117,0,0,0,114,19,0,0,0,122,23,59,32,123, - 33,114,125,32,105,115,32,110,111,116,32,97,32,112,97,99, - 107,97,103,101,41,1,114,15,0,0,0,233,2,0,0,0, - 41,12,114,118,0,0,0,114,14,0,0,0,114,79,0,0, - 0,114,58,0,0,0,114,127,0,0,0,114,90,0,0,0, - 218,8,95,69,82,82,95,77,83,71,114,38,0,0,0,218, - 19,77,111,100,117,108,101,78,111,116,70,111,117,110,100,69, - 114,114,111,114,114,170,0,0,0,114,141,0,0,0,114,5, - 0,0,0,41,8,114,15,0,0,0,218,7,105,109,112,111, - 114,116,95,114,144,0,0,0,114,119,0,0,0,90,13,112, - 97,114,101,110,116,95,109,111,100,117,108,101,114,139,0,0, - 0,114,82,0,0,0,114,83,0,0,0,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,218,23,95,102,105,110, - 100,95,97,110,100,95,108,111,97,100,95,117,110,108,111,99, - 107,101,100,163,3,0,0,115,42,0,0,0,0,1,4,1, - 14,1,4,1,10,1,10,2,10,1,10,1,10,1,2,1, - 10,1,14,1,16,1,20,1,10,1,8,1,20,2,8,1, - 4,2,10,1,22,1,114,180,0,0,0,99,2,0,0,0, - 0,0,0,0,4,0,0,0,11,0,0,0,67,0,0,0, - 115,104,0,0,0,116,0,106,1,131,0,1,0,124,0,116, - 2,106,3,107,7,114,44,116,4,124,0,131,1,143,12,1, - 0,116,5,124,0,124,1,131,2,83,0,81,0,82,0,88, - 0,116,2,106,3,124,0,25,0,125,2,124,2,100,1,107, - 8,114,92,116,0,106,6,131,0,1,0,100,2,106,7,124, - 0,131,1,125,3,116,8,124,3,124,0,100,3,141,2,130, - 1,116,9,124,0,131,1,1,0,124,2,83,0,41,4,122, - 25,70,105,110,100,32,97,110,100,32,108,111,97,100,32,116, - 104,101,32,109,111,100,117,108,101,46,78,122,40,105,109,112, - 111,114,116,32,111,102,32,123,125,32,104,97,108,116,101,100, - 59,32,78,111,110,101,32,105,110,32,115,121,115,46,109,111, - 100,117,108,101,115,41,1,114,15,0,0,0,41,10,114,46, - 0,0,0,114,137,0,0,0,114,14,0,0,0,114,79,0, - 0,0,114,42,0,0,0,114,180,0,0,0,114,47,0,0, - 0,114,38,0,0,0,114,178,0,0,0,114,56,0,0,0, - 41,4,114,15,0,0,0,114,179,0,0,0,114,83,0,0, - 0,114,67,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,14,95,102,105,110,100,95,97,110,100, - 95,108,111,97,100,190,3,0,0,115,24,0,0,0,0,2, - 8,1,10,1,10,1,16,1,10,1,8,1,8,1,4,1, - 6,1,12,1,8,1,114,181,0,0,0,114,19,0,0,0, - 99,3,0,0,0,0,0,0,0,3,0,0,0,4,0,0, - 0,67,0,0,0,115,42,0,0,0,116,0,124,0,124,1, - 124,2,131,3,1,0,124,2,100,1,107,4,114,32,116,1, - 124,0,124,1,124,2,131,3,125,0,116,2,124,0,116,3, - 131,2,83,0,41,2,97,50,1,0,0,73,109,112,111,114, - 116,32,97,110,100,32,114,101,116,117,114,110,32,116,104,101, - 32,109,111,100,117,108,101,32,98,97,115,101,100,32,111,110, - 32,105,116,115,32,110,97,109,101,44,32,116,104,101,32,112, - 97,99,107,97,103,101,32,116,104,101,32,99,97,108,108,32, - 105,115,10,32,32,32,32,98,101,105,110,103,32,109,97,100, - 101,32,102,114,111,109,44,32,97,110,100,32,116,104,101,32, - 108,101,118,101,108,32,97,100,106,117,115,116,109,101,110,116, - 46,10,10,32,32,32,32,84,104,105,115,32,102,117,110,99, - 116,105,111,110,32,114,101,112,114,101,115,101,110,116,115,32, - 116,104,101,32,103,114,101,97,116,101,115,116,32,99,111,109, - 109,111,110,32,100,101,110,111,109,105,110,97,116,111,114,32, - 111,102,32,102,117,110,99,116,105,111,110,97,108,105,116,121, - 10,32,32,32,32,98,101,116,119,101,101,110,32,105,109,112, - 111,114,116,95,109,111,100,117,108,101,32,97,110,100,32,95, - 95,105,109,112,111,114,116,95,95,46,32,84,104,105,115,32, - 105,110,99,108,117,100,101,115,32,115,101,116,116,105,110,103, - 32,95,95,112,97,99,107,97,103,101,95,95,32,105,102,10, - 32,32,32,32,116,104,101,32,108,111,97,100,101,114,32,100, - 105,100,32,110,111,116,46,10,10,32,32,32,32,114,19,0, - 0,0,41,4,114,175,0,0,0,114,163,0,0,0,114,181, - 0,0,0,218,11,95,103,99,100,95,105,109,112,111,114,116, - 41,3,114,15,0,0,0,114,161,0,0,0,114,162,0,0, - 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,182,0,0,0,206,3,0,0,115,8,0,0,0,0,9, - 12,1,8,1,12,1,114,182,0,0,0,99,3,0,0,0, - 0,0,0,0,6,0,0,0,17,0,0,0,67,0,0,0, - 115,164,0,0,0,116,0,124,0,100,1,131,2,114,160,100, - 2,124,1,107,6,114,58,116,1,124,1,131,1,125,1,124, - 1,106,2,100,2,131,1,1,0,116,0,124,0,100,3,131, - 2,114,58,124,1,106,3,124,0,106,4,131,1,1,0,120, - 100,124,1,68,0,93,92,125,3,116,0,124,0,124,3,131, - 2,115,64,100,4,106,5,124,0,106,6,124,3,131,2,125, - 4,121,14,116,7,124,2,124,4,131,2,1,0,87,0,113, - 64,4,0,116,8,107,10,114,154,1,0,125,5,1,0,122, - 20,124,5,106,9,124,4,107,2,114,136,119,64,130,0,87, - 0,89,0,100,5,100,5,125,5,126,5,88,0,113,64,88, - 0,113,64,87,0,124,0,83,0,41,6,122,238,70,105,103, - 117,114,101,32,111,117,116,32,119,104,97,116,32,95,95,105, - 109,112,111,114,116,95,95,32,115,104,111,117,108,100,32,114, - 101,116,117,114,110,46,10,10,32,32,32,32,84,104,101,32, - 105,109,112,111,114,116,95,32,112,97,114,97,109,101,116,101, - 114,32,105,115,32,97,32,99,97,108,108,97,98,108,101,32, - 119,104,105,99,104,32,116,97,107,101,115,32,116,104,101,32, - 110,97,109,101,32,111,102,32,109,111,100,117,108,101,32,116, - 111,10,32,32,32,32,105,109,112,111,114,116,46,32,73,116, - 32,105,115,32,114,101,113,117,105,114,101,100,32,116,111,32, - 100,101,99,111,117,112,108,101,32,116,104,101,32,102,117,110, - 99,116,105,111,110,32,102,114,111,109,32,97,115,115,117,109, - 105,110,103,32,105,109,112,111,114,116,108,105,98,39,115,10, - 32,32,32,32,105,109,112,111,114,116,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,105,115,32,100,101,115, - 105,114,101,100,46,10,10,32,32,32,32,114,127,0,0,0, - 250,1,42,218,7,95,95,97,108,108,95,95,122,5,123,125, - 46,123,125,78,41,10,114,4,0,0,0,114,126,0,0,0, - 218,6,114,101,109,111,118,101,218,6,101,120,116,101,110,100, - 114,184,0,0,0,114,38,0,0,0,114,1,0,0,0,114, - 58,0,0,0,114,178,0,0,0,114,15,0,0,0,41,6, - 114,83,0,0,0,218,8,102,114,111,109,108,105,115,116,114, - 179,0,0,0,218,1,120,90,9,102,114,111,109,95,110,97, - 109,101,90,3,101,120,99,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,218,16,95,104,97,110,100,108,101,95, - 102,114,111,109,108,105,115,116,221,3,0,0,115,32,0,0, - 0,0,10,10,1,8,1,8,1,10,1,10,1,12,1,10, - 1,10,1,14,1,2,1,14,1,16,4,10,1,2,1,24, - 1,114,189,0,0,0,99,1,0,0,0,0,0,0,0,3, - 0,0,0,6,0,0,0,67,0,0,0,115,146,0,0,0, - 124,0,106,0,100,1,131,1,125,1,124,0,106,0,100,2, - 131,1,125,2,124,1,100,3,107,9,114,82,124,2,100,3, - 107,9,114,78,124,1,124,2,106,1,107,3,114,78,116,2, - 106,3,100,4,124,1,155,2,100,5,124,2,106,1,155,2, - 100,6,157,5,116,4,100,7,100,8,141,3,1,0,124,1, - 83,0,124,2,100,3,107,9,114,96,124,2,106,1,83,0, - 116,2,106,3,100,9,116,4,100,7,100,8,141,3,1,0, - 124,0,100,10,25,0,125,1,100,11,124,0,107,7,114,142, - 124,1,106,5,100,12,131,1,100,13,25,0,125,1,124,1, - 83,0,41,14,122,167,67,97,108,99,117,108,97,116,101,32, - 119,104,97,116,32,95,95,112,97,99,107,97,103,101,95,95, - 32,115,104,111,117,108,100,32,98,101,46,10,10,32,32,32, - 32,95,95,112,97,99,107,97,103,101,95,95,32,105,115,32, - 110,111,116,32,103,117,97,114,97,110,116,101,101,100,32,116, - 111,32,98,101,32,100,101,102,105,110,101,100,32,111,114,32, - 99,111,117,108,100,32,98,101,32,115,101,116,32,116,111,32, - 78,111,110,101,10,32,32,32,32,116,111,32,114,101,112,114, - 101,115,101,110,116,32,116,104,97,116,32,105,116,115,32,112, - 114,111,112,101,114,32,118,97,108,117,101,32,105,115,32,117, - 110,107,110,111,119,110,46,10,10,32,32,32,32,114,130,0, - 0,0,114,89,0,0,0,78,122,32,95,95,112,97,99,107, - 97,103,101,95,95,32,33,61,32,95,95,115,112,101,99,95, - 95,46,112,97,114,101,110,116,32,40,122,4,32,33,61,32, - 250,1,41,233,3,0,0,0,41,1,90,10,115,116,97,99, - 107,108,101,118,101,108,122,89,99,97,110,39,116,32,114,101, - 115,111,108,118,101,32,112,97,99,107,97,103,101,32,102,114, - 111,109,32,95,95,115,112,101,99,95,95,32,111,114,32,95, - 95,112,97,99,107,97,103,101,95,95,44,32,102,97,108,108, - 105,110,103,32,98,97,99,107,32,111,110,32,95,95,110,97, - 109,101,95,95,32,97,110,100,32,95,95,112,97,116,104,95, - 95,114,1,0,0,0,114,127,0,0,0,114,117,0,0,0, - 114,19,0,0,0,41,6,114,30,0,0,0,114,119,0,0, - 0,114,167,0,0,0,114,168,0,0,0,114,169,0,0,0, - 114,118,0,0,0,41,3,218,7,103,108,111,98,97,108,115, - 114,161,0,0,0,114,82,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,218,17,95,99,97,108,99, - 95,95,95,112,97,99,107,97,103,101,95,95,252,3,0,0, - 115,30,0,0,0,0,7,10,1,10,1,8,1,18,1,22, - 2,10,1,4,1,8,1,6,2,6,2,10,1,8,1,8, - 1,14,1,114,193,0,0,0,99,5,0,0,0,0,0,0, - 0,9,0,0,0,5,0,0,0,67,0,0,0,115,166,0, - 0,0,124,4,100,1,107,2,114,18,116,0,124,0,131,1, - 125,5,110,36,124,1,100,2,107,9,114,30,124,1,110,2, - 105,0,125,6,116,1,124,6,131,1,125,7,116,0,124,0, - 124,7,124,4,131,3,125,5,124,3,115,150,124,4,100,1, - 107,2,114,84,116,0,124,0,106,2,100,3,131,1,100,1, - 25,0,131,1,83,0,124,0,115,92,124,5,83,0,116,3, - 124,0,131,1,116,3,124,0,106,2,100,3,131,1,100,1, - 25,0,131,1,24,0,125,8,116,4,106,5,124,5,106,6, - 100,2,116,3,124,5,106,6,131,1,124,8,24,0,133,2, - 25,0,25,0,83,0,110,12,116,7,124,5,124,3,116,0, - 131,3,83,0,100,2,83,0,41,4,97,215,1,0,0,73, - 109,112,111,114,116,32,97,32,109,111,100,117,108,101,46,10, - 10,32,32,32,32,84,104,101,32,39,103,108,111,98,97,108, - 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,117, - 115,101,100,32,116,111,32,105,110,102,101,114,32,119,104,101, - 114,101,32,116,104,101,32,105,109,112,111,114,116,32,105,115, - 32,111,99,99,117,114,114,105,110,103,32,102,114,111,109,10, - 32,32,32,32,116,111,32,104,97,110,100,108,101,32,114,101, - 108,97,116,105,118,101,32,105,109,112,111,114,116,115,46,32, - 84,104,101,32,39,108,111,99,97,108,115,39,32,97,114,103, - 117,109,101,110,116,32,105,115,32,105,103,110,111,114,101,100, - 46,32,84,104,101,10,32,32,32,32,39,102,114,111,109,108, - 105,115,116,39,32,97,114,103,117,109,101,110,116,32,115,112, - 101,99,105,102,105,101,115,32,119,104,97,116,32,115,104,111, - 117,108,100,32,101,120,105,115,116,32,97,115,32,97,116,116, - 114,105,98,117,116,101,115,32,111,110,32,116,104,101,32,109, - 111,100,117,108,101,10,32,32,32,32,98,101,105,110,103,32, - 105,109,112,111,114,116,101,100,32,40,101,46,103,46,32,96, - 96,102,114,111,109,32,109,111,100,117,108,101,32,105,109,112, - 111,114,116,32,60,102,114,111,109,108,105,115,116,62,96,96, - 41,46,32,32,84,104,101,32,39,108,101,118,101,108,39,10, - 32,32,32,32,97,114,103,117,109,101,110,116,32,114,101,112, - 114,101,115,101,110,116,115,32,116,104,101,32,112,97,99,107, - 97,103,101,32,108,111,99,97,116,105,111,110,32,116,111,32, - 105,109,112,111,114,116,32,102,114,111,109,32,105,110,32,97, - 32,114,101,108,97,116,105,118,101,10,32,32,32,32,105,109, - 112,111,114,116,32,40,101,46,103,46,32,96,96,102,114,111, - 109,32,46,46,112,107,103,32,105,109,112,111,114,116,32,109, - 111,100,96,96,32,119,111,117,108,100,32,104,97,118,101,32, - 97,32,39,108,101,118,101,108,39,32,111,102,32,50,41,46, - 10,10,32,32,32,32,114,19,0,0,0,78,114,117,0,0, - 0,41,8,114,182,0,0,0,114,193,0,0,0,218,9,112, - 97,114,116,105,116,105,111,110,114,159,0,0,0,114,14,0, - 0,0,114,79,0,0,0,114,1,0,0,0,114,189,0,0, - 0,41,9,114,15,0,0,0,114,192,0,0,0,218,6,108, - 111,99,97,108,115,114,187,0,0,0,114,162,0,0,0,114, - 83,0,0,0,90,8,103,108,111,98,97,108,115,95,114,161, - 0,0,0,90,7,99,117,116,95,111,102,102,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,10,95,95,105, - 109,112,111,114,116,95,95,23,4,0,0,115,26,0,0,0, - 0,11,8,1,10,2,16,1,8,1,12,1,4,3,8,1, - 18,1,4,1,4,4,26,3,32,2,114,196,0,0,0,99, - 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, - 67,0,0,0,115,38,0,0,0,116,0,106,1,124,0,131, - 1,125,1,124,1,100,0,107,8,114,30,116,2,100,1,124, - 0,23,0,131,1,130,1,116,3,124,1,131,1,83,0,41, - 2,78,122,25,110,111,32,98,117,105,108,116,45,105,110,32, - 109,111,100,117,108,101,32,110,97,109,101,100,32,41,4,114, - 142,0,0,0,114,146,0,0,0,114,70,0,0,0,114,141, - 0,0,0,41,2,114,15,0,0,0,114,82,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,18, - 95,98,117,105,108,116,105,110,95,102,114,111,109,95,110,97, - 109,101,58,4,0,0,115,8,0,0,0,0,1,10,1,8, - 1,12,1,114,197,0,0,0,99,2,0,0,0,0,0,0, - 0,12,0,0,0,12,0,0,0,67,0,0,0,115,244,0, - 0,0,124,1,97,0,124,0,97,1,116,2,116,1,131,1, - 125,2,120,86,116,1,106,3,106,4,131,0,68,0,93,72, - 92,2,125,3,125,4,116,5,124,4,124,2,131,2,114,28, - 124,3,116,1,106,6,107,6,114,62,116,7,125,5,110,18, - 116,0,106,8,124,3,131,1,114,28,116,9,125,5,110,2, - 113,28,116,10,124,4,124,5,131,2,125,6,116,11,124,6, - 124,4,131,2,1,0,113,28,87,0,116,1,106,3,116,12, - 25,0,125,7,120,54,100,5,68,0,93,46,125,8,124,8, - 116,1,106,3,107,7,114,144,116,13,124,8,131,1,125,9, - 110,10,116,1,106,3,124,8,25,0,125,9,116,14,124,7, - 124,8,124,9,131,3,1,0,113,120,87,0,121,12,116,13, - 100,2,131,1,125,10,87,0,110,24,4,0,116,15,107,10, - 114,206,1,0,1,0,1,0,100,3,125,10,89,0,110,2, - 88,0,116,14,124,7,100,2,124,10,131,3,1,0,116,13, - 100,4,131,1,125,11,116,14,124,7,100,4,124,11,131,3, - 1,0,100,3,83,0,41,6,122,250,83,101,116,117,112,32, - 105,109,112,111,114,116,108,105,98,32,98,121,32,105,109,112, - 111,114,116,105,110,103,32,110,101,101,100,101,100,32,98,117, - 105,108,116,45,105,110,32,109,111,100,117,108,101,115,32,97, - 110,100,32,105,110,106,101,99,116,105,110,103,32,116,104,101, - 109,10,32,32,32,32,105,110,116,111,32,116,104,101,32,103, - 108,111,98,97,108,32,110,97,109,101,115,112,97,99,101,46, - 10,10,32,32,32,32,65,115,32,115,121,115,32,105,115,32, - 110,101,101,100,101,100,32,102,111,114,32,115,121,115,46,109, - 111,100,117,108,101,115,32,97,99,99,101,115,115,32,97,110, - 100,32,95,105,109,112,32,105,115,32,110,101,101,100,101,100, - 32,116,111,32,108,111,97,100,32,98,117,105,108,116,45,105, - 110,10,32,32,32,32,109,111,100,117,108,101,115,44,32,116, - 104,111,115,101,32,116,119,111,32,109,111,100,117,108,101,115, - 32,109,117,115,116,32,98,101,32,101,120,112,108,105,99,105, - 116,108,121,32,112,97,115,115,101,100,32,105,110,46,10,10, - 32,32,32,32,114,167,0,0,0,114,20,0,0,0,78,114, - 55,0,0,0,41,1,114,167,0,0,0,41,16,114,46,0, - 0,0,114,14,0,0,0,114,13,0,0,0,114,79,0,0, - 0,218,5,105,116,101,109,115,114,171,0,0,0,114,69,0, - 0,0,114,142,0,0,0,114,75,0,0,0,114,152,0,0, - 0,114,128,0,0,0,114,133,0,0,0,114,1,0,0,0, - 114,197,0,0,0,114,5,0,0,0,114,70,0,0,0,41, - 12,218,10,115,121,115,95,109,111,100,117,108,101,218,11,95, - 105,109,112,95,109,111,100,117,108,101,90,11,109,111,100,117, - 108,101,95,116,121,112,101,114,15,0,0,0,114,83,0,0, - 0,114,93,0,0,0,114,82,0,0,0,90,11,115,101,108, - 102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,105, - 110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,95, - 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, - 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, - 111,100,117,108,101,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,6,95,115,101,116,117,112,65,4,0,0, - 115,50,0,0,0,0,9,4,1,4,3,8,1,20,1,10, - 1,10,1,6,1,10,1,6,2,2,1,10,1,14,3,10, - 1,10,1,10,1,10,2,10,1,16,3,2,1,12,1,14, - 2,10,1,12,3,8,1,114,201,0,0,0,99,2,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, - 0,115,66,0,0,0,116,0,124,0,124,1,131,2,1,0, - 116,1,106,2,106,3,116,4,131,1,1,0,116,1,106,2, - 106,3,116,5,131,1,1,0,100,1,100,2,108,6,125,2, - 124,2,97,7,124,2,106,8,116,1,106,9,116,10,25,0, - 131,1,1,0,100,2,83,0,41,3,122,50,73,110,115,116, - 97,108,108,32,105,109,112,111,114,116,108,105,98,32,97,115, - 32,116,104,101,32,105,109,112,108,101,109,101,110,116,97,116, - 105,111,110,32,111,102,32,105,109,112,111,114,116,46,114,19, - 0,0,0,78,41,11,114,201,0,0,0,114,14,0,0,0, - 114,166,0,0,0,114,109,0,0,0,114,142,0,0,0,114, - 152,0,0,0,218,26,95,102,114,111,122,101,110,95,105,109, - 112,111,114,116,108,105,98,95,101,120,116,101,114,110,97,108, - 114,115,0,0,0,218,8,95,105,110,115,116,97,108,108,114, - 79,0,0,0,114,1,0,0,0,41,3,114,199,0,0,0, - 114,200,0,0,0,114,202,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,203,0,0,0,112,4, - 0,0,115,12,0,0,0,0,2,10,2,12,1,12,3,8, - 1,4,1,114,203,0,0,0,41,2,78,78,41,1,78,41, - 2,78,114,19,0,0,0,41,50,114,3,0,0,0,114,115, - 0,0,0,114,12,0,0,0,114,16,0,0,0,114,51,0, - 0,0,114,29,0,0,0,114,36,0,0,0,114,17,0,0, - 0,114,18,0,0,0,114,41,0,0,0,114,42,0,0,0, - 114,45,0,0,0,114,56,0,0,0,114,58,0,0,0,114, - 68,0,0,0,114,74,0,0,0,114,77,0,0,0,114,84, - 0,0,0,114,95,0,0,0,114,96,0,0,0,114,102,0, - 0,0,114,78,0,0,0,218,6,111,98,106,101,99,116,90, - 9,95,80,79,80,85,76,65,84,69,114,128,0,0,0,114, - 133,0,0,0,114,136,0,0,0,114,91,0,0,0,114,80, - 0,0,0,114,140,0,0,0,114,141,0,0,0,114,81,0, - 0,0,114,142,0,0,0,114,152,0,0,0,114,157,0,0, - 0,114,163,0,0,0,114,165,0,0,0,114,170,0,0,0, - 114,175,0,0,0,90,15,95,69,82,82,95,77,83,71,95, - 80,82,69,70,73,88,114,177,0,0,0,114,180,0,0,0, - 114,181,0,0,0,114,182,0,0,0,114,189,0,0,0,114, - 193,0,0,0,114,196,0,0,0,114,197,0,0,0,114,201, - 0,0,0,114,203,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,8,60,109, - 111,100,117,108,101,62,8,0,0,0,115,94,0,0,0,4, - 17,4,2,8,8,8,7,4,2,4,3,16,4,14,68,14, - 21,14,19,8,19,8,19,8,11,14,8,8,11,8,12,8, - 16,8,36,14,27,14,101,16,26,6,3,10,45,14,60,8, - 17,8,17,8,25,8,29,8,23,8,16,14,73,14,77,14, - 13,8,9,8,9,10,47,8,20,4,1,8,2,8,27,8, - 16,10,15,8,31,8,27,18,35,8,7,8,47, + 0,218,8,60,109,111,100,117,108,101,62,8,0,0,0,115, + 94,0,0,0,4,17,4,2,8,8,8,8,4,2,4,3, + 16,4,14,68,14,21,14,16,8,27,8,17,8,11,14,8, + 8,11,8,12,8,16,8,36,14,27,14,101,16,26,6,3, + 10,45,14,60,8,17,8,17,8,24,8,29,8,23,8,15, + 14,73,14,77,14,13,8,9,8,9,10,47,8,20,4,1, + 8,2,8,27,8,16,10,15,8,31,8,27,18,35,8,7, + 8,47, }; From ecfe4f678bfb0e3c19c90fd7db79c5f3c76023e4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 11 Jul 2017 07:16:11 +0300 Subject: [PATCH 0554/1345] [3.6] bpo-30879: os.listdir() and os.scandir() now emit bytes names when (GH-2634) (#2656) called with bytes-like argument.. (cherry picked from commit 1180e5a51871fa53ca6892e83fd2e69dc2600447) --- Lib/test/test_os.py | 16 ++++++++++++++++ Lib/test/test_posix.py | 14 +++++++++++--- Misc/NEWS | 3 +++ Modules/posixmodule.c | 8 +++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f74c199c84146a..e4951b44ce4955 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3402,6 +3402,22 @@ def test_bytes(self): self.assertEqual(entry.path, os.fsencode(os.path.join(self.path, 'file.txt'))) + def test_bytes_like(self): + self.create_file("file.txt") + + for cls in bytearray, memoryview: + path_bytes = cls(os.fsencode(self.path)) + with self.assertWarns(DeprecationWarning): + entries = list(os.scandir(path_bytes)) + self.assertEqual(len(entries), 1, entries) + entry = entries[0] + + self.assertEqual(entry.name, b'file.txt') + self.assertEqual(entry.path, + os.fsencode(os.path.join(self.path, 'file.txt'))) + self.assertIs(type(entry.name), bytes) + self.assertIs(type(entry.path), bytes) + def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 4ca623050b8494..a6e2fed302f878 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -581,17 +581,25 @@ def test_chdir(self): self.assertRaises(OSError, posix.chdir, support.TESTFN) def test_listdir(self): - self.assertTrue(support.TESTFN in posix.listdir(os.curdir)) + self.assertIn(support.TESTFN, posix.listdir(os.curdir)) def test_listdir_default(self): # When listdir is called without argument, # it's the same as listdir(os.curdir). - self.assertTrue(support.TESTFN in posix.listdir()) + self.assertIn(support.TESTFN, posix.listdir()) def test_listdir_bytes(self): # When listdir is called with a bytes object, # the returned strings are of type bytes. - self.assertTrue(os.fsencode(support.TESTFN) in posix.listdir(b'.')) + self.assertIn(os.fsencode(support.TESTFN), posix.listdir(b'.')) + + def test_listdir_bytes_like(self): + for cls in bytearray, memoryview: + with self.assertWarns(DeprecationWarning): + names = posix.listdir(cls(b'.')) + self.assertIn(os.fsencode(support.TESTFN), names) + for name in names: + self.assertIs(type(name), bytes) @unittest.skipUnless(posix.listdir in os.supports_fd, "test needs fd support for posix.listdir()") diff --git a/Misc/NEWS b/Misc/NEWS index 3c16a66e6ffba0..3cdacc2b637fe4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,9 @@ Core and Builtins Library ------- +- bpo-30879: os.listdir() and os.scandir() now emit bytes names when called + with bytes-like argument. + - bpo-30746: Prohibited the '=' character in environment variable names in ``os.putenv()`` and ``os.spawn*()``. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c205dd9b054afd..97b57198cd4332 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -949,6 +949,8 @@ path_converter(PyObject *o, void *p) Py_INCREF(bytes); } else if (is_buffer) { + /* XXX Replace PyObject_CheckBuffer with PyBytes_Check in other code + after removing suport of non-bytes buffer objects. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", @@ -3511,8 +3513,8 @@ _posix_listdir(path_t *path, PyObject *list) const char *name; if (path->narrow) { name = path->narrow; - /* only return bytes if they specified a bytes object */ - return_str = !(PyBytes_Check(path->object)); + /* only return bytes if they specified a bytes-like object */ + return_str = !PyObject_CheckBuffer(path->object); } else { name = "."; @@ -11708,7 +11710,7 @@ DirEntry_from_posix_info(path_t *path, const char *name, Py_ssize_t name_len, if (!joined_path) goto error; - if (!path->narrow || !PyBytes_Check(path->object)) { + if (!path->narrow || !PyObject_CheckBuffer(path->object)) { entry->name = PyUnicode_DecodeFSDefaultAndSize(name, name_len); entry->path = PyUnicode_DecodeFSDefault(joined_path); } From 82a907560011c7b784badccc78082cef70ea7128 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 11 Jul 2017 07:27:56 +0300 Subject: [PATCH 0555/1345] [3.6] bpo-22207: Add checks for possible integer overflows in unicodeobject.c. (GH-2623) (#2658) Based on patch by Victor Stinner. (cherry picked from commit 64e461b) --- Objects/unicodeobject.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 494cdbd42e033b..1650370baccae9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5513,13 +5513,12 @@ _PyUnicode_EncodeUTF32(PyObject *str, /* four bytes are reserved for each surrogate */ if (moreunits > 1) { Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v); - Py_ssize_t morebytes = 4 * (moreunits - 1); - if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 4) { /* integer overflow */ PyErr_NoMemory(); goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 4 * (moreunits - 1)) < 0) goto error; out = (uint32_t*) PyBytes_AS_STRING(v) + outpos; } @@ -5865,13 +5864,12 @@ _PyUnicode_EncodeUTF16(PyObject *str, /* two bytes are reserved for each surrogate */ if (moreunits > 1) { Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v); - Py_ssize_t morebytes = 2 * (moreunits - 1); - if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 2) { /* integer overflow */ PyErr_NoMemory(); goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 2 * (moreunits - 1)) < 0) goto error; out = (unsigned short*) PyBytes_AS_STRING(v) + outpos; } @@ -6551,6 +6549,10 @@ _PyUnicode_DecodeUnicodeInternal(const char *s, 1)) return NULL; + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } if (size == 0) _Py_RETURN_UNICODE_EMPTY(); @@ -7352,6 +7354,10 @@ decode_code_page_stateful(int code_page, PyErr_SetString(PyExc_ValueError, "invalid code page number"); return NULL; } + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } if (consumed) *consumed = 0; From 953e527763f5af293668135acdf5f0a20c3f6f4f Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 11 Jul 2017 02:16:41 -0400 Subject: [PATCH 0556/1345] [3.6] bpo-30870: IDLE -- fix logic error in eae2537. (GH-2660) (#2661) (cherry picked from commit 5b62b35) --- Lib/idlelib/configdialog.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index cf5bea7ec7ece6..6dc075fbadadf2 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -157,7 +157,9 @@ def create_page_font_tab(self): frame_font_name, justify=LEFT, text='Font Face :') self.fontlist = Listbox( frame_font_name, height=5, takefocus=FALSE, exportselection=FALSE) - self.fontlist.bind('<>', self.on_fontlist_select) + self.fontlist.bind('', self.on_fontlist_select) + self.fontlist.bind('', self.on_fontlist_select) + self.fontlist.bind('', self.on_fontlist_select) scroll_font = Scrollbar(frame_font_name) scroll_font.config(command=self.fontlist.yview) self.fontlist.config(yscrollcommand=scroll_font.set) @@ -973,7 +975,8 @@ def on_fontlist_select(self, event): Event can result from either mouse click or Up or Down key. Set font_name and example display to selection. """ - font = self.fontlist.get(ANCHOR if event.type == 3 else ACTIVE) + font = self.fontlist.get( + ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) self.font_name.set(font.lower()) self.set_font_sample() From 675c1adfe5c87573e81fb295ccad7b6cfa03a3ee Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 11 Jul 2017 02:53:32 -0400 Subject: [PATCH 0557/1345] [3.6] bpo-30881: IDLE: add docstrings to browser.py (GH-2638) (#2663) Patch by Cheryl Sabella. (cherry picked from commit ba35227) --- Lib/idlelib/browser.py | 97 ++++++++++++++++++- .../2017-07-11-02-21-42.bpo-30881.4KAq_9.rst | 1 + 2 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-11-02-21-42.bpo-30881.4KAq_9.rst diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 73c251e76d3cce..4cf4744fb0a8ed 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -23,12 +23,28 @@ # Normally pyshell.flist.open, but there is no pyshell.flist for htest. class ClassBrowser: + """Browse module classes and functions in IDLE. + """ def __init__(self, flist, name, path, _htest=False): # XXX This API should change, if the file doesn't end in ".py" # XXX the code here is bogus! - """ - _htest - bool, change box when location running htest. + """Create a window for browsing a module's structure. + + Args: + flist: filelist.FileList instance used as the root for the window. + name: Python module to parse. + path: Module search path. + _htest - bool, change box when location running htest. + + Global variables: + file_open: Function used for opening a file. + + Instance variables: + name: Module name. + file: Full path and module with .py extension. Used in + creating ModuleBrowserTreeItem as the rootnode for + the tree and subsequently in the children. """ global file_open if not _htest: @@ -39,10 +55,12 @@ def __init__(self, flist, name, path, _htest=False): self.init(flist) def close(self, event=None): + "Dismiss the window and the tree nodes." self.top.destroy() self.node.destroy() def init(self, flist): + "Create browser tkinter widgets, including the tree." self.flist = flist # reset pyclbr pyclbr._modules.clear() @@ -66,24 +84,42 @@ def init(self, flist): node.expand() def settitle(self): + "Set the window title." self.top.wm_title("Class Browser - " + self.name) self.top.wm_iconname("Class Browser") def rootnode(self): + "Return a ModuleBrowserTreeItem as the root of the tree." return ModuleBrowserTreeItem(self.file) class ModuleBrowserTreeItem(TreeItem): + """Browser tree for Python module. + + Uses TreeItem as the basis for the structure of the tree. + """ def __init__(self, file): + """Create a TreeItem for the file. + + Args: + file: Full path and module name. + """ self.file = file def GetText(self): + "Return the module name as the text string to display." return os.path.basename(self.file) def GetIconName(self): + "Return the name of the icon to display." return "python" def GetSubList(self): + """Return the list of ClassBrowserTreeItem items. + + Each item returned from listclasses is the first level of + classes/functions within the module. + """ sublist = [] for name in self.listclasses(): item = ClassBrowserTreeItem(name, self.classes, self.file) @@ -91,6 +127,7 @@ def GetSubList(self): return sublist def OnDoubleClick(self): + "Open a module in an editor window when double clicked." if os.path.normcase(self.file[-3:]) != ".py": return if not os.path.exists(self.file): @@ -98,9 +135,20 @@ def OnDoubleClick(self): pyshell.flist.open(self.file) def IsExpandable(self): + "Return True if Python (.py) file." return os.path.normcase(self.file[-3:]) == ".py" def listclasses(self): + """Return list of classes and functions in the module. + + The dictionary output from pyclbr is re-written as a + list of tuples in the form (lineno, name) and + then sorted so that the classes and functions are + processed in line number order. The returned list only + contains the name and not the line number. An instance + variable self.classes contains the pyclbr dictionary values, + which are instances of Class and Function. + """ dir, file = os.path.split(self.file) name, ext = os.path.splitext(file) if os.path.normcase(ext) != ".py": @@ -134,9 +182,25 @@ def listclasses(self): return list class ClassBrowserTreeItem(TreeItem): + """Browser tree for classes within a module. + + Uses TreeItem as the basis for the structure of the tree. + """ def __init__(self, name, classes, file): + """Create a TreeItem for the class/function. + + Args: + name: Name of the class/function. + classes: Dictonary of Class/Function instances from pyclbr. + file: Full path and module name. + + Instance variables: + self.cl: Class/Function instance for the class/function name. + self.isfunction: True if self.cl is a Function. + """ self.name = name + # XXX - Does classes need to be an instance variable? self.classes = classes self.file = file try: @@ -146,25 +210,33 @@ def __init__(self, name, classes, file): self.isfunction = isinstance(self.cl, pyclbr.Function) def GetText(self): + "Return the name of the function/class to display." if self.isfunction: return "def " + self.name + "(...)" else: return "class " + self.name def GetIconName(self): + "Return the name of the icon to display." if self.isfunction: return "python" else: return "folder" def IsExpandable(self): + "Return True if this class has methods." if self.cl: try: return not not self.cl.methods except AttributeError: return False + return None def GetSubList(self): + """Return Class methods as a list of MethodBrowserTreeItem items. + + Each item is a method within the class. + """ if not self.cl: return [] sublist = [] @@ -174,6 +246,7 @@ def GetSubList(self): return sublist def OnDoubleClick(self): + "Open module with file_open and position to lineno, if it exists." if not os.path.exists(self.file): return edit = file_open(self.file) @@ -182,6 +255,7 @@ def OnDoubleClick(self): edit.gotoline(lineno) def listmethods(self): + "Return list of methods within a class sorted by lineno." if not self.cl: return [] items = [] @@ -194,22 +268,37 @@ def listmethods(self): return list class MethodBrowserTreeItem(TreeItem): + """Browser tree for methods within a class. + + Uses TreeItem as the basis for the structure of the tree. + """ def __init__(self, name, cl, file): + """Create a TreeItem for the methods. + + Args: + name: Name of the class/function. + cl: pyclbr.Class instance for name. + file: Full path and module name. + """ self.name = name self.cl = cl self.file = file def GetText(self): + "Return the method name to display." return "def " + self.name + "(...)" def GetIconName(self): - return "python" # XXX + "Return the name of the icon to display." + return "python" def IsExpandable(self): - return 0 + "Return False as there are no tree items after methods." + return False def OnDoubleClick(self): + "Open module with file_open and position at the method start." if not os.path.exists(self.file): return edit = file_open(self.file) diff --git a/Misc/NEWS.d/next/IDLE/2017-07-11-02-21-42.bpo-30881.4KAq_9.rst b/Misc/NEWS.d/next/IDLE/2017-07-11-02-21-42.bpo-30881.4KAq_9.rst new file mode 100644 index 00000000000000..4e4e0e69a01e2e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-11-02-21-42.bpo-30881.4KAq_9.rst @@ -0,0 +1 @@ +IDLE: Add docstrings to browser.py. Patch by Cheryl Sabella. From c0179483f13be81910ed73889dcad92528e20ef2 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Tue, 11 Jul 2017 19:50:10 -0400 Subject: [PATCH 0558/1345] [3.6] bpo-30779: IDLE: fix changes.delete_section calls in configdialog (GH-2667) (#2674) Also improve test of config.ConfigChanges.delete_section. Original patch by Cheryl Sabella. (cherry picked from commit 6d13b22) --- Lib/idlelib/config.py | 3 ++- Lib/idlelib/configdialog.py | 4 ++-- Lib/idlelib/idle_test/test_config.py | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index ed37f11a9cb8e0..bbb3e3cde52251 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -794,7 +794,8 @@ class ConfigChanges(dict): add_option: Add option and value to changes. save_option: Save option and value to config parser. save_all: Save all the changes to the config parser and file. - delete_section: Delete section if it exists. + delete_section: If section exists, + delete from changes, userCfg, and file. clear: Clear all changes by clearing each page. """ def __init__(self): diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 6dc075fbadadf2..a2cfaab0d0acbe 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -839,7 +839,7 @@ def delete_custom_keys(self): return self.deactivate_current_config() # Remove key set from changes, config, and file. - changes.remove(keyset_name) + changes.delete_section('keys', keyset_name) # Reload user key set list. item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() @@ -873,7 +873,7 @@ def delete_custom_theme(self): return self.deactivate_current_config() # Remove theme from changes, config, and file. - changes.delete_section('highlight') + changes.delete_section('highlight', theme_name) # Reload user theme list. item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index e157bbb5b52c46..a8e3a3b89d3e7d 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -188,6 +188,7 @@ def test_save_added(self): userkeys.remove_section('Ksec') def test_save_help(self): + # Any change to HelpFiles overwrites entire section. changes = self.changes changes.save_option('main', 'HelpFiles', 'IDLE', 'idledoc') changes.add_option('main', 'HelpFiles', 'ELDI', 'codeldi') @@ -207,10 +208,12 @@ def test_delete_section(self): changes.delete_section('main', 'fake') # Test no exception. self.assertEqual(changes, self.loaded) # Test nothing deleted. for cfgtype, section in (('main', 'Msec'), ('keys', 'Ksec')): + testcfg[cfgtype].SetOption(section, 'name', 'value') changes.delete_section(cfgtype, section) with self.assertRaises(KeyError): - changes[cfgtype][section] # Test section gone. - # TODO Test change to userkeys and maybe save call. + changes[cfgtype][section] # Test section gone from changes + testcfg[cfgtype][section] # and from mock userCfg. + # TODO test for save call. def test_clear(self): changes = self.load() From a9bf62cadf6ccd55b84a8d4f353529b02a537830 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Wed, 12 Jul 2017 14:42:37 -0400 Subject: [PATCH 0559/1345] [3.6] bpo-30899: Add unittests, 100% coverage, for IDLE's two ConfigParser subclasses. (GH-2662) (#2685) Patch by Louie Lu. (cherry picked from commit 50c9435) --- Lib/idlelib/config.py | 57 +++--- Lib/idlelib/idle_test/test_config.py | 169 +++++++++++++++++- Lib/test/cfgparser.1 | 1 + .../2017-07-11-02-26-17.bpo-30899.SQmVO8.rst | 1 + 4 files changed, 198 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-11-02-26-17.bpo-30899.SQmVO8.rst diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index bbb3e3cde52251..4cf44271f3a750 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -81,31 +81,6 @@ class IdleUserConfParser(IdleConfParser): IdleConfigParser specialised for user configuration handling. """ - def AddSection(self, section): - "If section doesn't exist, add it." - if not self.has_section(section): - self.add_section(section) - - def RemoveEmptySections(self): - "Remove any sections that have no options." - for section in self.sections(): - if not self.GetOptionList(section): - self.remove_section(section) - - def IsEmpty(self): - "Return True if no sections after removing empty sections." - self.RemoveEmptySections() - return not self.sections() - - def RemoveOption(self, section, option): - """Return True if option is removed from section, else False. - - False if either section does not exist or did not have option. - """ - if self.has_section(section): - return self.remove_option(section, option) - return False - def SetOption(self, section, option, value): """Return True if option is added or changed to value, else False. @@ -123,6 +98,31 @@ def SetOption(self, section, option, value): self.set(section, option, value) return True + def RemoveOption(self, section, option): + """Return True if option is removed from section, else False. + + False if either section does not exist or did not have option. + """ + if self.has_section(section): + return self.remove_option(section, option) + return False + + def AddSection(self, section): + "If section doesn't exist, add it." + if not self.has_section(section): + self.add_section(section) + + def RemoveEmptySections(self): + "Remove any sections that have no options." + for section in self.sections(): + if not self.GetOptionList(section): + self.remove_section(section) + + def IsEmpty(self): + "Return True if no sections after removing empty sections." + self.RemoveEmptySections() + return not self.sections() + def RemoveFile(self): "Remove user config file self.file from disk if it exists." if os.path.exists(self.file): @@ -829,9 +829,12 @@ def save_option(config_type, section, item, value): def save_all(self): """Save configuration changes to the user config file. - Then clear self in preparation for additional changes. + Clear self in preparation for additional changes. + Return changed for testing. """ idleConf.userCfg['main'].Save() + + changed = False for config_type in self: cfg_type_changed = False page = self[config_type] @@ -844,12 +847,14 @@ def save_all(self): cfg_type_changed = True if cfg_type_changed: idleConf.userCfg[config_type].Save() + changed = True for config_type in ['keys', 'highlight']: # Save these even if unchanged! idleConf.userCfg[config_type].Save() self.clear() # ConfigDialog caller must add the following call # self.save_all_changed_extensions() # Uses a different mechanism. + return changed def delete_section(self, config_type, section): """Delete a section from self, userCfg, and file. diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index a8e3a3b89d3e7d..f5a609f6d1600b 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -1,9 +1,13 @@ '''Test idlelib.config. -Much is tested by opening config dialog live or in test_configdialog. -Coverage: 27% +Coverage: 46% (100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). +* Except is OSError clause in Save method. +Much of IdleConf is exercised by ConfigDialog and test_configdialog, +but it should be tested here. ''' -from test.support import captured_stderr +import os +import tempfile +from test.support import captured_stderr, findfile import unittest from idlelib import config @@ -26,6 +30,161 @@ def tearDownModule(): idleConf.userCfg = usercfg +class IdleConfParserTest(unittest.TestCase): + """Test that IdleConfParser works""" + + config = """ + [one] + one = false + two = true + three = 10 + + [two] + one = a string + two = true + three = false + """ + + def test_get(self): + parser = config.IdleConfParser('') + parser.read_string(self.config) + eq = self.assertEqual + + # Test with type argument. + self.assertIs(parser.Get('one', 'one', type='bool'), False) + self.assertIs(parser.Get('one', 'two', type='bool'), True) + eq(parser.Get('one', 'three', type='int'), 10) + eq(parser.Get('two', 'one'), 'a string') + self.assertIs(parser.Get('two', 'two', type='bool'), True) + self.assertIs(parser.Get('two', 'three', type='bool'), False) + + # Test without type should fallback to string. + eq(parser.Get('two', 'two'), 'true') + eq(parser.Get('two', 'three'), 'false') + + # If option not exist, should return None, or default. + self.assertIsNone(parser.Get('not', 'exist')) + eq(parser.Get('not', 'exist', default='DEFAULT'), 'DEFAULT') + + def test_get_option_list(self): + parser = config.IdleConfParser('') + parser.read_string(self.config) + get_list = parser.GetOptionList + self.assertCountEqual(get_list('one'), ['one', 'two', 'three']) + self.assertCountEqual(get_list('two'), ['one', 'two', 'three']) + self.assertEqual(get_list('not exist'), []) + + def test_load_nothing(self): + parser = config.IdleConfParser('') + parser.Load() + self.assertEqual(parser.sections(), []) + + def test_load_file(self): + # Borrow test/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1') + parser = config.IdleConfParser(config_path) + parser.Load() + + self.assertEqual(parser.Get('Foo Bar', 'foo'), 'newbar') + self.assertEqual(parser.GetOptionList('Foo Bar'), ['foo']) + + +class IdleUserConfParserTest(unittest.TestCase): + """Test that IdleUserConfParser works""" + + def new_parser(self, path=''): + return config.IdleUserConfParser(path) + + def test_set_option(self): + parser = self.new_parser() + parser.add_section('Foo') + # Setting new option in existing section should return True. + self.assertTrue(parser.SetOption('Foo', 'bar', 'true')) + # Setting existing option with same value should return False. + self.assertFalse(parser.SetOption('Foo', 'bar', 'true')) + # Setting exiting option with new value should return True. + self.assertTrue(parser.SetOption('Foo', 'bar', 'false')) + self.assertEqual(parser.Get('Foo', 'bar'), 'false') + + # Setting option in new section should create section and return True. + self.assertTrue(parser.SetOption('Bar', 'bar', 'true')) + self.assertCountEqual(parser.sections(), ['Bar', 'Foo']) + self.assertEqual(parser.Get('Bar', 'bar'), 'true') + + def test_remove_option(self): + parser = self.new_parser() + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + + self.assertTrue(parser.RemoveOption('Foo', 'bar')) + self.assertFalse(parser.RemoveOption('Foo', 'bar')) + self.assertFalse(parser.RemoveOption('Not', 'Exist')) + + def test_add_section(self): + parser = self.new_parser() + self.assertEqual(parser.sections(), []) + + # Should not add duplicate section. + # Configparser raises DuplicateError, IdleParser not. + parser.AddSection('Foo') + parser.AddSection('Foo') + parser.AddSection('Bar') + self.assertCountEqual(parser.sections(), ['Bar', 'Foo']) + + def test_remove_empty_sections(self): + parser = self.new_parser() + + parser.AddSection('Foo') + parser.AddSection('Bar') + parser.SetOption('Idle', 'name', 'val') + self.assertCountEqual(parser.sections(), ['Bar', 'Foo', 'Idle']) + parser.RemoveEmptySections() + self.assertEqual(parser.sections(), ['Idle']) + + def test_is_empty(self): + parser = self.new_parser() + + parser.AddSection('Foo') + parser.AddSection('Bar') + self.assertTrue(parser.IsEmpty()) + self.assertEqual(parser.sections(), []) + + parser.SetOption('Foo', 'bar', 'false') + parser.AddSection('Bar') + self.assertFalse(parser.IsEmpty()) + self.assertCountEqual(parser.sections(), ['Foo']) + + def test_remove_file(self): + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 'test.cfg') + parser = self.new_parser(path) + parser.RemoveFile() # Should not raise exception. + + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + parser.Save() + self.assertTrue(os.path.exists(path)) + parser.RemoveFile() + self.assertFalse(os.path.exists(path)) + + def test_save(self): + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 'test.cfg') + parser = self.new_parser(path) + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + + # Should save to path when config is not empty. + self.assertFalse(os.path.exists(path)) + parser.Save() + self.assertTrue(os.path.exists(path)) + + # Should remove the file from disk when config is empty. + parser.remove_section('Foo') + parser.Save() + self.assertFalse(os.path.exists(path)) + + class CurrentColorKeysTest(unittest.TestCase): """ Test colorkeys function with user config [Theme] and [Keys] patterns. @@ -179,10 +338,12 @@ def test_save_option(self): # Static function does not touch changes. def test_save_added(self): changes = self.load() - changes.save_all() + self.assertTrue(changes.save_all()) self.assertEqual(usermain['Msec']['mitem'], 'mval') self.assertEqual(userhigh['Hsec']['hitem'], 'hval') self.assertEqual(userkeys['Ksec']['kitem'], 'kval') + changes.add_option('main', 'Msec', 'mitem', 'mval') + self.assertFalse(changes.save_all()) usermain.remove_section('Msec') userhigh.remove_section('Hsec') userkeys.remove_section('Ksec') diff --git a/Lib/test/cfgparser.1 b/Lib/test/cfgparser.1 index 3387f52eca6f94..0c0e0d35273a39 100644 --- a/Lib/test/cfgparser.1 +++ b/Lib/test/cfgparser.1 @@ -1,2 +1,3 @@ +# Also used by idlelib.test_idle.test_config. [Foo Bar] foo=newbar diff --git a/Misc/NEWS.d/next/IDLE/2017-07-11-02-26-17.bpo-30899.SQmVO8.rst b/Misc/NEWS.d/next/IDLE/2017-07-11-02-26-17.bpo-30899.SQmVO8.rst new file mode 100644 index 00000000000000..665c7cd5bbf86b --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-11-02-26-17.bpo-30899.SQmVO8.rst @@ -0,0 +1 @@ +IDLE: Add tests for ConfigParser subclasses in config. Patch by Louie Lu. From 7d0a99582050ccac2e7126484d23072be342f317 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Jul 2017 11:40:25 +0300 Subject: [PATCH 0560/1345] [3.6] bpo-30911: Add tests for bad boolean arguments for accelerated json (GH-2690) (#2692) encoder and decoder. (cherry picked from commit d3aaa2f) --- Lib/test/test_json/test_speedups.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 109a2466c21bdb..8d98ab5ae1c398 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,6 +1,11 @@ from test.test_json import CTest +class BadBool: + def __bool__(self): + 1/0 + + class TestSpeedups(CTest): def test_scanstring(self): self.assertEqual(self.json.decoder.scanstring.__module__, "_json") @@ -17,8 +22,25 @@ class TestDecode(CTest): def test_make_scanner(self): self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1) + def test_bad_bool_args(self): + def test(value): + self.json.decoder.JSONDecoder(strict=BadBool()).decode(value) + self.assertRaises(ZeroDivisionError, test, '""') + self.assertRaises(ZeroDivisionError, test, '{}') + + +class TestEncode(CTest): def test_make_encoder(self): self.assertRaises(TypeError, self.json.encoder.c_make_encoder, (True, False), b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) + + def test_bad_bool_args(self): + def test(name): + self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) + self.assertRaises(ZeroDivisionError, test, 'skipkeys') + self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') + self.assertRaises(ZeroDivisionError, test, 'check_circular') + self.assertRaises(ZeroDivisionError, test, 'allow_nan') + self.assertRaises(ZeroDivisionError, test, 'sort_keys') From d8e522f7cf8d59993acae9409b6af0ee9a35038f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 13 Jul 2017 21:51:50 +0200 Subject: [PATCH 0561/1345] bpo-30731: python.manifest fix (#2328) (#2699) bpo-30731: python.manifest fix --- .../Windows/2017-07-13-11-22-53.bpo-30731.nmMDwI.rst | 1 + PC/python.manifest | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2017-07-13-11-22-53.bpo-30731.nmMDwI.rst diff --git a/Misc/NEWS.d/next/Windows/2017-07-13-11-22-53.bpo-30731.nmMDwI.rst b/Misc/NEWS.d/next/Windows/2017-07-13-11-22-53.bpo-30731.nmMDwI.rst new file mode 100644 index 00000000000000..8a65a9e022c073 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-07-13-11-22-53.bpo-30731.nmMDwI.rst @@ -0,0 +1 @@ +Add a missing xmlns to python.manifest so that it matches the schema. diff --git a/PC/python.manifest b/PC/python.manifest index 4e73d60df2842d..8e1bc022adfb4f 100644 --- a/PC/python.manifest +++ b/PC/python.manifest @@ -1,20 +1,20 @@ - + - - - + + + - + From 42abf7f9737f78a5da311a42945d781dfcd6c6c0 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 13 Jul 2017 22:24:55 -0400 Subject: [PATCH 0562/1345] [3.6] bpo-30870: IDLE: Add configdialog fontlist selection unittest (GH-2666) (#2701) Initial patch by Louie Lu. (cherry picked from commit 9b622fb) --- Lib/idlelib/configdialog.py | 5 +- Lib/idlelib/idle_test/mock_idle.py | 19 ++-- Lib/idlelib/idle_test/test_configdialog.py | 113 +++++++++++++++++---- 3 files changed, 108 insertions(+), 29 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index a2cfaab0d0acbe..0d68b8081e3805 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -47,7 +47,8 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.parent = parent if _htest: parent.instance_dict = {} - self.withdraw() + if not _utest: + self.withdraw() self.configure(borderwidth=5) self.title(title or 'IDLE Preferences') @@ -76,7 +77,6 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.create_widgets() self.resizable(height=FALSE, width=FALSE) self.transient(parent) - self.grab_set() self.protocol("WM_DELETE_WINDOW", self.cancel) self.fontlist.focus_set() # XXX Decide whether to keep or delete these key bindings. @@ -88,6 +88,7 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.attach_var_callbacks() # Avoid callbacks during load_configs. if not _utest: + self.grab_set() self.wm_deiconify() self.wait_window() diff --git a/Lib/idlelib/idle_test/mock_idle.py b/Lib/idlelib/idle_test/mock_idle.py index c7b49ef02b8851..8f3147b1c5ce3d 100644 --- a/Lib/idlelib/idle_test/mock_idle.py +++ b/Lib/idlelib/idle_test/mock_idle.py @@ -6,24 +6,25 @@ from idlelib.idle_test.mock_tk import Text class Func: - '''Mock function captures args and returns result set by test. + '''Record call, capture args, return/raise result set by test. - Attributes: - self.called - records call even if no args, kwds passed. - self.result - set by init, returned by call. - self.args - captures positional arguments. - self.kwds - captures keyword arguments. + When mock function is called, set or use attributes: + self.called - increment call number even if no args, kwds passed. + self.args - capture positional arguments. + self.kwds - capture keyword arguments. + self.result - return or raise value set in __init__. - Most common use will probably be to mock methods. + Most common use will probably be to mock instance methods. + Given class instance, can set and delete as instance attribute. Mock_tk.Var and Mbox_func are special variants of this. ''' def __init__(self, result=None): - self.called = False + self.called = 0 self.result = result self.args = None self.kwds = None def __call__(self, *args, **kwds): - self.called = True + self.called += 1 self.args = args self.kwds = kwds if isinstance(self.result, BaseException): diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 26b045d353550b..4f1f9af180abb1 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -1,7 +1,7 @@ """Test idlelib.configdialog. Half the class creates dialog, half works with user customizations. -Coverage: 46% just by creating dialog, 56% with current tests. +Coverage: 46% just by creating dialog, 60% with current tests. """ from idlelib.configdialog import ConfigDialog, idleConf, changes from test.support import requires @@ -9,6 +9,7 @@ from tkinter import Tk import unittest import idlelib.config as config +from idlelib.idle_test.mock_idle import Func # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. @@ -22,27 +23,29 @@ } root = None -configure = None +dialog = None mainpage = changes['main'] highpage = changes['highlight'] keyspage = changes['keys'] -class TestDialog(ConfigDialog): pass # Delete? + +class TestDialog(ConfigDialog): + pass # Delete? def setUpModule(): - global root, configure + global root, dialog idleConf.userCfg = testcfg root = Tk() - root.withdraw() - configure = TestDialog(root, 'Test', _utest=True) + # root.withdraw() # Comment out, see issue 30870 + dialog = TestDialog(root, 'Test', _utest=True) def tearDownModule(): - global root, configure + global root, dialog idleConf.userCfg = usercfg - configure.remove_var_callbacks() - del configure + dialog.remove_var_callbacks() + del dialog root.update_idletasks() root.destroy() del root @@ -58,31 +61,105 @@ def test_font(self): default_font = idleConf.GetFont(root, 'main', 'EditorWindow') default_size = str(default_font[1]) default_bold = default_font[2] == 'bold' - configure.font_name.set('Test Font') + dialog.font_name.set('Test Font') expected = {'EditorWindow': {'font': 'Test Font', 'font-size': default_size, 'font-bold': str(default_bold)}} self.assertEqual(mainpage, expected) changes.clear() - configure.font_size.set(20) + dialog.font_size.set(20) expected = {'EditorWindow': {'font': 'Test Font', 'font-size': '20', 'font-bold': str(default_bold)}} self.assertEqual(mainpage, expected) changes.clear() - configure.font_bold.set(not default_bold) + dialog.font_bold.set(not default_bold) expected = {'EditorWindow': {'font': 'Test Font', 'font-size': '20', 'font-bold': str(not default_bold)}} self.assertEqual(mainpage, expected) - #def test_sample(self): pass # TODO + def test_set_sample(self): + # Set_font_sample also sets highlight_sample. + pass def test_tabspace(self): - configure.space_num.set(6) + dialog.space_num.set(6) self.assertEqual(mainpage, {'Indent': {'num-spaces': '6'}}) +class FontSelectTest(unittest.TestCase): + # These two functions test that selecting a new font in the + # list of fonts changes font_name and calls set_font_sample. + # The fontlist widget and on_fontlist_select event handler + # are tested here together. + + @classmethod + def setUpClass(cls): + if dialog.fontlist.size() < 2: + cls.skipTest('need at least 2 fonts') + dialog.set_font_sample = Func() # Mask instance method. + + @classmethod + def tearDownClass(cls): + del dialog.set_font_sample # Unmask instance method. + + def setUp(self): + dialog.set_font_sample.called = 0 + changes.clear() + + def test_select_font_key(self): + # Up and Down keys should select a new font. + + fontlist = dialog.fontlist + fontlist.activate(0) + font = dialog.fontlist.get('active') + + # Test Down key. + fontlist.focus_force() + fontlist.update() + fontlist.event_generate('') + fontlist.event_generate('') + + down_font = fontlist.get('active') + self.assertNotEqual(down_font, font) + self.assertIn(dialog.font_name.get(), down_font.lower()) + self.assertEqual(dialog.set_font_sample.called, 1) + + # Test Up key. + fontlist.focus_force() + fontlist.update() + fontlist.event_generate('') + fontlist.event_generate('') + + up_font = fontlist.get('active') + self.assertEqual(up_font, font) + self.assertIn(dialog.font_name.get(), up_font.lower()) + self.assertEqual(dialog.set_font_sample.called, 2) + + def test_select_font_mouse(self): + # Click on item should select that item. + + fontlist = dialog.fontlist + fontlist.activate(0) + + # Select next item in listbox + fontlist.focus_force() + fontlist.see(1) + fontlist.update() + x, y, dx, dy = fontlist.bbox(1) + x += dx // 2 + y += dy // 2 + fontlist.event_generate('', x=x, y=y) + fontlist.event_generate('', x=x, y=y) + + font1 = fontlist.get(1) + select_font = fontlist.get('anchor') + self.assertEqual(select_font, font1) + self.assertIn(dialog.font_name.get(), font1.lower()) + self.assertEqual(dialog.set_font_sample.called, 1) + + class HighlightTest(unittest.TestCase): def setUp(self): @@ -103,19 +180,19 @@ def setUp(self): changes.clear() def test_startup(self): - configure.radio_startup_edit.invoke() + dialog.radio_startup_edit.invoke() self.assertEqual(mainpage, {'General': {'editor-on-startup': '1'}}) def test_autosave(self): - configure.radio_save_auto.invoke() + dialog.radio_save_auto.invoke() self.assertEqual(mainpage, {'General': {'autosave': '1'}}) def test_editor_size(self): - configure.entry_win_height.insert(0, '1') + dialog.entry_win_height.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'height': '140'}}) changes.clear() - configure.entry_win_width.insert(0, '1') + dialog.entry_win_width.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'width': '180'}}) #def test_help_sources(self): pass # TODO From 9a09c667cf133e722a3e7ed2185f3e9e5af25a99 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 13 Jul 2017 23:53:30 -0400 Subject: [PATCH 0563/1345] [3.6] bpo-30913: IDLE: Document tk vars, attributes, and methods for ConfigDialog (GH-2697) (#2702) The will help writing dialog improvements and splitting the class into multiple classes. Original patch by Cheryl Sabella. (cherry picked from commit 36329a4) --- Lib/idlelib/configdialog.py | 482 ++++++++++++++++-- .../2017-07-13-23-07-33.bpo-30913.aezn_e.rst | 3 + 2 files changed, 440 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-13-23-07-33.bpo-30913.aezn_e.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 0d68b8081e3805..c36f0f3f67bed0 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -38,10 +38,17 @@ class ConfigDialog(Toplevel): def __init__(self, parent, title='', _htest=False, _utest=False): """Show the tabbed dialog for user configuration. - parent - parent of this dialog - title - string which is the title of this popup dialog - _htest - bool, change box location when running htest - _utest - bool, don't wait_window when running unittest + Args: + parent - parent of this dialog + title - string which is the title of this popup dialog + _htest - bool, change box location when running htest + _utest - bool, don't wait_window when running unittest + + Note: Focus set on font page fontlist. + + Methods: + create_widgets + cancel: Bound to DELETE_WINDOW protocol. """ Toplevel.__init__(self, parent) self.parent = parent @@ -58,22 +65,6 @@ def __init__(self, parent, title='', _htest=False, _utest=False): # Each theme element key is its display name. # The first value of the tuple is the sample area tag name. # The second value is the display name list sort index. - self.theme_elements={ - 'Normal Text': ('normal', '00'), - 'Python Keywords': ('keyword', '01'), - 'Python Definitions': ('definition', '02'), - 'Python Builtins': ('builtin', '03'), - 'Python Comments': ('comment', '04'), - 'Python Strings': ('string', '05'), - 'Selected Text': ('hilite', '06'), - 'Found Text': ('hit', '07'), - 'Cursor': ('cursor', '08'), - 'Editor Breakpoint': ('break', '09'), - 'Shell Normal Text': ('console', '10'), - 'Shell Error Text': ('error', '11'), - 'Shell Stdout Text': ('stdout', '12'), - 'Shell Stderr Text': ('stderr', '13'), - } self.create_widgets() self.resizable(height=FALSE, width=FALSE) self.transient(parent) @@ -93,7 +84,23 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.wait_window() def create_widgets(self): - "Create and place widgets for tabbed dialog." + """Create and place widgets for tabbed dialog. + + Widgets Bound to self: + tab_pages: TabbedPageSet + + Methods: + create_page_font_tab + create_page_highlight + create_page_keys + create_page_general + create_page_extensions + create_action_buttons + load_configs: Load pages except for extensions. + attach_var_callbacks + remove_var_callbacks + activate_config_changes: Tell editors to reload. + """ self.tab_pages = TabbedPageSet(self, page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', 'Extensions']) @@ -106,7 +113,23 @@ def create_widgets(self): self.create_action_buttons().pack(side=BOTTOM) def create_action_buttons(self): - "Return frame of action buttons for dialog." + """Return frame of action buttons for dialog. + + Methods: + ok + apply + cancel + help + + Widget Structure: + outer: Frame + buttons: Frame + (no assignment): Button (ok) + (no assignment): Button (apply) + (no assignment): Button (cancel) + (no assignment): Button (help) + (no assignment): Frame + """ if macosx.isAquaTk(): # Changing the default padding on OSX results in unreadable # text in the buttons. @@ -130,12 +153,41 @@ def create_action_buttons(self): def create_page_font_tab(self): """Return frame of widgets for Font/Tabs tab. - Configuration attributes: + Tk Variables: font_size: Font size. font_bold: Select font bold or not. font_name: Font face. + Note: these 3 share var_changed_font callback. space_num: Indentation width. + + Data Attribute: edit_font: Font widget with default font name, size, and weight. + + Methods: + load_font_cfg: Set vars and fontlist. + on_fontlist_select: Bound to fontlist button release + or key release. + set_font_sample: Command for opt_menu_font_size and + check_font_bold. + load_tab_cfg: Get current. + + Widget Structure: (*) widgets bound to self + frame + frame_font: LabelFrame + frame_font_name: Frame + font_name_title: Label + (*)fontlist: ListBox + scroll_font: Scrollbar + frame_font_param: Frame + font_size_title: Label + (*)opt_menu_font_size: DynOptionMenu - font_size + check_font_bold: Checkbutton - font_bold + frame_font_sample: Frame + (*)font_sample: Label + frame_indent: LabelFrame + frame_indent_size: Frame + indent_size_title: Label + (*)scale_indent_size: Scale - space_num """ parent = self.parent self.font_size = StringVar(parent) @@ -208,14 +260,71 @@ def create_page_font_tab(self): def create_page_highlight(self): """Return frame of widgets for Highlighting tab. - Configuration attributes: + Tk Variables: + colour: Color of selected target. builtin_theme: Menu variable for built-in theme. custom_theme: Menu variable for custom theme. fg_bg_toggle: Toggle for foreground/background color. - colour: Color of selected target. + Note: this has no callback. is_builtin_theme: Selector for built-in or custom theme. highlight_target: Menu variable for the highlight tag target. + + Instance Data Attributes: + theme_elements: Dictionary of tags for text highlighting. + The key is the display name and the value is a tuple of + (tag name, display sort order). + + Methods [attachment]: + load_theme_cfg: Load current highlight colors. + get_colour: Invoke colorchooser [button_set_colour]. + set_colour_sample_binding: Call set_colour_sample [fg_bg_toggle]. + set_highlight_target: set fg_bg_toggle, set_color_sample(). + set_colour_sample: Set frame background to target. + on_new_colour_set: Set new color and add option. + paint_theme_sample: Recolor sample. + get_new_theme_name: Get from popup. + create_new_theme: Combine theme with changes and save. + save_as_new_theme: Save [button_save_custom_theme]. + set_theme_type: Command for [is_builtin_theme]. + delete_custom_theme: Ativate default [button_delete_custom_theme]. + save_new_theme: Save to userCfg['theme'] (is function). + + Widget Structure: (*) widgets bound to self + frame + frame_custom: LabelFrame + (*)text_highlight_sample: Text + (*)frame_colour_set: Frame + button_set_colour: Button + (*)opt_menu_highlight_target: DynOptionMenu - highlight_target + frame_fg_bg_toggle: Frame + (*)radio_fg: Radiobutton - fg_bg_toggle + (*)radio_bg: Radiobutton - fg_bg_toggle + button_save_custom_theme: Button + frame_theme: LabelFrame + theme_type_title: Label + (*)radio_theme_builtin: Radiobutton - is_builtin_theme + (*)radio_theme_custom: Radiobutton - is_builtin_theme + (*)opt_menu_theme_builtin: DynOptionMenu - builtin_theme + (*)opt_menu_theme_custom: DynOptionMenu - custom_theme + (*)button_delete_custom_theme: Button + (*)new_custom_theme: Label """ + self.theme_elements={ + 'Normal Text': ('normal', '00'), + 'Python Keywords': ('keyword', '01'), + 'Python Definitions': ('definition', '02'), + 'Python Builtins': ('builtin', '03'), + 'Python Comments': ('comment', '04'), + 'Python Strings': ('string', '05'), + 'Selected Text': ('hilite', '06'), + 'Found Text': ('hit', '07'), + 'Cursor': ('cursor', '08'), + 'Editor Breakpoint': ('break', '09'), + 'Shell Normal Text': ('console', '10'), + 'Shell Error Text': ('error', '11'), + 'Shell Stdout Text': ('stdout', '12'), + 'Shell Stderr Text': ('stderr', '13'), + } parent = self.parent self.builtin_theme = StringVar(parent) self.custom_theme = StringVar(parent) @@ -328,11 +437,44 @@ def tem(event, elem=element): def create_page_keys(self): """Return frame of widgets for Keys tab. - Configuration attributes: + Tk Variables: builtin_keys: Menu variable for built-in keybindings. custom_keys: Menu variable for custom keybindings. are_keys_builtin: Selector for built-in or custom keybindings. keybinding: Action/key bindings. + + Methods: + load_key_config: Set table. + load_keys_list: Reload active set. + keybinding_selected: Bound to list_bindings button release. + get_new_keys: Command for button_new_keys. + get_new_keys_name: Call popup. + create_new_key_set: Combine active keyset and changes. + set_keys_type: Command for are_keys_builtin. + delete_custom_keys: Command for button_delete_custom_keys. + save_as_new_key_set: Command for button_save_custom_keys. + save_new_key_set: Save to idleConf.userCfg['keys'] (is function). + deactivate_current_config: Remove keys bindings in editors. + + Widget Structure: (*) widgets bound to self + frame + frame_custom: LabelFrame + frame_target: Frame + target_title: Label + scroll_target_y: Scrollbar + scroll_target_x: Scrollbar + (*)list_bindings: ListBox + (*)button_new_keys: Button + frame_key_sets: LabelFrame + frames[0]: Frame + (*)radio_keys_builtin: Radiobutton - are_keys_builtin + (*)radio_keys_custom: Radiobutton - are_keys_builtin + (*)opt_menu_keys_builtin: DynOptionMenu - builtin_keys + (*)opt_menu_keys_custom: DynOptionMenu - custom_keys + (*)new_custom_keys: Label + frames[1]: Frame + (*)button_delete_custom_keys: Button + button_save_custom_keys: Button """ parent = self.parent self.builtin_keys = StringVar(parent) @@ -414,11 +556,45 @@ def create_page_keys(self): def create_page_general(self): """Return frame of widgets for General tab. - Configuration attributes: + Tk Variables: win_width: Initial window width in characters. win_height: Initial window height in characters. startup_edit: Selector for opening in editor or shell mode. autosave: Selector for save prompt popup when using Run. + + Methods: + load_general_config: + help_source_selected: Bound to list_help button release. + set_helplist_button_states: Toggle based on list. + helplist_item_edit: Command for button_helplist_edit. + helplist_item_add: Command for button_helplist_add. + helplist_item_remove: Command for button_helplist_remove. + update_user_help_changed_items: Fill in changes. + + Widget Structure: (*) widgets bound to self + frame + frame_run: LabelFrame + startup_title: Label + (*)radio_startup_edit: Radiobutton - startup_edit + (*)radio_startup_shell: Radiobutton - startup_edit + frame_save: LabelFrame + run_save_title: Label + (*)radio_save_ask: Radiobutton - autosave + (*)radio_save_auto: Radiobutton - autosave + frame_win_size: LabelFrame + win_size_title: Label + win_width_title: Label + (*)entry_win_width: Entry - win_width + win_height_title: Label + (*)entry_win_height: Entry - win_height + frame_help: LabelFrame + frame_helplist: Frame + frame_helplist_buttons: Frame + (*)button_helplist_edit + (*)button_helplist_add + (*)button_helplist_remove + scroll_helplist: Scrollbar + (*)list_help: ListBox """ parent = self.parent self.win_width = StringVar(parent) @@ -684,7 +860,23 @@ def var_changed_autosave(self, *params): changes.add_option('main', 'General', 'autosave', value) def set_theme_type(self): - "Set available screen options based on builtin or custom theme." + """Set available screen options based on builtin or custom theme. + + Attributes accessed: + is_builtin_theme + + Attributes updated: + opt_menu_theme_builtin + opt_menu_theme_custom + button_delete_custom_theme + radio_theme_custom + + Called from: + handler for radio_theme_builtin and radio_theme_custom + delete_custom_theme + create_new_theme + load_theme_cfg + """ if self.is_builtin_theme.get(): self.opt_menu_theme_builtin.config(state=NORMAL) self.opt_menu_theme_custom.config(state=DISABLED) @@ -866,6 +1058,21 @@ def delete_custom_theme(self): The current theme is deactivated and the default theme is activated. The custom theme is permanently removed from the config file. + + Attributes accessed: + custom_theme + + Attributes updated: + radio_theme_custom + opt_menu_theme_custom + is_builtin_theme + builtin_theme + + Methods: + deactivate_current_config + save_all_changed_extensions + activate_config_changes + set_theme_type """ theme_name = self.custom_theme.get() delmsg = 'Are you sure you wish to delete the theme %r ?' @@ -897,6 +1104,18 @@ def get_colour(self): If a new color is selected while using a builtin theme, a name must be supplied to create a custom theme. + + Attributes accessed: + highlight_target + frame_colour_set + is_builtin_theme + + Attributes updated: + colour + + Methods: + get_new_theme_name + create_new_theme """ target = self.highlight_target.get() prev_colour = self.frame_colour_set.cget('bg') @@ -937,7 +1156,12 @@ def get_new_theme_name(self, message): return new_theme def save_as_new_theme(self): - "Prompt for new theme name and create the theme." + """Prompt for new theme name and create the theme. + + Methods: + get_new_theme_name + create_new_theme + """ new_theme_name = self.get_new_theme_name('New Theme Name:') if new_theme_name: self.create_new_theme(new_theme_name) @@ -948,6 +1172,18 @@ def create_new_theme(self, new_theme_name): Create the new theme based on the previously active theme with the current changes applied. Once it is saved, then activate the new theme. + + Attributes accessed: + builtin_theme + custom_theme + + Attributes updated: + opt_menu_theme_custom + is_builtin_theme + + Method: + save_new_theme + set_theme_type """ if self.is_builtin_theme.get(): theme_type = 'default' @@ -975,6 +1211,12 @@ def on_fontlist_select(self, event): Event can result from either mouse click or Up or Down key. Set font_name and example display to selection. + + Attributes updated: + font_name: Set to name selected from fontlist. + + Methods: + set_font_sample """ font = self.fontlist.get( ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) @@ -982,7 +1224,22 @@ def on_fontlist_select(self, event): self.set_font_sample() def set_font_sample(self, event=None): - "Update the screen samples with the font settings from the dialog." + """Update the screen samples with the font settings from the dialog. + + Attributes accessed: + font_name + font_bold + font_size + + Attributes updated: + font_sample: Set to selected font name, size, and weight. + text_highlight_sample: Set to selected font name, size, and weight. + + Called from: + handler for opt_menu_font_size and check_font_bold + on_fontlist_select + load_font_cfg + """ font_name = self.font_name.get() font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) @@ -990,7 +1247,23 @@ def set_font_sample(self, event=None): self.text_highlight_sample.configure(font=new_font) def set_highlight_target(self): - "Set fg/bg toggle and color based on highlight tag target." + """Set fg/bg toggle and color based on highlight tag target. + + Instance variables accessed: + highlight_target + + Attributes updated: + radio_fg + radio_bg + fg_bg_toggle + + Methods: + set_colour_sample + + Called from: + var_changed_highlight_target + load_theme_cfg + """ if self.highlight_target.get() == 'Cursor': # bg not possible self.radio_fg.config(state=DISABLED) self.radio_bg.config(state=DISABLED) @@ -1002,11 +1275,25 @@ def set_highlight_target(self): self.set_colour_sample() def set_colour_sample_binding(self, *args): - "Change color sample based on foreground/background toggle." + """Change color sample based on foreground/background toggle. + + Methods: + set_colour_sample + """ self.set_colour_sample() def set_colour_sample(self): - "Set the color of the frame background to reflect the selected target." + """Set the color of the frame background to reflect the selected target. + + Instance variables accessed: + theme_elements + highlight_target + fg_bg_toggle + text_highlight_sample + + Attributes updated: + frame_colour_set + """ # Set the colour sample area. tag = self.theme_elements[self.highlight_target.get()][0] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' @@ -1014,7 +1301,25 @@ def set_colour_sample(self): self.frame_colour_set.config(bg=colour) def paint_theme_sample(self): - "Apply the theme colors to each element tag in the sample text." + """Apply the theme colors to each element tag in the sample text. + + Instance attributes accessed: + theme_elements + is_builtin_theme + builtin_theme + custom_theme + + Attributes updated: + text_highlight_sample: Set the tag elements to the theme. + + Methods: + set_colour_sample + + Called from: + var_changed_builtin_theme + var_changed_custom_theme + load_theme_cfg + """ if self.is_builtin_theme.get(): # Default theme theme = self.builtin_theme.get() else: # User theme @@ -1106,7 +1411,21 @@ def update_user_help_changed_items(self): ';'.join(self.user_helplist[num-1][:2])) def load_font_cfg(self): - "Load current configuration settings for the font options." + """Load current configuration settings for the font options. + + Retrieve current font values from idleConf.GetFont to set + as initial values for font widgets. + + Attributes updated: + fontlist: Populate with fonts from tkinter.font. + font_name: Set to current font. + opt_menu_font_size: Populate valid options tuple and set + to current size. + font_bold: Set to current font weight. + + Methods: + set_font_sample + """ # Set base editor font selection list. fonts = list(tkFont.families(self)) fonts.sort() @@ -1136,14 +1455,36 @@ def load_font_cfg(self): self.set_font_sample() def load_tab_cfg(self): - "Load current configuration settings for the tab options." + """Load current configuration settings for the tab options. + + Attributes updated: + space_num: Set to value from idleConf. + """ # Set indent sizes. space_num = idleConf.GetOption( 'main', 'Indent', 'num-spaces', default=4, type='int') self.space_num.set(space_num) def load_theme_cfg(self): - "Load current configuration settings for the theme options." + """Load current configuration settings for the theme options. + + Based on the is_builtin_theme toggle, the theme is set as + either builtin or custom and the initial widget values + reflect the current settings from idleConf. + + Attributes updated: + is_builtin_theme: Set from idleConf. + opt_menu_theme_builtin: List of default themes from idleConf. + opt_menu_theme_custom: List of custom themes from idleConf. + radio_theme_custom: Disabled if there are no custom themes. + custom_theme: Message with additional information. + opt_menu_highlight_target: Create menu from self.theme_elements. + + Methods: + set_theme_type + paint_theme_sample + set_highlight_target + """ # Set current theme type radiobutton. self.is_builtin_theme.set(idleConf.GetOption( 'main', 'Theme', 'default', type='bool', default=1)) @@ -1231,6 +1572,13 @@ def load_configs(self): Load configuration from default and user config files and populate the widgets on the config dialog pages. + + Methods: + load_font_cfg + load_tab_cfg + load_theme_cfg + load_key_cfg + load_general_cfg """ self.load_font_cfg() self.load_tab_cfg() @@ -1252,7 +1600,7 @@ def save_new_key_set(self, keyset_name, keyset): idleConf.userCfg['keys'].SetOption(keyset_name, event, value) def save_new_theme(self, theme_name, theme): - """Save a newly created theme. + """Save a newly created theme to idleConf. theme_name - string, the name of the new theme theme - dictionary containing the new theme @@ -1264,7 +1612,11 @@ def save_new_theme(self, theme_name, theme): idleConf.userCfg['highlight'].SetOption(theme_name, element, value) def deactivate_current_config(self): - "Remove current key bindings." + """Remove current key bindings. + + Iterate over window instances defined in parent and remove + the keybindings. + """ # Before a config is saved, some cleanup of current # config must be done - remove the previous keybindings. win_instances = self.parent.instance_dict.keys() @@ -1272,7 +1624,11 @@ def deactivate_current_config(self): instance.RemoveKeybindings() def activate_config_changes(self): - "Dynamically apply configuration changes" + """Apply configuration changes to current windows. + + Dynamically update the current parent window instances + with some of the configuration changes. + """ win_instances = self.parent.instance_dict.keys() for instance in win_instances: instance.ResetColorizer() @@ -1282,23 +1638,45 @@ def activate_config_changes(self): instance.reset_help_menu_entries() def cancel(self): - "Dismiss config dialog." + """Dismiss config dialog. + + Methods: + destroy: inherited + """ self.destroy() def ok(self): - "Apply config changes, then dismiss dialog." + """Apply config changes, then dismiss dialog. + + Methods: + apply + destroy: inherited + """ self.apply() self.destroy() def apply(self): - "Apply config changes and leave dialog open." + """Apply config changes and leave dialog open. + + Methods: + deactivate_current_config + save_all_changed_extensions + activate_config_changes + """ self.deactivate_current_config() changes.save_all() self.save_all_changed_extensions() self.activate_config_changes() def help(self): - "Create textview for config dialog help." + """Create textview for config dialog help. + + Attrbutes accessed: + tab_pages + + Methods: + view_text: Method from textview module. + """ page = self.tab_pages._current_page view_text(self, title='Help for IDLE preferences', text=help_common+help_pages.get(page, '')) @@ -1319,6 +1697,13 @@ def create_page_extensions(self): All values are treated as text, and it is up to the user to supply reasonable values. The only exception to this are the 'enable*' options, which are boolean, and can be toggled with a True/False button. + + Methods: + load_extentions: + extension_selected: Handle selection from list. + create_extension_frame: Hold widgets for one extension. + set_extension_value: Set in userCfg['extensions']. + save_all_changed_extensions: Call extension page Save(). """ parent = self.parent frame = self.tab_pages.pages['Extensions'].frame @@ -1460,7 +1845,14 @@ def set_extension_value(self, section, opt): return self.ext_userCfg.SetOption(section, name, value) def save_all_changed_extensions(self): - """Save configuration changes to the user config file.""" + """Save configuration changes to the user config file. + + Attributes accessed: + extensions + + Methods: + set_extension_value + """ has_changes = False for ext_name in self.extensions: options = self.extensions[ext_name] diff --git a/Misc/NEWS.d/next/IDLE/2017-07-13-23-07-33.bpo-30913.aezn_e.rst b/Misc/NEWS.d/next/IDLE/2017-07-13-23-07-33.bpo-30913.aezn_e.rst new file mode 100644 index 00000000000000..23a9fe86ccbd32 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-13-23-07-33.bpo-30913.aezn_e.rst @@ -0,0 +1,3 @@ +IDLE: Document ConfigDialog tk Vars, methods, and widgets in docstrings This +will facilitate improving the dialog and splitting up the class. Original +patch by Cheryl Sabella. From 1aafd9c8bd10b44f62d05428e0e99ed50b5f5d39 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Fri, 14 Jul 2017 00:29:05 -0400 Subject: [PATCH 0564/1345] [3.6] Update idlelib/NEWS.txt. (GH-2703) (#2704) (cherry picked from commit 0d0a32f) --- Lib/idlelib/NEWS.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index af88048fa0ca2e..3c5cba6af477fe 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,37 @@ Released on 2017-09-25? ======================== +bpo-30913: Document ConfigDialog tk Vars, methods, and widgets in docstrings +This will facilitate improving the dialog and splitting up the class. +Original patch by Cheryl Sabella. + +bpo-30899: Add tests for ConfigParser subclasses in config. +Coverage is 100% for those classes and ConfigChanges. +Patch by Louie Lu. + +bpo-30881: Add docstrings to browser.py. +Patch by Cheryl Sabella. + +bpo-30851: Remove unused tk variables in configdialog. +One is a duplicate, one is set but cannot be altered by users. +Patch by Cheryl Sabella. + +bpo-30870: Select font option with Up and Down keys, as well as with mouse. +Added test increases configdialog coverage to 60% +Patches mostly by Louie Lu. + +bpo-8231: Call config.IdleConf.GetUserCfgDir only once per process. + +bpo-30779: Factor ConfigChanges class from configdialog, put in config; test. +* In config, put dump test code in a function; run it and unittest in + 'if __name__ == '__main__'. +* Add class config.ConfigChanges based on changes_class_v4.py on bpo issue. +* Add class test_config.ChangesTest, partly using configdialog_tests_v1.py. +* Revise configdialog to use ConfigChanges; see tracker msg297804. +* Revise test_configdialog to match configdialog changes. +* Remove configdialog functions unused or moved to ConfigChanges. +Cheryl Sabella contributed parts of the patch. + bpo-30777: Configdialog - add docstrings and improve comments. Patch by Cheryl Sabella. From ea9cbe0a5cc4d35643939c6ddee7fc96dbfff211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Rogalski?= Date: Fri, 14 Jul 2017 21:57:19 +0200 Subject: [PATCH 0565/1345] Doc that 'sorted' args are keyword-only, fix 'reverse' default (#2709) backport --- Doc/library/functions.rst | 2 +- Doc/library/stdtypes.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 6621f4aabf7058..916e3528d79dac 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1319,7 +1319,7 @@ are always available. They are listed here in alphabetical order. :func:`itertools.islice` for an alternate version that returns an iterator. -.. function:: sorted(iterable[, key][, reverse]) +.. function:: sorted(iterable, *, key=None, reverse=False) Return a new sorted list from the items in *iterable*. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 5ab8177490d6ce..e929f0b68767d4 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1161,7 +1161,7 @@ application). :ref:`mutable ` sequence operations. Lists also provide the following additional method: - .. method:: list.sort(*, key=None, reverse=None) + .. method:: list.sort(*, key=None, reverse=False) This method sorts the list in place, using only ``<`` comparisons between items. Exceptions are not suppressed - if any comparison operations From bbd0f423d2a19c6a4a9cda6914fc60b87e17250b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 15 Jul 2017 16:56:13 -0400 Subject: [PATCH 0566/1345] bpo-30939: Avoid Sphinx deprecation warning in docs build. (#2721) (#2722) (cherry picked from commit 50f58163a69abe2f35e91044d1df165ee7bdbb42) --- Doc/tools/extensions/pyspecific.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 865f2edee02205..8f507e461504bd 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -15,6 +15,7 @@ from time import asctime from pprint import pformat from docutils.io import StringOutput +from docutils.parsers.rst import Directive from docutils.utils import new_document from docutils import nodes, utils @@ -23,7 +24,6 @@ from sphinx.builders import Builder from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title -from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter from sphinx.writers.latex import LaTeXTranslator From 65de1f3672fd0a1e3ec34b654ef5213e15a03e26 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Sun, 16 Jul 2017 00:39:59 -0400 Subject: [PATCH 0567/1345] [3.6] bpo-30934: Document coverage details for idlelib tests (GH-2711) (#2726) * Add section to idlelib/idle-test/README.txt. * Include check that branches are taken both ways. * Exclude IDLE-specific code that does not run during unit tests. (cherry picked from commit 95bebb7) --- Lib/idlelib/idle_test/README.txt | 56 +++++++++++++++++++ .../2017-07-15-22-26-57.bpo-30934.BanuSB.rst | 7 +++ 2 files changed, 63 insertions(+) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-15-22-26-57.bpo-30934.BanuSB.rst diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index dc7a28697c1988..a54e74ddda2170 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -159,3 +159,59 @@ complete, though some tests need improvement. To run all htests, run the htest file from an editor or from the command line with: python -m idlelib.idle_test.htest + + +5. Test Coverage + +Install the coverage package into your Python 3.6 site-packages +directory. (Its exact location depends on the OS). +> python3 -m pip install coverage +(On Windows, replace 'python3 with 'py -3.6' or perhaps just 'python'.) + +The problem with running coverage with repository python is that +coverage uses absolute imports for its submodules, hence it needs to be +in a directory in sys.path. One solution: copy the package to the +directory containing the cpython repository. Call it 'dev'. Then run +coverage either directly or from a script in that directory so that +'dev' is prepended to sys.path. + +Either edit or add dev/.coveragerc so it looks something like this. +--- +# .coveragerc sets coverage options. +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + .*# htest # + if not _utest: + if _htest: +--- +The additions for IDLE are 'branch = True', to test coverage both ways, +and the last three exclude lines, to exclude things peculiar to IDLE +that are not executed during tests. + +A script like the following cover.bat (for Windows) is very handy. +--- +@echo off +rem Usage: cover filename [test_ suffix] # proper case required by coverage +rem filename without .py, 2nd parameter if test is not test_filename +setlocal +set py=f:\dev\3x\pcbuild\win32\python_d.exe +set src=idlelib.%1 +if "%2" EQU "" set tst=f:/dev/3x/Lib/idlelib/idle_test/test_%1.py +if "%2" NEQ "" set tst=f:/dev/ex/Lib/idlelib/idle_test/test_%2.py + +%py% -m coverage run --pylib --source=%src% %tst% +%py% -m coverage report --show-missing +%py% -m coverage html +start htmlcov\3x_Lib_idlelib_%1_py.html +rem Above opens new report; htmlcov\index.html displays report index +--- +The second parameter was added for tests of module x not named test_x. +(There were several before modules were renamed, now only one is left.) diff --git a/Misc/NEWS.d/next/IDLE/2017-07-15-22-26-57.bpo-30934.BanuSB.rst b/Misc/NEWS.d/next/IDLE/2017-07-15-22-26-57.bpo-30934.BanuSB.rst new file mode 100644 index 00000000000000..53beb43159c45e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-15-22-26-57.bpo-30934.BanuSB.rst @@ -0,0 +1,7 @@ +Document coverage details for idlelib tests. + +* Add section to idlelib/idle-test/README.txt. + +* Include check that branches are taken both ways. + +* Exclude IDLE-specific code that does not run during unit tests. From 28343e3392ca7b1ec7151f68d4d92c90efb91e50 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 16 Jul 2017 07:44:25 +0300 Subject: [PATCH 0568/1345] [3.6] bpo-30876: Relative import from unloaded package now reimports the package (GH-2639) (#2676) instead of failing with SystemError. Relative import from non-package now fails with ImportError rather than SystemError. (cherry picked from commit 8a9cd20edca7d01b68292036029ae3735ce65edd) --- Lib/importlib/_bootstrap.py | 4 - Lib/test/test_import/__init__.py | 21 +- .../test_import/data/package2/submodule1.py | 3 + .../test_import/data/package2/submodule2.py | 0 .../import_/test___package__.py | 2 +- .../2017-07-11-06-31-32.bpo-30876.x35jZX.rst | 3 + Python/import.c | 7 - Python/importlib.h | 720 +++++++++--------- 8 files changed, 381 insertions(+), 379 deletions(-) create mode 100644 Lib/test/test_import/data/package2/submodule1.py create mode 100644 Lib/test/test_import/data/package2/submodule2.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-07-11-06-31-32.bpo-30876.x35jZX.rst diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index f7790c1db3a0af..bf13e2aed017b7 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -919,10 +919,6 @@ def _sanity_check(name, package, level): elif not package: raise ImportError('attempted relative import with no known parent ' 'package') - elif package not in sys.modules: - msg = ('Parent module {!r} not loaded, cannot perform relative ' - 'import') - raise SystemError(msg.format(package)) if not name and level == 0: raise ValueError('Empty module name') diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3291d0896cf912..e8111214b92b61 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -24,8 +24,9 @@ EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, unlink, unload, create_empty_file, cpython_only, TESTFN_UNENCODABLE, - temp_dir) + temp_dir, DirsOnSysPath) from test.support import script_helper +from test.test_importlib.util import uncache skip_if_dont_write_bytecode = unittest.skipIf( @@ -640,11 +641,11 @@ def check_relative(): # Check relative import fails with only __package__ wrong ns = dict(__package__='foo', __name__='test.notarealmodule') - self.assertRaises(SystemError, check_relative) + self.assertRaises(ModuleNotFoundError, check_relative) # Check relative import fails with __package__ and __name__ wrong ns = dict(__package__='foo', __name__='notarealpkg.notarealmodule') - self.assertRaises(SystemError, check_relative) + self.assertRaises(ModuleNotFoundError, check_relative) # Check relative import fails with package set to a non-string ns = dict(__package__=object()) @@ -659,6 +660,20 @@ def test_absolute_import_without_future(self): self.fail("explicit relative import triggered an " "implicit absolute import") + def test_import_from_non_package(self): + path = os.path.join(os.path.dirname(__file__), 'data', 'package2') + with uncache('submodule1', 'submodule2'), DirsOnSysPath(path): + with self.assertRaises(ImportError): + import submodule1 + self.assertNotIn('submodule1', sys.modules) + self.assertNotIn('submodule2', sys.modules) + + def test_import_from_unloaded_package(self): + with uncache('package2', 'package2.submodule1', 'package2.submodule2'), \ + DirsOnSysPath(os.path.join(os.path.dirname(__file__), 'data')): + import package2.submodule1 + package2.submodule1.submodule2 + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): diff --git a/Lib/test/test_import/data/package2/submodule1.py b/Lib/test/test_import/data/package2/submodule1.py new file mode 100644 index 00000000000000..0698ed6de2a1db --- /dev/null +++ b/Lib/test/test_import/data/package2/submodule1.py @@ -0,0 +1,3 @@ +import sys +sys.modules.pop(__package__, None) +from . import submodule2 diff --git a/Lib/test/test_import/data/package2/submodule2.py b/Lib/test/test_import/data/package2/submodule2.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 7f64548748eec3..b26634ea4537a1 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -81,7 +81,7 @@ def test_warn_when_package_and_spec_disagree(self): def test_bad__package__(self): globals = {'__package__': ''} - with self.assertRaises(SystemError): + with self.assertRaises(ModuleNotFoundError): self.__import__('', globals, {}, ['relimport'], 1) def test_bunk__package__(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-11-06-31-32.bpo-30876.x35jZX.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-11-06-31-32.bpo-30876.x35jZX.rst new file mode 100644 index 00000000000000..4f3f4d29190728 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-11-06-31-32.bpo-30876.x35jZX.rst @@ -0,0 +1,3 @@ +Relative import from unloaded package now reimports the package instead of +failing with SystemError. Relative import from non-package now fails with +ImportError rather than SystemError. diff --git a/Python/import.c b/Python/import.c index 40e02c01ec95d4..4b3bedf87684b6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1345,7 +1345,6 @@ resolve_name(PyObject *name, PyObject *globals, int level) PyObject *abs_name; PyObject *package = NULL; PyObject *spec; - PyInterpreterState *interp = PyThreadState_GET()->interp; Py_ssize_t last_dot; PyObject *base; int level_up; @@ -1449,12 +1448,6 @@ resolve_name(PyObject *name, PyObject *globals, int level) "attempted relative import with no known parent package"); goto error; } - else if (PyDict_GetItem(interp->modules, package) == NULL) { - PyErr_Format(PyExc_SystemError, - "Parent module %R not loaded, cannot perform relative " - "import", package); - goto error; - } for (level_up = 1; level_up < level; level_up += 1) { last_dot = PyUnicode_FindChar(package, '.', 0, last_dot, -1); diff --git a/Python/importlib.h b/Python/importlib.h index e44285036ea5b0..30c554241fe1d2 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1442,371 +1442,363 @@ const unsigned char _Py_M__importlib[] = { 12,5,10,1,10,1,8,1,2,1,10,1,14,1,12,1, 8,1,8,2,22,1,8,2,16,1,10,1,2,1,10,1, 14,4,6,2,8,1,4,2,6,2,8,2,114,170,0,0, - 0,99,3,0,0,0,0,0,0,0,4,0,0,0,4,0, - 0,0,67,0,0,0,115,140,0,0,0,116,0,124,0,116, + 0,99,3,0,0,0,0,0,0,0,3,0,0,0,4,0, + 0,0,67,0,0,0,115,110,0,0,0,116,0,124,0,116, 1,131,2,115,28,116,2,100,1,106,3,116,4,124,0,131, 1,131,1,131,1,130,1,124,2,100,2,107,0,114,44,116, - 5,100,3,131,1,130,1,124,2,100,2,107,4,114,114,116, + 5,100,3,131,1,130,1,124,2,100,2,107,4,114,84,116, 0,124,1,116,1,131,2,115,72,116,2,100,4,131,1,130, - 1,110,42,124,1,115,86,116,6,100,5,131,1,130,1,110, - 28,124,1,116,7,106,8,107,7,114,114,100,6,125,3,116, - 9,124,3,106,3,124,1,131,1,131,1,130,1,124,0,12, - 0,114,136,124,2,100,2,107,2,114,136,116,5,100,7,131, - 1,130,1,100,8,83,0,41,9,122,28,86,101,114,105,102, - 121,32,97,114,103,117,109,101,110,116,115,32,97,114,101,32, - 34,115,97,110,101,34,46,122,31,109,111,100,117,108,101,32, - 110,97,109,101,32,109,117,115,116,32,98,101,32,115,116,114, - 44,32,110,111,116,32,123,125,114,19,0,0,0,122,18,108, - 101,118,101,108,32,109,117,115,116,32,98,101,32,62,61,32, - 48,122,31,95,95,112,97,99,107,97,103,101,95,95,32,110, - 111,116,32,115,101,116,32,116,111,32,97,32,115,116,114,105, - 110,103,122,54,97,116,116,101,109,112,116,101,100,32,114,101, - 108,97,116,105,118,101,32,105,109,112,111,114,116,32,119,105, - 116,104,32,110,111,32,107,110,111,119,110,32,112,97,114,101, - 110,116,32,112,97,99,107,97,103,101,122,61,80,97,114,101, - 110,116,32,109,111,100,117,108,101,32,123,33,114,125,32,110, - 111,116,32,108,111,97,100,101,100,44,32,99,97,110,110,111, - 116,32,112,101,114,102,111,114,109,32,114,101,108,97,116,105, - 118,101,32,105,109,112,111,114,116,122,17,69,109,112,116,121, - 32,109,111,100,117,108,101,32,110,97,109,101,78,41,10,218, - 10,105,115,105,110,115,116,97,110,99,101,218,3,115,116,114, - 218,9,84,121,112,101,69,114,114,111,114,114,38,0,0,0, - 114,13,0,0,0,114,160,0,0,0,114,71,0,0,0,114, - 14,0,0,0,114,80,0,0,0,218,11,83,121,115,116,101, - 109,69,114,114,111,114,41,4,114,15,0,0,0,114,161,0, - 0,0,114,162,0,0,0,114,139,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,13,95,115,97, - 110,105,116,121,95,99,104,101,99,107,142,3,0,0,115,28, - 0,0,0,0,2,10,1,18,1,8,1,8,1,8,1,10, - 1,10,1,4,1,10,2,10,1,4,2,14,1,14,1,114, - 175,0,0,0,122,16,78,111,32,109,111,100,117,108,101,32, - 110,97,109,101,100,32,122,4,123,33,114,125,99,2,0,0, - 0,0,0,0,0,8,0,0,0,12,0,0,0,67,0,0, - 0,115,220,0,0,0,100,0,125,2,124,0,106,0,100,1, - 131,1,100,2,25,0,125,3,124,3,114,134,124,3,116,1, - 106,2,107,7,114,42,116,3,124,1,124,3,131,2,1,0, - 124,0,116,1,106,2,107,6,114,62,116,1,106,2,124,0, - 25,0,83,0,116,1,106,2,124,3,25,0,125,4,121,10, - 124,4,106,4,125,2,87,0,110,50,4,0,116,5,107,10, - 114,132,1,0,1,0,1,0,116,6,100,3,23,0,106,7, - 124,0,124,3,131,2,125,5,116,8,124,5,124,0,100,4, - 141,2,100,0,130,2,89,0,110,2,88,0,116,9,124,0, - 124,2,131,2,125,6,124,6,100,0,107,8,114,172,116,8, - 116,6,106,7,124,0,131,1,124,0,100,4,141,2,130,1, - 110,8,116,10,124,6,131,1,125,7,124,3,114,216,116,1, - 106,2,124,3,25,0,125,4,116,11,124,4,124,0,106,0, - 100,1,131,1,100,5,25,0,124,7,131,3,1,0,124,7, - 83,0,41,6,78,114,118,0,0,0,114,19,0,0,0,122, - 23,59,32,123,33,114,125,32,105,115,32,110,111,116,32,97, - 32,112,97,99,107,97,103,101,41,1,114,15,0,0,0,233, - 2,0,0,0,41,12,114,119,0,0,0,114,14,0,0,0, - 114,80,0,0,0,114,59,0,0,0,114,128,0,0,0,114, - 91,0,0,0,218,8,95,69,82,82,95,77,83,71,114,38, - 0,0,0,218,19,77,111,100,117,108,101,78,111,116,70,111, - 117,110,100,69,114,114,111,114,114,170,0,0,0,114,141,0, - 0,0,114,5,0,0,0,41,8,114,15,0,0,0,218,7, - 105,109,112,111,114,116,95,114,144,0,0,0,114,120,0,0, - 0,90,13,112,97,114,101,110,116,95,109,111,100,117,108,101, - 114,139,0,0,0,114,83,0,0,0,114,84,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,23, - 95,102,105,110,100,95,97,110,100,95,108,111,97,100,95,117, - 110,108,111,99,107,101,100,165,3,0,0,115,42,0,0,0, - 0,1,4,1,14,1,4,1,10,1,10,2,10,1,10,1, - 10,1,2,1,10,1,14,1,16,1,20,1,10,1,8,1, - 20,2,8,1,4,2,10,1,22,1,114,180,0,0,0,99, - 2,0,0,0,0,0,0,0,4,0,0,0,11,0,0,0, - 67,0,0,0,115,92,0,0,0,116,0,124,0,131,1,143, - 26,1,0,124,0,116,1,106,2,107,7,114,30,116,3,124, - 0,124,1,131,2,83,0,87,0,100,1,81,0,82,0,88, - 0,116,1,106,2,124,0,25,0,125,2,124,2,100,1,107, - 8,114,80,100,2,106,4,124,0,131,1,125,3,116,5,124, - 3,124,0,100,3,141,2,130,1,116,6,124,0,131,1,1, - 0,124,2,83,0,41,4,122,25,70,105,110,100,32,97,110, - 100,32,108,111,97,100,32,116,104,101,32,109,111,100,117,108, - 101,46,78,122,40,105,109,112,111,114,116,32,111,102,32,123, - 125,32,104,97,108,116,101,100,59,32,78,111,110,101,32,105, - 110,32,115,121,115,46,109,111,100,117,108,101,115,41,1,114, - 15,0,0,0,41,7,114,42,0,0,0,114,14,0,0,0, - 114,80,0,0,0,114,180,0,0,0,114,38,0,0,0,114, - 178,0,0,0,114,57,0,0,0,41,4,114,15,0,0,0, - 114,179,0,0,0,114,84,0,0,0,114,68,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,14, - 95,102,105,110,100,95,97,110,100,95,108,111,97,100,192,3, - 0,0,115,20,0,0,0,0,2,10,1,10,1,20,2,10, - 1,8,1,4,1,6,1,12,2,8,1,114,181,0,0,0, - 114,19,0,0,0,99,3,0,0,0,0,0,0,0,3,0, - 0,0,4,0,0,0,67,0,0,0,115,42,0,0,0,116, - 0,124,0,124,1,124,2,131,3,1,0,124,2,100,1,107, - 4,114,32,116,1,124,0,124,1,124,2,131,3,125,0,116, - 2,124,0,116,3,131,2,83,0,41,2,97,50,1,0,0, - 73,109,112,111,114,116,32,97,110,100,32,114,101,116,117,114, - 110,32,116,104,101,32,109,111,100,117,108,101,32,98,97,115, - 101,100,32,111,110,32,105,116,115,32,110,97,109,101,44,32, - 116,104,101,32,112,97,99,107,97,103,101,32,116,104,101,32, - 99,97,108,108,32,105,115,10,32,32,32,32,98,101,105,110, - 103,32,109,97,100,101,32,102,114,111,109,44,32,97,110,100, - 32,116,104,101,32,108,101,118,101,108,32,97,100,106,117,115, - 116,109,101,110,116,46,10,10,32,32,32,32,84,104,105,115, - 32,102,117,110,99,116,105,111,110,32,114,101,112,114,101,115, - 101,110,116,115,32,116,104,101,32,103,114,101,97,116,101,115, - 116,32,99,111,109,109,111,110,32,100,101,110,111,109,105,110, - 97,116,111,114,32,111,102,32,102,117,110,99,116,105,111,110, - 97,108,105,116,121,10,32,32,32,32,98,101,116,119,101,101, - 110,32,105,109,112,111,114,116,95,109,111,100,117,108,101,32, - 97,110,100,32,95,95,105,109,112,111,114,116,95,95,46,32, - 84,104,105,115,32,105,110,99,108,117,100,101,115,32,115,101, - 116,116,105,110,103,32,95,95,112,97,99,107,97,103,101,95, - 95,32,105,102,10,32,32,32,32,116,104,101,32,108,111,97, - 100,101,114,32,100,105,100,32,110,111,116,46,10,10,32,32, - 32,32,114,19,0,0,0,41,4,114,175,0,0,0,114,163, - 0,0,0,114,181,0,0,0,218,11,95,103,99,100,95,105, - 109,112,111,114,116,41,3,114,15,0,0,0,114,161,0,0, - 0,114,162,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,182,0,0,0,208,3,0,0,115,8, - 0,0,0,0,9,12,1,8,1,12,1,114,182,0,0,0, - 99,3,0,0,0,0,0,0,0,6,0,0,0,17,0,0, - 0,67,0,0,0,115,164,0,0,0,116,0,124,0,100,1, - 131,2,114,160,100,2,124,1,107,6,114,58,116,1,124,1, - 131,1,125,1,124,1,106,2,100,2,131,1,1,0,116,0, - 124,0,100,3,131,2,114,58,124,1,106,3,124,0,106,4, - 131,1,1,0,120,100,124,1,68,0,93,92,125,3,116,0, - 124,0,124,3,131,2,115,64,100,4,106,5,124,0,106,6, - 124,3,131,2,125,4,121,14,116,7,124,2,124,4,131,2, - 1,0,87,0,113,64,4,0,116,8,107,10,114,154,1,0, - 125,5,1,0,122,20,124,5,106,9,124,4,107,2,114,136, - 119,64,130,0,87,0,89,0,100,5,100,5,125,5,126,5, - 88,0,113,64,88,0,113,64,87,0,124,0,83,0,41,6, - 122,238,70,105,103,117,114,101,32,111,117,116,32,119,104,97, - 116,32,95,95,105,109,112,111,114,116,95,95,32,115,104,111, - 117,108,100,32,114,101,116,117,114,110,46,10,10,32,32,32, - 32,84,104,101,32,105,109,112,111,114,116,95,32,112,97,114, - 97,109,101,116,101,114,32,105,115,32,97,32,99,97,108,108, - 97,98,108,101,32,119,104,105,99,104,32,116,97,107,101,115, - 32,116,104,101,32,110,97,109,101,32,111,102,32,109,111,100, - 117,108,101,32,116,111,10,32,32,32,32,105,109,112,111,114, - 116,46,32,73,116,32,105,115,32,114,101,113,117,105,114,101, - 100,32,116,111,32,100,101,99,111,117,112,108,101,32,116,104, - 101,32,102,117,110,99,116,105,111,110,32,102,114,111,109,32, - 97,115,115,117,109,105,110,103,32,105,109,112,111,114,116,108, - 105,98,39,115,10,32,32,32,32,105,109,112,111,114,116,32, - 105,109,112,108,101,109,101,110,116,97,116,105,111,110,32,105, - 115,32,100,101,115,105,114,101,100,46,10,10,32,32,32,32, - 114,128,0,0,0,250,1,42,218,7,95,95,97,108,108,95, - 95,122,5,123,125,46,123,125,78,41,10,114,4,0,0,0, - 114,127,0,0,0,218,6,114,101,109,111,118,101,218,6,101, - 120,116,101,110,100,114,184,0,0,0,114,38,0,0,0,114, - 1,0,0,0,114,59,0,0,0,114,178,0,0,0,114,15, - 0,0,0,41,6,114,84,0,0,0,218,8,102,114,111,109, - 108,105,115,116,114,179,0,0,0,218,1,120,90,9,102,114, - 111,109,95,110,97,109,101,90,3,101,120,99,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,16,95,104,97, - 110,100,108,101,95,102,114,111,109,108,105,115,116,223,3,0, - 0,115,32,0,0,0,0,10,10,1,8,1,8,1,10,1, - 10,1,12,1,10,1,10,1,14,1,2,1,14,1,16,4, - 10,1,2,1,24,1,114,189,0,0,0,99,1,0,0,0, - 0,0,0,0,3,0,0,0,6,0,0,0,67,0,0,0, - 115,146,0,0,0,124,0,106,0,100,1,131,1,125,1,124, - 0,106,0,100,2,131,1,125,2,124,1,100,3,107,9,114, - 82,124,2,100,3,107,9,114,78,124,1,124,2,106,1,107, - 3,114,78,116,2,106,3,100,4,124,1,155,2,100,5,124, - 2,106,1,155,2,100,6,157,5,116,4,100,7,100,8,141, - 3,1,0,124,1,83,0,124,2,100,3,107,9,114,96,124, - 2,106,1,83,0,116,2,106,3,100,9,116,4,100,7,100, - 8,141,3,1,0,124,0,100,10,25,0,125,1,100,11,124, - 0,107,7,114,142,124,1,106,5,100,12,131,1,100,13,25, - 0,125,1,124,1,83,0,41,14,122,167,67,97,108,99,117, - 108,97,116,101,32,119,104,97,116,32,95,95,112,97,99,107, - 97,103,101,95,95,32,115,104,111,117,108,100,32,98,101,46, - 10,10,32,32,32,32,95,95,112,97,99,107,97,103,101,95, - 95,32,105,115,32,110,111,116,32,103,117,97,114,97,110,116, - 101,101,100,32,116,111,32,98,101,32,100,101,102,105,110,101, - 100,32,111,114,32,99,111,117,108,100,32,98,101,32,115,101, - 116,32,116,111,32,78,111,110,101,10,32,32,32,32,116,111, - 32,114,101,112,114,101,115,101,110,116,32,116,104,97,116,32, - 105,116,115,32,112,114,111,112,101,114,32,118,97,108,117,101, - 32,105,115,32,117,110,107,110,111,119,110,46,10,10,32,32, - 32,32,114,131,0,0,0,114,90,0,0,0,78,122,32,95, - 95,112,97,99,107,97,103,101,95,95,32,33,61,32,95,95, - 115,112,101,99,95,95,46,112,97,114,101,110,116,32,40,122, - 4,32,33,61,32,250,1,41,233,3,0,0,0,41,1,90, - 10,115,116,97,99,107,108,101,118,101,108,122,89,99,97,110, - 39,116,32,114,101,115,111,108,118,101,32,112,97,99,107,97, - 103,101,32,102,114,111,109,32,95,95,115,112,101,99,95,95, - 32,111,114,32,95,95,112,97,99,107,97,103,101,95,95,44, - 32,102,97,108,108,105,110,103,32,98,97,99,107,32,111,110, - 32,95,95,110,97,109,101,95,95,32,97,110,100,32,95,95, - 112,97,116,104,95,95,114,1,0,0,0,114,128,0,0,0, - 114,118,0,0,0,114,19,0,0,0,41,6,114,30,0,0, - 0,114,120,0,0,0,114,167,0,0,0,114,168,0,0,0, - 114,169,0,0,0,114,119,0,0,0,41,3,218,7,103,108, - 111,98,97,108,115,114,161,0,0,0,114,83,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,17, - 95,99,97,108,99,95,95,95,112,97,99,107,97,103,101,95, - 95,254,3,0,0,115,30,0,0,0,0,7,10,1,10,1, - 8,1,18,1,22,2,10,1,4,1,8,1,6,2,6,2, - 10,1,8,1,8,1,14,1,114,193,0,0,0,99,5,0, - 0,0,0,0,0,0,9,0,0,0,5,0,0,0,67,0, - 0,0,115,166,0,0,0,124,4,100,1,107,2,114,18,116, - 0,124,0,131,1,125,5,110,36,124,1,100,2,107,9,114, - 30,124,1,110,2,105,0,125,6,116,1,124,6,131,1,125, - 7,116,0,124,0,124,7,124,4,131,3,125,5,124,3,115, - 150,124,4,100,1,107,2,114,84,116,0,124,0,106,2,100, - 3,131,1,100,1,25,0,131,1,83,0,124,0,115,92,124, - 5,83,0,116,3,124,0,131,1,116,3,124,0,106,2,100, - 3,131,1,100,1,25,0,131,1,24,0,125,8,116,4,106, - 5,124,5,106,6,100,2,116,3,124,5,106,6,131,1,124, - 8,24,0,133,2,25,0,25,0,83,0,110,12,116,7,124, - 5,124,3,116,0,131,3,83,0,100,2,83,0,41,4,97, - 215,1,0,0,73,109,112,111,114,116,32,97,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,84,104,101,32,39,103, - 108,111,98,97,108,115,39,32,97,114,103,117,109,101,110,116, - 32,105,115,32,117,115,101,100,32,116,111,32,105,110,102,101, - 114,32,119,104,101,114,101,32,116,104,101,32,105,109,112,111, - 114,116,32,105,115,32,111,99,99,117,114,114,105,110,103,32, - 102,114,111,109,10,32,32,32,32,116,111,32,104,97,110,100, - 108,101,32,114,101,108,97,116,105,118,101,32,105,109,112,111, - 114,116,115,46,32,84,104,101,32,39,108,111,99,97,108,115, - 39,32,97,114,103,117,109,101,110,116,32,105,115,32,105,103, - 110,111,114,101,100,46,32,84,104,101,10,32,32,32,32,39, - 102,114,111,109,108,105,115,116,39,32,97,114,103,117,109,101, - 110,116,32,115,112,101,99,105,102,105,101,115,32,119,104,97, - 116,32,115,104,111,117,108,100,32,101,120,105,115,116,32,97, - 115,32,97,116,116,114,105,98,117,116,101,115,32,111,110,32, - 116,104,101,32,109,111,100,117,108,101,10,32,32,32,32,98, - 101,105,110,103,32,105,109,112,111,114,116,101,100,32,40,101, - 46,103,46,32,96,96,102,114,111,109,32,109,111,100,117,108, - 101,32,105,109,112,111,114,116,32,60,102,114,111,109,108,105, - 115,116,62,96,96,41,46,32,32,84,104,101,32,39,108,101, - 118,101,108,39,10,32,32,32,32,97,114,103,117,109,101,110, - 116,32,114,101,112,114,101,115,101,110,116,115,32,116,104,101, - 32,112,97,99,107,97,103,101,32,108,111,99,97,116,105,111, - 110,32,116,111,32,105,109,112,111,114,116,32,102,114,111,109, - 32,105,110,32,97,32,114,101,108,97,116,105,118,101,10,32, - 32,32,32,105,109,112,111,114,116,32,40,101,46,103,46,32, - 96,96,102,114,111,109,32,46,46,112,107,103,32,105,109,112, - 111,114,116,32,109,111,100,96,96,32,119,111,117,108,100,32, - 104,97,118,101,32,97,32,39,108,101,118,101,108,39,32,111, - 102,32,50,41,46,10,10,32,32,32,32,114,19,0,0,0, - 78,114,118,0,0,0,41,8,114,182,0,0,0,114,193,0, - 0,0,218,9,112,97,114,116,105,116,105,111,110,114,159,0, - 0,0,114,14,0,0,0,114,80,0,0,0,114,1,0,0, - 0,114,189,0,0,0,41,9,114,15,0,0,0,114,192,0, - 0,0,218,6,108,111,99,97,108,115,114,187,0,0,0,114, - 162,0,0,0,114,84,0,0,0,90,8,103,108,111,98,97, - 108,115,95,114,161,0,0,0,90,7,99,117,116,95,111,102, - 102,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,10,95,95,105,109,112,111,114,116,95,95,25,4,0,0, - 115,26,0,0,0,0,11,8,1,10,2,16,1,8,1,12, - 1,4,3,8,1,18,1,4,1,4,4,26,3,32,2,114, - 196,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, - 0,3,0,0,0,67,0,0,0,115,38,0,0,0,116,0, - 106,1,124,0,131,1,125,1,124,1,100,0,107,8,114,30, - 116,2,100,1,124,0,23,0,131,1,130,1,116,3,124,1, - 131,1,83,0,41,2,78,122,25,110,111,32,98,117,105,108, - 116,45,105,110,32,109,111,100,117,108,101,32,110,97,109,101, - 100,32,41,4,114,142,0,0,0,114,146,0,0,0,114,71, - 0,0,0,114,141,0,0,0,41,2,114,15,0,0,0,114, - 83,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,18,95,98,117,105,108,116,105,110,95,102,114, - 111,109,95,110,97,109,101,60,4,0,0,115,8,0,0,0, - 0,1,10,1,8,1,12,1,114,197,0,0,0,99,2,0, - 0,0,0,0,0,0,12,0,0,0,12,0,0,0,67,0, - 0,0,115,244,0,0,0,124,1,97,0,124,0,97,1,116, - 2,116,1,131,1,125,2,120,86,116,1,106,3,106,4,131, - 0,68,0,93,72,92,2,125,3,125,4,116,5,124,4,124, - 2,131,2,114,28,124,3,116,1,106,6,107,6,114,62,116, - 7,125,5,110,18,116,0,106,8,124,3,131,1,114,28,116, - 9,125,5,110,2,113,28,116,10,124,4,124,5,131,2,125, - 6,116,11,124,6,124,4,131,2,1,0,113,28,87,0,116, - 1,106,3,116,12,25,0,125,7,120,54,100,5,68,0,93, - 46,125,8,124,8,116,1,106,3,107,7,114,144,116,13,124, - 8,131,1,125,9,110,10,116,1,106,3,124,8,25,0,125, - 9,116,14,124,7,124,8,124,9,131,3,1,0,113,120,87, - 0,121,12,116,13,100,2,131,1,125,10,87,0,110,24,4, - 0,116,15,107,10,114,206,1,0,1,0,1,0,100,3,125, - 10,89,0,110,2,88,0,116,14,124,7,100,2,124,10,131, - 3,1,0,116,13,100,4,131,1,125,11,116,14,124,7,100, - 4,124,11,131,3,1,0,100,3,83,0,41,6,122,250,83, - 101,116,117,112,32,105,109,112,111,114,116,108,105,98,32,98, - 121,32,105,109,112,111,114,116,105,110,103,32,110,101,101,100, - 101,100,32,98,117,105,108,116,45,105,110,32,109,111,100,117, - 108,101,115,32,97,110,100,32,105,110,106,101,99,116,105,110, - 103,32,116,104,101,109,10,32,32,32,32,105,110,116,111,32, - 116,104,101,32,103,108,111,98,97,108,32,110,97,109,101,115, - 112,97,99,101,46,10,10,32,32,32,32,65,115,32,115,121, - 115,32,105,115,32,110,101,101,100,101,100,32,102,111,114,32, - 115,121,115,46,109,111,100,117,108,101,115,32,97,99,99,101, - 115,115,32,97,110,100,32,95,105,109,112,32,105,115,32,110, - 101,101,100,101,100,32,116,111,32,108,111,97,100,32,98,117, - 105,108,116,45,105,110,10,32,32,32,32,109,111,100,117,108, - 101,115,44,32,116,104,111,115,101,32,116,119,111,32,109,111, - 100,117,108,101,115,32,109,117,115,116,32,98,101,32,101,120, - 112,108,105,99,105,116,108,121,32,112,97,115,115,101,100,32, - 105,110,46,10,10,32,32,32,32,114,167,0,0,0,114,20, - 0,0,0,78,114,55,0,0,0,41,1,114,167,0,0,0, - 41,16,114,52,0,0,0,114,14,0,0,0,114,13,0,0, - 0,114,80,0,0,0,218,5,105,116,101,109,115,114,171,0, - 0,0,114,70,0,0,0,114,142,0,0,0,114,76,0,0, - 0,114,152,0,0,0,114,129,0,0,0,114,134,0,0,0, - 114,1,0,0,0,114,197,0,0,0,114,5,0,0,0,114, - 71,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, - 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, - 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, - 0,114,84,0,0,0,114,94,0,0,0,114,83,0,0,0, - 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, - 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, - 108,116,105,110,95,109,111,100,117,108,101,90,13,116,104,114, - 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, - 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, - 112,67,4,0,0,115,50,0,0,0,0,9,4,1,4,3, - 8,1,20,1,10,1,10,1,6,1,10,1,6,2,2,1, - 10,1,14,3,10,1,10,1,10,1,10,2,10,1,16,3, - 2,1,12,1,14,2,10,1,12,3,8,1,114,201,0,0, - 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, - 0,0,67,0,0,0,115,66,0,0,0,116,0,124,0,124, - 1,131,2,1,0,116,1,106,2,106,3,116,4,131,1,1, - 0,116,1,106,2,106,3,116,5,131,1,1,0,100,1,100, - 2,108,6,125,2,124,2,97,7,124,2,106,8,116,1,106, - 9,116,10,25,0,131,1,1,0,100,2,83,0,41,3,122, - 50,73,110,115,116,97,108,108,32,105,109,112,111,114,116,108, - 105,98,32,97,115,32,116,104,101,32,105,109,112,108,101,109, - 101,110,116,97,116,105,111,110,32,111,102,32,105,109,112,111, - 114,116,46,114,19,0,0,0,78,41,11,114,201,0,0,0, - 114,14,0,0,0,114,166,0,0,0,114,110,0,0,0,114, - 142,0,0,0,114,152,0,0,0,218,26,95,102,114,111,122, - 101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,116, - 101,114,110,97,108,114,116,0,0,0,218,8,95,105,110,115, - 116,97,108,108,114,80,0,0,0,114,1,0,0,0,41,3, - 114,199,0,0,0,114,200,0,0,0,114,202,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,203, - 0,0,0,114,4,0,0,115,12,0,0,0,0,2,10,2, - 12,1,12,3,8,1,4,1,114,203,0,0,0,41,2,78, - 78,41,1,78,41,2,78,114,19,0,0,0,41,50,114,3, - 0,0,0,114,116,0,0,0,114,12,0,0,0,114,16,0, - 0,0,114,49,0,0,0,114,29,0,0,0,114,36,0,0, - 0,114,17,0,0,0,114,18,0,0,0,114,41,0,0,0, - 114,42,0,0,0,114,45,0,0,0,114,57,0,0,0,114, - 59,0,0,0,114,69,0,0,0,114,75,0,0,0,114,78, - 0,0,0,114,85,0,0,0,114,96,0,0,0,114,97,0, - 0,0,114,103,0,0,0,114,79,0,0,0,218,6,111,98, - 106,101,99,116,90,9,95,80,79,80,85,76,65,84,69,114, - 129,0,0,0,114,134,0,0,0,114,137,0,0,0,114,92, - 0,0,0,114,81,0,0,0,114,140,0,0,0,114,141,0, - 0,0,114,82,0,0,0,114,142,0,0,0,114,152,0,0, - 0,114,157,0,0,0,114,163,0,0,0,114,165,0,0,0, - 114,170,0,0,0,114,175,0,0,0,90,15,95,69,82,82, - 95,77,83,71,95,80,82,69,70,73,88,114,177,0,0,0, - 114,180,0,0,0,114,181,0,0,0,114,182,0,0,0,114, - 189,0,0,0,114,193,0,0,0,114,196,0,0,0,114,197, - 0,0,0,114,201,0,0,0,114,203,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,8,60,109,111,100,117,108,101,62,8,0,0,0,115, - 94,0,0,0,4,17,4,2,8,8,8,8,4,2,4,3, - 16,4,14,68,14,21,14,16,8,27,8,17,8,11,14,8, - 8,11,8,12,8,16,8,36,14,27,14,101,16,26,6,3, - 10,45,14,60,8,17,8,17,8,24,8,29,8,23,8,15, - 14,73,14,77,14,13,8,9,8,9,10,47,8,20,4,1, - 8,2,8,27,8,16,10,15,8,31,8,27,18,35,8,7, - 8,47, + 1,110,12,124,1,115,84,116,6,100,5,131,1,130,1,124, + 0,12,0,114,106,124,2,100,2,107,2,114,106,116,5,100, + 6,131,1,130,1,100,7,83,0,41,8,122,28,86,101,114, + 105,102,121,32,97,114,103,117,109,101,110,116,115,32,97,114, + 101,32,34,115,97,110,101,34,46,122,31,109,111,100,117,108, + 101,32,110,97,109,101,32,109,117,115,116,32,98,101,32,115, + 116,114,44,32,110,111,116,32,123,125,114,19,0,0,0,122, + 18,108,101,118,101,108,32,109,117,115,116,32,98,101,32,62, + 61,32,48,122,31,95,95,112,97,99,107,97,103,101,95,95, + 32,110,111,116,32,115,101,116,32,116,111,32,97,32,115,116, + 114,105,110,103,122,54,97,116,116,101,109,112,116,101,100,32, + 114,101,108,97,116,105,118,101,32,105,109,112,111,114,116,32, + 119,105,116,104,32,110,111,32,107,110,111,119,110,32,112,97, + 114,101,110,116,32,112,97,99,107,97,103,101,122,17,69,109, + 112,116,121,32,109,111,100,117,108,101,32,110,97,109,101,78, + 41,7,218,10,105,115,105,110,115,116,97,110,99,101,218,3, + 115,116,114,218,9,84,121,112,101,69,114,114,111,114,114,38, + 0,0,0,114,13,0,0,0,114,160,0,0,0,114,71,0, + 0,0,41,3,114,15,0,0,0,114,161,0,0,0,114,162, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, + 0,0,218,13,95,115,97,110,105,116,121,95,99,104,101,99, + 107,142,3,0,0,115,22,0,0,0,0,2,10,1,18,1, + 8,1,8,1,8,1,10,1,10,1,4,1,8,2,14,1, + 114,174,0,0,0,122,16,78,111,32,109,111,100,117,108,101, + 32,110,97,109,101,100,32,122,4,123,33,114,125,99,2,0, + 0,0,0,0,0,0,8,0,0,0,12,0,0,0,67,0, + 0,0,115,220,0,0,0,100,0,125,2,124,0,106,0,100, + 1,131,1,100,2,25,0,125,3,124,3,114,134,124,3,116, + 1,106,2,107,7,114,42,116,3,124,1,124,3,131,2,1, + 0,124,0,116,1,106,2,107,6,114,62,116,1,106,2,124, + 0,25,0,83,0,116,1,106,2,124,3,25,0,125,4,121, + 10,124,4,106,4,125,2,87,0,110,50,4,0,116,5,107, + 10,114,132,1,0,1,0,1,0,116,6,100,3,23,0,106, + 7,124,0,124,3,131,2,125,5,116,8,124,5,124,0,100, + 4,141,2,100,0,130,2,89,0,110,2,88,0,116,9,124, + 0,124,2,131,2,125,6,124,6,100,0,107,8,114,172,116, + 8,116,6,106,7,124,0,131,1,124,0,100,4,141,2,130, + 1,110,8,116,10,124,6,131,1,125,7,124,3,114,216,116, + 1,106,2,124,3,25,0,125,4,116,11,124,4,124,0,106, + 0,100,1,131,1,100,5,25,0,124,7,131,3,1,0,124, + 7,83,0,41,6,78,114,118,0,0,0,114,19,0,0,0, + 122,23,59,32,123,33,114,125,32,105,115,32,110,111,116,32, + 97,32,112,97,99,107,97,103,101,41,1,114,15,0,0,0, + 233,2,0,0,0,41,12,114,119,0,0,0,114,14,0,0, + 0,114,80,0,0,0,114,59,0,0,0,114,128,0,0,0, + 114,91,0,0,0,218,8,95,69,82,82,95,77,83,71,114, + 38,0,0,0,218,19,77,111,100,117,108,101,78,111,116,70, + 111,117,110,100,69,114,114,111,114,114,170,0,0,0,114,141, + 0,0,0,114,5,0,0,0,41,8,114,15,0,0,0,218, + 7,105,109,112,111,114,116,95,114,144,0,0,0,114,120,0, + 0,0,90,13,112,97,114,101,110,116,95,109,111,100,117,108, + 101,114,139,0,0,0,114,83,0,0,0,114,84,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, + 23,95,102,105,110,100,95,97,110,100,95,108,111,97,100,95, + 117,110,108,111,99,107,101,100,161,3,0,0,115,42,0,0, + 0,0,1,4,1,14,1,4,1,10,1,10,2,10,1,10, + 1,10,1,2,1,10,1,14,1,16,1,20,1,10,1,8, + 1,20,2,8,1,4,2,10,1,22,1,114,179,0,0,0, + 99,2,0,0,0,0,0,0,0,4,0,0,0,11,0,0, + 0,67,0,0,0,115,92,0,0,0,116,0,124,0,131,1, + 143,26,1,0,124,0,116,1,106,2,107,7,114,30,116,3, + 124,0,124,1,131,2,83,0,87,0,100,1,81,0,82,0, + 88,0,116,1,106,2,124,0,25,0,125,2,124,2,100,1, + 107,8,114,80,100,2,106,4,124,0,131,1,125,3,116,5, + 124,3,124,0,100,3,141,2,130,1,116,6,124,0,131,1, + 1,0,124,2,83,0,41,4,122,25,70,105,110,100,32,97, + 110,100,32,108,111,97,100,32,116,104,101,32,109,111,100,117, + 108,101,46,78,122,40,105,109,112,111,114,116,32,111,102,32, + 123,125,32,104,97,108,116,101,100,59,32,78,111,110,101,32, + 105,110,32,115,121,115,46,109,111,100,117,108,101,115,41,1, + 114,15,0,0,0,41,7,114,42,0,0,0,114,14,0,0, + 0,114,80,0,0,0,114,179,0,0,0,114,38,0,0,0, + 114,177,0,0,0,114,57,0,0,0,41,4,114,15,0,0, + 0,114,178,0,0,0,114,84,0,0,0,114,68,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, + 14,95,102,105,110,100,95,97,110,100,95,108,111,97,100,188, + 3,0,0,115,20,0,0,0,0,2,10,1,10,1,20,2, + 10,1,8,1,4,1,6,1,12,2,8,1,114,180,0,0, + 0,114,19,0,0,0,99,3,0,0,0,0,0,0,0,3, + 0,0,0,4,0,0,0,67,0,0,0,115,42,0,0,0, + 116,0,124,0,124,1,124,2,131,3,1,0,124,2,100,1, + 107,4,114,32,116,1,124,0,124,1,124,2,131,3,125,0, + 116,2,124,0,116,3,131,2,83,0,41,2,97,50,1,0, + 0,73,109,112,111,114,116,32,97,110,100,32,114,101,116,117, + 114,110,32,116,104,101,32,109,111,100,117,108,101,32,98,97, + 115,101,100,32,111,110,32,105,116,115,32,110,97,109,101,44, + 32,116,104,101,32,112,97,99,107,97,103,101,32,116,104,101, + 32,99,97,108,108,32,105,115,10,32,32,32,32,98,101,105, + 110,103,32,109,97,100,101,32,102,114,111,109,44,32,97,110, + 100,32,116,104,101,32,108,101,118,101,108,32,97,100,106,117, + 115,116,109,101,110,116,46,10,10,32,32,32,32,84,104,105, + 115,32,102,117,110,99,116,105,111,110,32,114,101,112,114,101, + 115,101,110,116,115,32,116,104,101,32,103,114,101,97,116,101, + 115,116,32,99,111,109,109,111,110,32,100,101,110,111,109,105, + 110,97,116,111,114,32,111,102,32,102,117,110,99,116,105,111, + 110,97,108,105,116,121,10,32,32,32,32,98,101,116,119,101, + 101,110,32,105,109,112,111,114,116,95,109,111,100,117,108,101, + 32,97,110,100,32,95,95,105,109,112,111,114,116,95,95,46, + 32,84,104,105,115,32,105,110,99,108,117,100,101,115,32,115, + 101,116,116,105,110,103,32,95,95,112,97,99,107,97,103,101, + 95,95,32,105,102,10,32,32,32,32,116,104,101,32,108,111, + 97,100,101,114,32,100,105,100,32,110,111,116,46,10,10,32, + 32,32,32,114,19,0,0,0,41,4,114,174,0,0,0,114, + 163,0,0,0,114,180,0,0,0,218,11,95,103,99,100,95, + 105,109,112,111,114,116,41,3,114,15,0,0,0,114,161,0, + 0,0,114,162,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,181,0,0,0,204,3,0,0,115, + 8,0,0,0,0,9,12,1,8,1,12,1,114,181,0,0, + 0,99,3,0,0,0,0,0,0,0,6,0,0,0,17,0, + 0,0,67,0,0,0,115,164,0,0,0,116,0,124,0,100, + 1,131,2,114,160,100,2,124,1,107,6,114,58,116,1,124, + 1,131,1,125,1,124,1,106,2,100,2,131,1,1,0,116, + 0,124,0,100,3,131,2,114,58,124,1,106,3,124,0,106, + 4,131,1,1,0,120,100,124,1,68,0,93,92,125,3,116, + 0,124,0,124,3,131,2,115,64,100,4,106,5,124,0,106, + 6,124,3,131,2,125,4,121,14,116,7,124,2,124,4,131, + 2,1,0,87,0,113,64,4,0,116,8,107,10,114,154,1, + 0,125,5,1,0,122,20,124,5,106,9,124,4,107,2,114, + 136,119,64,130,0,87,0,89,0,100,5,100,5,125,5,126, + 5,88,0,113,64,88,0,113,64,87,0,124,0,83,0,41, + 6,122,238,70,105,103,117,114,101,32,111,117,116,32,119,104, + 97,116,32,95,95,105,109,112,111,114,116,95,95,32,115,104, + 111,117,108,100,32,114,101,116,117,114,110,46,10,10,32,32, + 32,32,84,104,101,32,105,109,112,111,114,116,95,32,112,97, + 114,97,109,101,116,101,114,32,105,115,32,97,32,99,97,108, + 108,97,98,108,101,32,119,104,105,99,104,32,116,97,107,101, + 115,32,116,104,101,32,110,97,109,101,32,111,102,32,109,111, + 100,117,108,101,32,116,111,10,32,32,32,32,105,109,112,111, + 114,116,46,32,73,116,32,105,115,32,114,101,113,117,105,114, + 101,100,32,116,111,32,100,101,99,111,117,112,108,101,32,116, + 104,101,32,102,117,110,99,116,105,111,110,32,102,114,111,109, + 32,97,115,115,117,109,105,110,103,32,105,109,112,111,114,116, + 108,105,98,39,115,10,32,32,32,32,105,109,112,111,114,116, + 32,105,109,112,108,101,109,101,110,116,97,116,105,111,110,32, + 105,115,32,100,101,115,105,114,101,100,46,10,10,32,32,32, + 32,114,128,0,0,0,250,1,42,218,7,95,95,97,108,108, + 95,95,122,5,123,125,46,123,125,78,41,10,114,4,0,0, + 0,114,127,0,0,0,218,6,114,101,109,111,118,101,218,6, + 101,120,116,101,110,100,114,183,0,0,0,114,38,0,0,0, + 114,1,0,0,0,114,59,0,0,0,114,177,0,0,0,114, + 15,0,0,0,41,6,114,84,0,0,0,218,8,102,114,111, + 109,108,105,115,116,114,178,0,0,0,218,1,120,90,9,102, + 114,111,109,95,110,97,109,101,90,3,101,120,99,114,10,0, + 0,0,114,10,0,0,0,114,11,0,0,0,218,16,95,104, + 97,110,100,108,101,95,102,114,111,109,108,105,115,116,219,3, + 0,0,115,32,0,0,0,0,10,10,1,8,1,8,1,10, + 1,10,1,12,1,10,1,10,1,14,1,2,1,14,1,16, + 4,10,1,2,1,24,1,114,188,0,0,0,99,1,0,0, + 0,0,0,0,0,3,0,0,0,6,0,0,0,67,0,0, + 0,115,146,0,0,0,124,0,106,0,100,1,131,1,125,1, + 124,0,106,0,100,2,131,1,125,2,124,1,100,3,107,9, + 114,82,124,2,100,3,107,9,114,78,124,1,124,2,106,1, + 107,3,114,78,116,2,106,3,100,4,124,1,155,2,100,5, + 124,2,106,1,155,2,100,6,157,5,116,4,100,7,100,8, + 141,3,1,0,124,1,83,0,124,2,100,3,107,9,114,96, + 124,2,106,1,83,0,116,2,106,3,100,9,116,4,100,7, + 100,8,141,3,1,0,124,0,100,10,25,0,125,1,100,11, + 124,0,107,7,114,142,124,1,106,5,100,12,131,1,100,13, + 25,0,125,1,124,1,83,0,41,14,122,167,67,97,108,99, + 117,108,97,116,101,32,119,104,97,116,32,95,95,112,97,99, + 107,97,103,101,95,95,32,115,104,111,117,108,100,32,98,101, + 46,10,10,32,32,32,32,95,95,112,97,99,107,97,103,101, + 95,95,32,105,115,32,110,111,116,32,103,117,97,114,97,110, + 116,101,101,100,32,116,111,32,98,101,32,100,101,102,105,110, + 101,100,32,111,114,32,99,111,117,108,100,32,98,101,32,115, + 101,116,32,116,111,32,78,111,110,101,10,32,32,32,32,116, + 111,32,114,101,112,114,101,115,101,110,116,32,116,104,97,116, + 32,105,116,115,32,112,114,111,112,101,114,32,118,97,108,117, + 101,32,105,115,32,117,110,107,110,111,119,110,46,10,10,32, + 32,32,32,114,131,0,0,0,114,90,0,0,0,78,122,32, + 95,95,112,97,99,107,97,103,101,95,95,32,33,61,32,95, + 95,115,112,101,99,95,95,46,112,97,114,101,110,116,32,40, + 122,4,32,33,61,32,250,1,41,233,3,0,0,0,41,1, + 90,10,115,116,97,99,107,108,101,118,101,108,122,89,99,97, + 110,39,116,32,114,101,115,111,108,118,101,32,112,97,99,107, + 97,103,101,32,102,114,111,109,32,95,95,115,112,101,99,95, + 95,32,111,114,32,95,95,112,97,99,107,97,103,101,95,95, + 44,32,102,97,108,108,105,110,103,32,98,97,99,107,32,111, + 110,32,95,95,110,97,109,101,95,95,32,97,110,100,32,95, + 95,112,97,116,104,95,95,114,1,0,0,0,114,128,0,0, + 0,114,118,0,0,0,114,19,0,0,0,41,6,114,30,0, + 0,0,114,120,0,0,0,114,167,0,0,0,114,168,0,0, + 0,114,169,0,0,0,114,119,0,0,0,41,3,218,7,103, + 108,111,98,97,108,115,114,161,0,0,0,114,83,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, + 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, + 95,95,250,3,0,0,115,30,0,0,0,0,7,10,1,10, + 1,8,1,18,1,22,2,10,1,4,1,8,1,6,2,6, + 2,10,1,8,1,8,1,14,1,114,192,0,0,0,99,5, + 0,0,0,0,0,0,0,9,0,0,0,5,0,0,0,67, + 0,0,0,115,166,0,0,0,124,4,100,1,107,2,114,18, + 116,0,124,0,131,1,125,5,110,36,124,1,100,2,107,9, + 114,30,124,1,110,2,105,0,125,6,116,1,124,6,131,1, + 125,7,116,0,124,0,124,7,124,4,131,3,125,5,124,3, + 115,150,124,4,100,1,107,2,114,84,116,0,124,0,106,2, + 100,3,131,1,100,1,25,0,131,1,83,0,124,0,115,92, + 124,5,83,0,116,3,124,0,131,1,116,3,124,0,106,2, + 100,3,131,1,100,1,25,0,131,1,24,0,125,8,116,4, + 106,5,124,5,106,6,100,2,116,3,124,5,106,6,131,1, + 124,8,24,0,133,2,25,0,25,0,83,0,110,12,116,7, + 124,5,124,3,116,0,131,3,83,0,100,2,83,0,41,4, + 97,215,1,0,0,73,109,112,111,114,116,32,97,32,109,111, + 100,117,108,101,46,10,10,32,32,32,32,84,104,101,32,39, + 103,108,111,98,97,108,115,39,32,97,114,103,117,109,101,110, + 116,32,105,115,32,117,115,101,100,32,116,111,32,105,110,102, + 101,114,32,119,104,101,114,101,32,116,104,101,32,105,109,112, + 111,114,116,32,105,115,32,111,99,99,117,114,114,105,110,103, + 32,102,114,111,109,10,32,32,32,32,116,111,32,104,97,110, + 100,108,101,32,114,101,108,97,116,105,118,101,32,105,109,112, + 111,114,116,115,46,32,84,104,101,32,39,108,111,99,97,108, + 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,105, + 103,110,111,114,101,100,46,32,84,104,101,10,32,32,32,32, + 39,102,114,111,109,108,105,115,116,39,32,97,114,103,117,109, + 101,110,116,32,115,112,101,99,105,102,105,101,115,32,119,104, + 97,116,32,115,104,111,117,108,100,32,101,120,105,115,116,32, + 97,115,32,97,116,116,114,105,98,117,116,101,115,32,111,110, + 32,116,104,101,32,109,111,100,117,108,101,10,32,32,32,32, + 98,101,105,110,103,32,105,109,112,111,114,116,101,100,32,40, + 101,46,103,46,32,96,96,102,114,111,109,32,109,111,100,117, + 108,101,32,105,109,112,111,114,116,32,60,102,114,111,109,108, + 105,115,116,62,96,96,41,46,32,32,84,104,101,32,39,108, + 101,118,101,108,39,10,32,32,32,32,97,114,103,117,109,101, + 110,116,32,114,101,112,114,101,115,101,110,116,115,32,116,104, + 101,32,112,97,99,107,97,103,101,32,108,111,99,97,116,105, + 111,110,32,116,111,32,105,109,112,111,114,116,32,102,114,111, + 109,32,105,110,32,97,32,114,101,108,97,116,105,118,101,10, + 32,32,32,32,105,109,112,111,114,116,32,40,101,46,103,46, + 32,96,96,102,114,111,109,32,46,46,112,107,103,32,105,109, + 112,111,114,116,32,109,111,100,96,96,32,119,111,117,108,100, + 32,104,97,118,101,32,97,32,39,108,101,118,101,108,39,32, + 111,102,32,50,41,46,10,10,32,32,32,32,114,19,0,0, + 0,78,114,118,0,0,0,41,8,114,181,0,0,0,114,192, + 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,159, + 0,0,0,114,14,0,0,0,114,80,0,0,0,114,1,0, + 0,0,114,188,0,0,0,41,9,114,15,0,0,0,114,191, + 0,0,0,218,6,108,111,99,97,108,115,114,186,0,0,0, + 114,162,0,0,0,114,84,0,0,0,90,8,103,108,111,98, + 97,108,115,95,114,161,0,0,0,90,7,99,117,116,95,111, + 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,10,95,95,105,109,112,111,114,116,95,95,21,4,0, + 0,115,26,0,0,0,0,11,8,1,10,2,16,1,8,1, + 12,1,4,3,8,1,18,1,4,1,4,4,26,3,32,2, + 114,195,0,0,0,99,1,0,0,0,0,0,0,0,2,0, + 0,0,3,0,0,0,67,0,0,0,115,38,0,0,0,116, + 0,106,1,124,0,131,1,125,1,124,1,100,0,107,8,114, + 30,116,2,100,1,124,0,23,0,131,1,130,1,116,3,124, + 1,131,1,83,0,41,2,78,122,25,110,111,32,98,117,105, + 108,116,45,105,110,32,109,111,100,117,108,101,32,110,97,109, + 101,100,32,41,4,114,142,0,0,0,114,146,0,0,0,114, + 71,0,0,0,114,141,0,0,0,41,2,114,15,0,0,0, + 114,83,0,0,0,114,10,0,0,0,114,10,0,0,0,114, + 11,0,0,0,218,18,95,98,117,105,108,116,105,110,95,102, + 114,111,109,95,110,97,109,101,56,4,0,0,115,8,0,0, + 0,0,1,10,1,8,1,12,1,114,196,0,0,0,99,2, + 0,0,0,0,0,0,0,12,0,0,0,12,0,0,0,67, + 0,0,0,115,244,0,0,0,124,1,97,0,124,0,97,1, + 116,2,116,1,131,1,125,2,120,86,116,1,106,3,106,4, + 131,0,68,0,93,72,92,2,125,3,125,4,116,5,124,4, + 124,2,131,2,114,28,124,3,116,1,106,6,107,6,114,62, + 116,7,125,5,110,18,116,0,106,8,124,3,131,1,114,28, + 116,9,125,5,110,2,113,28,116,10,124,4,124,5,131,2, + 125,6,116,11,124,6,124,4,131,2,1,0,113,28,87,0, + 116,1,106,3,116,12,25,0,125,7,120,54,100,5,68,0, + 93,46,125,8,124,8,116,1,106,3,107,7,114,144,116,13, + 124,8,131,1,125,9,110,10,116,1,106,3,124,8,25,0, + 125,9,116,14,124,7,124,8,124,9,131,3,1,0,113,120, + 87,0,121,12,116,13,100,2,131,1,125,10,87,0,110,24, + 4,0,116,15,107,10,114,206,1,0,1,0,1,0,100,3, + 125,10,89,0,110,2,88,0,116,14,124,7,100,2,124,10, + 131,3,1,0,116,13,100,4,131,1,125,11,116,14,124,7, + 100,4,124,11,131,3,1,0,100,3,83,0,41,6,122,250, + 83,101,116,117,112,32,105,109,112,111,114,116,108,105,98,32, + 98,121,32,105,109,112,111,114,116,105,110,103,32,110,101,101, + 100,101,100,32,98,117,105,108,116,45,105,110,32,109,111,100, + 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, + 110,103,32,116,104,101,109,10,32,32,32,32,105,110,116,111, + 32,116,104,101,32,103,108,111,98,97,108,32,110,97,109,101, + 115,112,97,99,101,46,10,10,32,32,32,32,65,115,32,115, + 121,115,32,105,115,32,110,101,101,100,101,100,32,102,111,114, + 32,115,121,115,46,109,111,100,117,108,101,115,32,97,99,99, + 101,115,115,32,97,110,100,32,95,105,109,112,32,105,115,32, + 110,101,101,100,101,100,32,116,111,32,108,111,97,100,32,98, + 117,105,108,116,45,105,110,10,32,32,32,32,109,111,100,117, + 108,101,115,44,32,116,104,111,115,101,32,116,119,111,32,109, + 111,100,117,108,101,115,32,109,117,115,116,32,98,101,32,101, + 120,112,108,105,99,105,116,108,121,32,112,97,115,115,101,100, + 32,105,110,46,10,10,32,32,32,32,114,167,0,0,0,114, + 20,0,0,0,78,114,55,0,0,0,41,1,114,167,0,0, + 0,41,16,114,52,0,0,0,114,14,0,0,0,114,13,0, + 0,0,114,80,0,0,0,218,5,105,116,101,109,115,114,171, + 0,0,0,114,70,0,0,0,114,142,0,0,0,114,76,0, + 0,0,114,152,0,0,0,114,129,0,0,0,114,134,0,0, + 0,114,1,0,0,0,114,196,0,0,0,114,5,0,0,0, + 114,71,0,0,0,41,12,218,10,115,121,115,95,109,111,100, + 117,108,101,218,11,95,105,109,112,95,109,111,100,117,108,101, + 90,11,109,111,100,117,108,101,95,116,121,112,101,114,15,0, + 0,0,114,84,0,0,0,114,94,0,0,0,114,83,0,0, + 0,90,11,115,101,108,102,95,109,111,100,117,108,101,90,12, + 98,117,105,108,116,105,110,95,110,97,109,101,90,14,98,117, + 105,108,116,105,110,95,109,111,100,117,108,101,90,13,116,104, + 114,101,97,100,95,109,111,100,117,108,101,90,14,119,101,97, + 107,114,101,102,95,109,111,100,117,108,101,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,218,6,95,115,101,116, + 117,112,63,4,0,0,115,50,0,0,0,0,9,4,1,4, + 3,8,1,20,1,10,1,10,1,6,1,10,1,6,2,2, + 1,10,1,14,3,10,1,10,1,10,1,10,2,10,1,16, + 3,2,1,12,1,14,2,10,1,12,3,8,1,114,200,0, + 0,0,99,2,0,0,0,0,0,0,0,3,0,0,0,3, + 0,0,0,67,0,0,0,115,66,0,0,0,116,0,124,0, + 124,1,131,2,1,0,116,1,106,2,106,3,116,4,131,1, + 1,0,116,1,106,2,106,3,116,5,131,1,1,0,100,1, + 100,2,108,6,125,2,124,2,97,7,124,2,106,8,116,1, + 106,9,116,10,25,0,131,1,1,0,100,2,83,0,41,3, + 122,50,73,110,115,116,97,108,108,32,105,109,112,111,114,116, + 108,105,98,32,97,115,32,116,104,101,32,105,109,112,108,101, + 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, + 111,114,116,46,114,19,0,0,0,78,41,11,114,200,0,0, + 0,114,14,0,0,0,114,166,0,0,0,114,110,0,0,0, + 114,142,0,0,0,114,152,0,0,0,218,26,95,102,114,111, + 122,101,110,95,105,109,112,111,114,116,108,105,98,95,101,120, + 116,101,114,110,97,108,114,116,0,0,0,218,8,95,105,110, + 115,116,97,108,108,114,80,0,0,0,114,1,0,0,0,41, + 3,114,198,0,0,0,114,199,0,0,0,114,201,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, + 202,0,0,0,110,4,0,0,115,12,0,0,0,0,2,10, + 2,12,1,12,3,8,1,4,1,114,202,0,0,0,41,2, + 78,78,41,1,78,41,2,78,114,19,0,0,0,41,50,114, + 3,0,0,0,114,116,0,0,0,114,12,0,0,0,114,16, + 0,0,0,114,49,0,0,0,114,29,0,0,0,114,36,0, + 0,0,114,17,0,0,0,114,18,0,0,0,114,41,0,0, + 0,114,42,0,0,0,114,45,0,0,0,114,57,0,0,0, + 114,59,0,0,0,114,69,0,0,0,114,75,0,0,0,114, + 78,0,0,0,114,85,0,0,0,114,96,0,0,0,114,97, + 0,0,0,114,103,0,0,0,114,79,0,0,0,218,6,111, + 98,106,101,99,116,90,9,95,80,79,80,85,76,65,84,69, + 114,129,0,0,0,114,134,0,0,0,114,137,0,0,0,114, + 92,0,0,0,114,81,0,0,0,114,140,0,0,0,114,141, + 0,0,0,114,82,0,0,0,114,142,0,0,0,114,152,0, + 0,0,114,157,0,0,0,114,163,0,0,0,114,165,0,0, + 0,114,170,0,0,0,114,174,0,0,0,90,15,95,69,82, + 82,95,77,83,71,95,80,82,69,70,73,88,114,176,0,0, + 0,114,179,0,0,0,114,180,0,0,0,114,181,0,0,0, + 114,188,0,0,0,114,192,0,0,0,114,195,0,0,0,114, + 196,0,0,0,114,200,0,0,0,114,202,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, + 0,0,218,8,60,109,111,100,117,108,101,62,8,0,0,0, + 115,94,0,0,0,4,17,4,2,8,8,8,8,4,2,4, + 3,16,4,14,68,14,21,14,16,8,27,8,17,8,11,14, + 8,8,11,8,12,8,16,8,36,14,27,14,101,16,26,6, + 3,10,45,14,60,8,17,8,17,8,24,8,29,8,23,8, + 15,14,73,14,77,14,13,8,9,8,9,10,47,8,16,4, + 1,8,2,8,27,8,16,10,15,8,31,8,27,18,35,8, + 7,8,47, }; From a819e5e1e66a1a5f9a7073c8a1ff3c3f304c917b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 16 Jul 2017 07:48:08 +0300 Subject: [PATCH 0569/1345] [3.6] bpo-30936: Fix a reference leak in json when fail to sort keys. (GH-2712). (#2727) (cherry picked from commit 49f6449ef4b81537c19b82329caaf60596c516c2) --- Lib/test/test_json/test_speedups.py | 4 ++++ Modules/_json.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 8d98ab5ae1c398..56f1882001d666 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -44,3 +44,7 @@ def test(name): self.assertRaises(ZeroDivisionError, test, 'check_circular') self.assertRaises(ZeroDivisionError, test, 'allow_nan') self.assertRaises(ZeroDivisionError, test, 'sort_keys') + + def test_unsortable_keys(self): + with self.assertRaises(TypeError): + self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) diff --git a/Modules/_json.c b/Modules/_json.c index a84b0851090779..1be4c17cf95369 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1601,8 +1601,10 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, if (items == NULL) goto bail; sortkeys = PyObject_IsTrue(s->sort_keys); - if (sortkeys < 0 || (sortkeys && PyList_Sort(items) < 0)) + if (sortkeys < 0 || (sortkeys && PyList_Sort(items) < 0)) { + Py_DECREF(items); goto bail; + } it = PyObject_GetIter(items); Py_DECREF(items); if (it == NULL) From a61089f57337130710b20c71e5c33f873e842fcc Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 17 Jul 2017 00:54:03 -0400 Subject: [PATCH 0570/1345] Bump to v3.6.2+ --- Include/patchlevel.h | 6 +++--- Misc/NEWS | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 0c82f5c0db855a..67d0059b2797e0 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 2 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 2 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.2rc2+" +#define PY_VERSION "3.6.2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 3cdacc2b637fe4..e7e9cb449d308f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,14 @@ Library They now always return bytes. +What's New in Python 3.6.2? +=========================== + +*Release date: 2017-07-17* + +- No changes since release candidate 2 + + What's New in Python 3.6.2 release candidate 2? =============================================== From 8663c4549109cda14e039a4b22b85343a20793aa Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 17 Jul 2017 00:59:03 -0400 Subject: [PATCH 0571/1345] Also bump README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 71eb257b529d4e..4cbffcdb756e8e 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.6.2 release candidate 1 -================================================ +This is Python version 3.6.2+ +============================= .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI From 241c4a28d9fec7f2beffc9cbd21f156999959a9d Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 19 Jul 2017 00:50:52 +0800 Subject: [PATCH 0572/1345] bpo-30961: Fix decrementing a borrowed reference in tracemalloc. (#2747) (#2748) (cherry picked from commit 4ed5ad79ec6c6270e6018bd0a55656305ee60907) --- .../next/Library/2017-07-18-23-47-51.bpo-30961.064jz0.rst | 1 + Modules/_tracemalloc.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-18-23-47-51.bpo-30961.064jz0.rst diff --git a/Misc/NEWS.d/next/Library/2017-07-18-23-47-51.bpo-30961.064jz0.rst b/Misc/NEWS.d/next/Library/2017-07-18-23-47-51.bpo-30961.064jz0.rst new file mode 100644 index 00000000000000..b89c6d4769dab9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-18-23-47-51.bpo-30961.064jz0.rst @@ -0,0 +1 @@ +Fix decrementing a borrowed reference in tracemalloc. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index ec8bd960a7f7e0..0ce38ffc78c53f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1731,7 +1731,7 @@ _PyTraceMalloc_Init(void) if (key == NULL) return -1; - value = PyDict_GetItemWithError(xoptions, key); + value = PyDict_GetItemWithError(xoptions, key); /* borrowed */ Py_DECREF(key); if (value == NULL) { if (PyErr_Occurred()) @@ -1742,7 +1742,6 @@ _PyTraceMalloc_Init(void) } nframe = parse_sys_xoptions(value); - Py_DECREF(value); if (nframe < 0) { Py_FatalError("-X tracemalloc=NFRAME: invalid number of frames"); } From e99d3a52a50b3f836fb9fb88f317aacddd494858 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Jul 2017 09:11:08 +0200 Subject: [PATCH 0573/1345] [3.6] bpo-30450: Improved logic for obtaining dependencies (#2751) Adds alternate download approach for nuget.exe Fall back to git.exe if no Python is found. (#2739) Also check whether git.exe is on PATH if it will be used. Add support for HOST_PYTHON variable. Clear internal environment variables used in find_python.bat Use HOST_PYTHON as the actual Python if it is recent enough. Adds HOST_PYTHON variable to AppVeyor configuration --- .github/appveyor.yml | 2 + Doc/make.bat | 13 ++++++- PCbuild/build.bat | 2 - PCbuild/find_python.bat | 75 +++++++++++++++++++++++++++++++++++++ PCbuild/get_externals.bat | 34 +++++++---------- PCbuild/urlretrieve.py | 39 +++++++++++++++++++ Tools/msi/buildrelease.bat | 3 -- Tools/msi/get_externals.bat | 24 ++---------- 8 files changed, 145 insertions(+), 47 deletions(-) create mode 100644 PCbuild/find_python.bat create mode 100644 PCbuild/urlretrieve.py diff --git a/.github/appveyor.yml b/.github/appveyor.yml index 04566eafe58bfd..38ca3c4de1501c 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -9,6 +9,8 @@ build_script: - cmd: PCbuild\build.bat -e test_script: - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 +environment: + HOST_PYTHON: C:\Python36\python.exe # Only trigger AppVeyor if actual code or its configuration changes only_commits: diff --git a/Doc/make.bat b/Doc/make.bat index d9c0ad0adecfe6..b9e8a759c51a72 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -5,8 +5,19 @@ pushd %~dp0 set this=%~n0 -if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build +call ..\PCBuild\find_python.bat %PYTHON% +if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" ( + set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe + rem Cannot use %SPHINXBUILD% in the same block where we set it + if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" ( + echo Installing sphinx with %PYTHON% + "%PYTHON%" -m pip install sphinx + if errorlevel 1 exit /B + ) +) + if "%PYTHON%" EQU "" set PYTHON=py +if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build if "%1" NEQ "htmlhelp" goto :skiphhcsearch if exist "%HTMLHELP%" goto :skiphhcsearch diff --git a/PCbuild/build.bat b/PCbuild/build.bat index b63843fa95d971..81e500d5544286 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options] echo. echo.Build CPython from the command line. Requires the appropriate echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). -echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is -echo.given. echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat new file mode 100644 index 00000000000000..ba3a0f5a10daa3 --- /dev/null +++ b/PCbuild/find_python.bat @@ -0,0 +1,75 @@ +@rem +@rem Searches for python.exe and may download a private copy from nuget. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found + +:begin_search +@set PYTHON= + +@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR% +@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) + +@rem If we have Python in externals, use that one +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found + +@rem If HOST_PYTHON is recent enough, use that +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found + +@rem If py.exe finds a recent enough version, use that one +@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found + +@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" +@set _Py_NUGET=%NUGET% +@set _Py_NUGET_URL=%NUGET_URL% +@set _Py_HOST_PYTHON=%HOST_PYTHON% +@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py +@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if NOT exist "%_Py_NUGET%" ( + @echo Downloading nuget... + @rem NB: Must use single quotes around NUGET here, NOT double! + @rem Otherwise, a space in the path would break things + @rem If it fails, retry with any available copy of Python + @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%' + @if errorlevel 1 ( + @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%" + ) +) +@echo Installing Python via nuget... +@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work +@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found + + +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= +@exit /b 1 + +:found +@echo Using %PYTHON% (%_Py_Python_Source%) +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 6e466a34484d77..0a86892c3500ec 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -4,8 +4,6 @@ rem Simple script to fetch source for external libraries if "%PCBUILD%"=="" (set PCBUILD=%~dp0) if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) -if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) set DO_FETCH=true set DO_CLEAN=false @@ -13,7 +11,7 @@ set DO_CLEAN=false :CheckOpts if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts -if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts @@ -33,22 +31,10 @@ if "%DO_FETCH%"=="false" goto end if "%ORG%"=="" (set ORG=python) -if "%PYTHON_FOR_BUILD%"=="" ( - echo Checking for installed python... - py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) -) -if "%PYTHON_FOR_BUILD%"=="" ( - if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" - if NOT exist "%NUGET%" ( - echo Downloading nuget... - rem NB: Must use single quotes around NUGET here, NOT double! - rem Otherwise, a space in the path would break things - powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' - ) - echo Installing Python via nuget... - "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" - rem Quote it here; it's not quoted later because "py -3.6" wouldn't work - set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" +call "%PCBUILD%find_python.bat" "%PYTHON%" + +if "%PYTHON%"=="" ( + where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 ) echo.Fetching external libraries... @@ -65,9 +51,12 @@ set libraries=%libraries% xz-5.2.2 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( echo.%%e already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%e with git... + git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e" ) else ( echo.Fetching %%e... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e + %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e ) ) @@ -80,9 +69,12 @@ if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( echo.%%b already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%b with git... + git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b + %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b ) ) diff --git a/PCbuild/urlretrieve.py b/PCbuild/urlretrieve.py new file mode 100644 index 00000000000000..9df773c35d7ed6 --- /dev/null +++ b/PCbuild/urlretrieve.py @@ -0,0 +1,39 @@ +# Simple Python script to download a file. Used as a fallback +# when other more reliable methods fail. +# +from __future__ import print_function +import sys + +try: + from requests import get +except ImportError: + try: + from urllib.request import urlretrieve + USING = "urllib.request.urlretrieve" + except ImportError: + try: + from urllib import urlretrieve + USING = "urllib.retrieve" + except ImportError: + print("Python at", sys.executable, "is not suitable", + "for downloading files.", file=sys.stderr) + sys.exit(2) +else: + USING = "requests.get" + + def urlretrieve(url, filename): + r = get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + f.write(chunk) + return filename + +if __name__ == '__main__': + if len(sys.argv) != 3: + print("Usage: urlretrieve.py [url] [filename]", file=sys.stderr) + sys.exit(1) + URL = sys.argv[1] + FILENAME = sys.argv[2] + print("Downloading from", URL, "to", FILENAME, "using", USING) + urlretrieve(URL, FILENAME) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index f60fe6af393165..1f5278207b474b 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -76,9 +76,6 @@ if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & if "%SKIPBUILD%" EQU "1" goto skipdoc if "%SKIPDOC%" EQU "1" goto skipdoc -if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1 -if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1 - call "%D%..\..\doc\make.bat" htmlhelp if errorlevel 1 goto :eof :skipdoc diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index aece81fbb1cf3e..913512a778fa6b 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -12,7 +12,7 @@ set DO_FETCH=true set DO_CLEAN=false :CheckOpts -if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts @@ -32,23 +32,7 @@ if "%DO_FETCH%"=="false" goto end if "%ORG%"=="" (set ORG=python) -if "%PYTHON_FOR_BUILD%"=="" ( - echo Checking for installed python... - py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) -) -if "%PYTHON_FOR_BUILD%"=="" ( - if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" - if NOT exist "%NUGET%" ( - echo Downloading nuget... - rem NB: Must use single quotes around NUGET here, NOT double! - rem Otherwise, a space in the path would break things - powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' - ) - echo Installing Python via nuget... - "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" - rem Quote it here; it's not quoted later because "py -3.6" wouldn't work - set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" -) +call "%PCBUILD%\find_python.bat" "%PYTHON%" echo.Fetching external libraries... @@ -59,7 +43,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e + %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e ) ) @@ -79,7 +63,7 @@ for %%b in (%binaries%) do ( echo.%%b already exists, skipping. ) else ( echo.Fetching %%b... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b + %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b ) ) From ae4dca7701ca77a37aa8c956450ff8e21fe6883e Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Wed, 19 Jul 2017 17:21:09 -0700 Subject: [PATCH 0574/1345] [3.6] bpo-30883: Use pythontest.net instead of debian.org in test_urllib2net (GH-2755) --- Lib/test/test_urllib2net.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index cad83fd5aa6a88..13e2dd5b784205 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -97,8 +97,8 @@ def setUp(self): def test_ftp(self): urls = [ - 'ftp://ftp.debian.org/debian/README', - ('ftp://ftp.debian.org/debian/non-existent-file', + 'ftp://www.pythontest.net/README', + ('ftp://www.pythontest.net/non-existent-file', None, urllib.error.URLError), ] self._test_urls(urls, self._extra_handlers()) @@ -287,7 +287,7 @@ def test_http_timeout(self): self.addCleanup(u.close) self.assertEqual(u.fp.raw._sock.gettimeout(), 120) - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) From 65c24c846797b93d7adb72fc400f95a570f43fa9 Mon Sep 17 00:00:00 2001 From: terryjreedy Date: Thu, 20 Jul 2017 01:23:00 -0400 Subject: [PATCH 0575/1345] [3.6] bpo-30917: IDLE: Add config.IdleConf unittests (GH-2691) (#2753) Patch by Louie Lu. (cherry picked from commit f776eb0) (includes diffs of ed014f7 and 9f9192a) * fix config reset from pr 2754 * Fix test_get_font (from pr 2769) --- Lib/idlelib/config.py | 22 +- Lib/idlelib/idle_test/test_config.py | 453 +++++++++++++++++- .../2017-07-17-23-35-57.bpo-30917.hSiuuO.rst | 3 + 3 files changed, 460 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-17-23-35-57.bpo-30917.hSiuuO.rst diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 4cf44271f3a750..63d9a44234560b 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -30,6 +30,7 @@ import sys from tkinter.font import Font +import idlelib class InvalidConfigType(Exception): pass class InvalidConfigSet(Exception): pass @@ -159,14 +160,15 @@ class IdleConf: for config_type in self.config_types: (user home dir)/.idlerc/config-{config-type}.cfg """ - def __init__(self): + def __init__(self, _utest=False): self.config_types = ('main', 'highlight', 'keys', 'extensions') self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg - self.CreateConfigHandlers() - self.LoadCfgFiles() + if not _utest: + self.CreateConfigHandlers() + self.LoadCfgFiles() def CreateConfigHandlers(self): "Populate default and user config parser dictionaries." @@ -215,7 +217,8 @@ def GetUserCfgDir(self): except OSError: warn = ('\n Warning: unable to create user config directory\n' + userDir + '\n Check path and permissions.\n Exiting!\n') - print(warn, file=sys.stderr) + if not idlelib.testing: + print(warn, file=sys.stderr) raise SystemExit # TODO continue without userDIr instead of exit return userDir @@ -463,16 +466,7 @@ def GetExtensions(self, active_only=True, def RemoveKeyBindNames(self, extnNameList): "Return extnNameList with keybinding section names removed." - # TODO Easier to return filtered copy with list comp - names = extnNameList - kbNameIndicies = [] - for name in names: - if name.endswith(('_bindings', '_cfgBindings')): - kbNameIndicies.append(names.index(name)) - kbNameIndicies.sort(reverse=True) - for index in kbNameIndicies: #delete each keybinding section name - del(names[index]) - return names + return [n for n in extnNameList if not n.endswith(('_bindings', '_cfgBindings'))] def GetExtnNameForEvent(self, virtualEvent): """Return the name of the extension binding virtualEvent, or None. diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index f5a609f6d1600b..bbf06504fc2552 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -1,15 +1,19 @@ '''Test idlelib.config. -Coverage: 46% (100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). -* Except is OSError clause in Save method. -Much of IdleConf is exercised by ConfigDialog and test_configdialog, -but it should be tested here. +Coverage: 96% (100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). +* Exception is OSError clause in Save method. +Much of IdleConf is also exercised by ConfigDialog and test_configdialog. ''' +import copy +import sys import os import tempfile from test.support import captured_stderr, findfile import unittest +from unittest import mock +import idlelib from idlelib import config +from idlelib.idle_test.mock_idle import Func # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. @@ -25,9 +29,11 @@ def setUpModule(): idleConf.userCfg = testcfg + idlelib.testing = True def tearDownModule(): idleConf.userCfg = usercfg + idlelib.testing = False class IdleConfParserTest(unittest.TestCase): @@ -185,6 +191,445 @@ def test_save(self): self.assertFalse(os.path.exists(path)) +class IdleConfTest(unittest.TestCase): + """Test for idleConf""" + + @classmethod + def setUpClass(cls): + cls.config_string = {} + + conf = config.IdleConf(_utest=True) + if __name__ != '__main__': + idle_dir = os.path.dirname(__file__) + else: + idle_dir = os.path.abspath(sys.path[0]) + for ctype in conf.config_types: + config_path = os.path.join(idle_dir, '../config-%s.def' % ctype) + with open(config_path, 'r') as f: + cls.config_string[ctype] = f.read() + + cls.orig_warn = config._warn + config._warn = Func() + + @classmethod + def tearDownClass(cls): + config._warn = cls.orig_warn + + def new_config(self, _utest=False): + return config.IdleConf(_utest=_utest) + + def mock_config(self): + """Return a mocked idleConf + + Both default and user config used the same config-*.def + """ + conf = config.IdleConf(_utest=True) + for ctype in conf.config_types: + conf.defaultCfg[ctype] = config.IdleConfParser('') + conf.defaultCfg[ctype].read_string(self.config_string[ctype]) + conf.userCfg[ctype] = config.IdleUserConfParser('') + conf.userCfg[ctype].read_string(self.config_string[ctype]) + + return conf + + @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system') + def test_get_user_cfg_dir_unix(self): + "Test to get user config directory under unix" + conf = self.new_config(_utest=True) + + # Check normal way should success + with mock.patch('os.path.expanduser', return_value='/home/foo'): + with mock.patch('os.path.exists', return_value=True): + self.assertEqual(conf.GetUserCfgDir(), '/home/foo/.idlerc') + + # Check os.getcwd should success + with mock.patch('os.path.expanduser', return_value='~'): + with mock.patch('os.getcwd', return_value='/home/foo/cpython'): + with mock.patch('os.mkdir'): + self.assertEqual(conf.GetUserCfgDir(), + '/home/foo/cpython/.idlerc') + + # Check user dir not exists and created failed should raise SystemExit + with mock.patch('os.path.join', return_value='/path/not/exists'): + with self.assertRaises(SystemExit): + with self.assertRaises(FileNotFoundError): + conf.GetUserCfgDir() + + @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for windows system') + def test_get_user_cfg_dir_windows(self): + "Test to get user config directory under windows" + conf = self.new_config(_utest=True) + + # Check normal way should success + with mock.patch('os.path.expanduser', return_value='C:\\foo'): + with mock.patch('os.path.exists', return_value=True): + self.assertEqual(conf.GetUserCfgDir(), 'C:\\foo\\.idlerc') + + # Check os.getcwd should success + with mock.patch('os.path.expanduser', return_value='~'): + with mock.patch('os.getcwd', return_value='C:\\foo\\cpython'): + with mock.patch('os.mkdir'): + self.assertEqual(conf.GetUserCfgDir(), + 'C:\\foo\\cpython\\.idlerc') + + # Check user dir not exists and created failed should raise SystemExit + with mock.patch('os.path.join', return_value='/path/not/exists'): + with self.assertRaises(SystemExit): + with self.assertRaises(FileNotFoundError): + conf.GetUserCfgDir() + + def test_create_config_handlers(self): + conf = self.new_config(_utest=True) + + # Mock out idle_dir + idle_dir = '/home/foo' + with mock.patch.dict({'__name__': '__foo__'}): + with mock.patch('os.path.dirname', return_value=idle_dir): + conf.CreateConfigHandlers() + + # Check keys are equal + self.assertCountEqual(conf.defaultCfg.keys(), conf.config_types) + self.assertCountEqual(conf.userCfg.keys(), conf.config_types) + + # Check conf parser are correct type + for default_parser in conf.defaultCfg.values(): + self.assertIsInstance(default_parser, config.IdleConfParser) + for user_parser in conf.userCfg.values(): + self.assertIsInstance(user_parser, config.IdleUserConfParser) + + # Check config path are correct + for config_type, parser in conf.defaultCfg.items(): + self.assertEqual(parser.file, + os.path.join(idle_dir, 'config-%s.def' % config_type)) + for config_type, parser in conf.userCfg.items(): + self.assertEqual(parser.file, + os.path.join(conf.userdir, 'config-%s.cfg' % config_type)) + + def test_load_cfg_files(self): + conf = self.new_config(_utest=True) + + # Borrow test/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1') + conf.defaultCfg['foo'] = config.IdleConfParser(config_path) + conf.userCfg['foo'] = config.IdleUserConfParser(config_path) + + # Load all config from path + conf.LoadCfgFiles() + + eq = self.assertEqual + + # Check defaultCfg is loaded + eq(conf.defaultCfg['foo'].Get('Foo Bar', 'foo'), 'newbar') + eq(conf.defaultCfg['foo'].GetOptionList('Foo Bar'), ['foo']) + + # Check userCfg is loaded + eq(conf.userCfg['foo'].Get('Foo Bar', 'foo'), 'newbar') + eq(conf.userCfg['foo'].GetOptionList('Foo Bar'), ['foo']) + + def test_save_user_cfg_files(self): + conf = self.mock_config() + + with mock.patch('idlelib.config.IdleUserConfParser.Save') as m: + conf.SaveUserCfgFiles() + self.assertEqual(m.call_count, len(conf.userCfg)) + + def test_get_option(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetOption('main', 'EditorWindow', 'width'), '80') + eq(conf.GetOption('main', 'EditorWindow', 'width', type='int'), 80) + with mock.patch('idlelib.config._warn') as _warn: + eq(conf.GetOption('main', 'EditorWindow', 'font', type='int'), None) + eq(conf.GetOption('main', 'EditorWindow', 'NotExists'), None) + eq(conf.GetOption('main', 'EditorWindow', 'NotExists', default='NE'), 'NE') + eq(_warn.call_count, 4) + + def test_set_option(self): + conf = self.mock_config() + + conf.SetOption('main', 'Foo', 'bar', 'newbar') + self.assertEqual(conf.GetOption('main', 'Foo', 'bar'), 'newbar') + + def test_get_section_list(self): + conf = self.mock_config() + + self.assertCountEqual( + conf.GetSectionList('default', 'main'), + ['General', 'EditorWindow', 'Indent', 'Theme', + 'Keys', 'History', 'HelpFiles']) + self.assertCountEqual( + conf.GetSectionList('user', 'main'), + ['General', 'EditorWindow', 'Indent', 'Theme', + 'Keys', 'History', 'HelpFiles']) + + with self.assertRaises(config.InvalidConfigSet): + conf.GetSectionList('foobar', 'main') + with self.assertRaises(config.InvalidConfigType): + conf.GetSectionList('default', 'notexists') + + def test_get_highlight(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetHighlight('IDLE Classic', 'normal'), {'foreground': '#000000', + 'background': '#ffffff'}) + eq(conf.GetHighlight('IDLE Classic', 'normal', 'fg'), '#000000') + eq(conf.GetHighlight('IDLE Classic', 'normal', 'bg'), '#ffffff') + with self.assertRaises(config.InvalidFgBg): + conf.GetHighlight('IDLE Classic', 'normal', 'fb') + + # Test cursor (this background should be normal-background) + eq(conf.GetHighlight('IDLE Classic', 'cursor'), {'foreground': 'black', + 'background': '#ffffff'}) + + # Test get user themes + conf.SetOption('highlight', 'Foobar', 'normal-foreground', '#747474') + conf.SetOption('highlight', 'Foobar', 'normal-background', '#171717') + with mock.patch('idlelib.config._warn'): + eq(conf.GetHighlight('Foobar', 'normal'), {'foreground': '#747474', + 'background': '#171717'}) + + def test_get_theme_dict(self): + "XXX: NOT YET DONE" + conf = self.mock_config() + + # These two should be the same + self.assertEqual( + conf.GetThemeDict('default', 'IDLE Classic'), + conf.GetThemeDict('user', 'IDLE Classic')) + + with self.assertRaises(config.InvalidTheme): + conf.GetThemeDict('bad', 'IDLE Classic') + + def test_get_current_theme_and_keys(self): + conf = self.mock_config() + + self.assertEqual(conf.CurrentTheme(), conf.current_colors_and_keys('Theme')) + self.assertEqual(conf.CurrentKeys(), conf.current_colors_and_keys('Keys')) + + def test_current_colors_and_keys(self): + conf = self.mock_config() + + self.assertEqual(conf.current_colors_and_keys('Theme'), 'IDLE Classic') + + def test_default_keys(self): + current_platform = sys.platform + conf = self.new_config(_utest=True) + + sys.platform = 'win32' + self.assertEqual(conf.default_keys(), 'IDLE Classic Windows') + + sys.platform = 'darwin' + self.assertEqual(conf.default_keys(), 'IDLE Classic OSX') + + sys.platform = 'some-linux' + self.assertEqual(conf.default_keys(), 'IDLE Modern Unix') + + # Restore platform + sys.platform = current_platform + + def test_get_extensions(self): + conf = self.mock_config() + + # Add disable extensions + conf.SetOption('extensions', 'DISABLE', 'enable', 'False') + + eq = self.assertEqual + eq(conf.GetExtensions(), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', 'ScriptBinding', + 'ZoomHeight']) + eq(conf.GetExtensions(active_only=False), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', 'ScriptBinding', + 'ZoomHeight', 'DISABLE']) + eq(conf.GetExtensions(editor_only=True), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', 'ScriptBinding', + 'ZoomHeight']) + eq(conf.GetExtensions(shell_only=True), + ['AutoComplete', 'AutoExpand', 'CallTips', 'FormatParagraph', + 'ParenMatch', 'ZoomHeight']) + eq(conf.GetExtensions(active_only=False, editor_only=True), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', + 'ScriptBinding', 'ZoomHeight', 'DISABLE']) + eq(conf.GetExtensions(active_only=False, shell_only=True), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', 'ScriptBinding', + 'ZoomHeight', 'DISABLE']) + + # Add user extensions + conf.SetOption('extensions', 'Foobar', 'enable', 'True') + eq(conf.GetExtensions(), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', + 'ScriptBinding', 'ZoomHeight', 'Foobar']) # User extensions didn't sort + eq(conf.GetExtensions(active_only=False), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', + 'ScriptBinding', 'ZoomHeight', 'DISABLE', 'Foobar']) + + def test_remove_key_bind_names(self): + conf = self.mock_config() + + self.assertCountEqual( + conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), + ['AutoComplete', 'AutoExpand', 'CallTips', 'CodeContext', + 'FormatParagraph', 'ParenMatch', 'RstripExtension', 'ScriptBinding', + 'ZoomHeight']) + + def test_get_extn_name_for_event(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetExtnNameForEvent('force-open-completions'), 'AutoComplete') + eq(conf.GetExtnNameForEvent('expand-word'), 'AutoExpand') + eq(conf.GetExtnNameForEvent('force-open-calltip'), 'CallTips') + eq(conf.GetExtnNameForEvent('zoom-height'), 'ZoomHeight') + + def test_get_extension_keys(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetExtensionKeys('AutoComplete'), + {'<>': ['']}) + eq(conf.GetExtensionKeys('ParenMatch'), + {'<>': ['']}) + + key = [''] if sys.platform == 'darwin' else [''] + eq(conf.GetExtensionKeys('ZoomHeight'), {'<>': key}) + + def test_get_extension_bindings(self): + conf = self.mock_config() + + self.assertEqual(conf.GetExtensionBindings('NotExists'), {}) + + key = [''] if sys.platform == 'darwin' else [''] + self.assertEqual( + conf.GetExtensionBindings('ZoomHeight'), {'<>': key}) + + # Add non-configuarable bindings + conf.defaultCfg['extensions'].add_section('Foobar') + conf.defaultCfg['extensions'].add_section('Foobar_bindings') + conf.defaultCfg['extensions'].set('Foobar', 'enable', 'True') + conf.defaultCfg['extensions'].set('Foobar_bindings', 'foobar', '') + self.assertEqual(conf.GetExtensionBindings('Foobar'), {'<>': ['']}) + + def test_get_keybinding(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetKeyBinding('IDLE Modern Unix', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Unix', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Windows', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Mac', '<>'), ['']) + eq(conf.GetKeyBinding('IDLE Classic OSX', '<>'), ['']) + + # Test keybinding not exists + eq(conf.GetKeyBinding('NOT EXISTS', '<>'), []) + eq(conf.GetKeyBinding('IDLE Modern Unix', 'NOT EXISTS'), []) + + def test_get_current_keyset(self): + current_platform = sys.platform + conf = self.mock_config() + + # Ensure that platform isn't darwin + sys.platform = 'some-linux' + self.assertEqual(conf.GetCurrentKeySet(), conf.GetKeySet(conf.CurrentKeys())) + + # This should not be the same, sicne replace ') + self.assertEqual(conf.GetKeySet('IDLE Modern Unix')['<>'], '') + + def test_is_core_binding(self): + # XXX: Should move out the core keys to config file or other place + conf = self.mock_config() + + self.assertTrue(conf.IsCoreBinding('copy')) + self.assertTrue(conf.IsCoreBinding('cut')) + self.assertTrue(conf.IsCoreBinding('del-word-right')) + self.assertFalse(conf.IsCoreBinding('not-exists')) + + def test_extra_help_source_list(self): + # Test GetExtraHelpSourceList and GetAllExtraHelpSourcesList in same + # place to prevent prepare input data twice. + conf = self.mock_config() + + # Test default with no extra help source + self.assertEqual(conf.GetExtraHelpSourceList('default'), []) + self.assertEqual(conf.GetExtraHelpSourceList('user'), []) + with self.assertRaises(config.InvalidConfigSet): + self.assertEqual(conf.GetExtraHelpSourceList('bad'), []) + self.assertCountEqual( + conf.GetAllExtraHelpSourcesList(), + conf.GetExtraHelpSourceList('default') + conf.GetExtraHelpSourceList('user')) + + # Add help source to user config + conf.userCfg['main'].SetOption('HelpFiles', '4', 'Python;https://python.org') # This is bad input + conf.userCfg['main'].SetOption('HelpFiles', '3', 'Python:https://python.org') # This is bad input + conf.userCfg['main'].SetOption('HelpFiles', '2', 'Pillow;https://pillow.readthedocs.io/en/latest/') + conf.userCfg['main'].SetOption('HelpFiles', '1', 'IDLE;C:/Programs/Python36/Lib/idlelib/help.html') + self.assertEqual(conf.GetExtraHelpSourceList('user'), + [('IDLE', 'C:/Programs/Python36/Lib/idlelib/help.html', '1'), + ('Pillow', 'https://pillow.readthedocs.io/en/latest/', '2'), + ('Python', 'https://python.org', '4')]) + self.assertCountEqual( + conf.GetAllExtraHelpSourcesList(), + conf.GetExtraHelpSourceList('default') + conf.GetExtraHelpSourceList('user')) + + def test_get_font(self): + from test.support import requires + from tkinter import Tk + from tkinter.font import Font + conf = self.mock_config() + + requires('gui') + root = Tk() + root.withdraw() + + f = Font.actual(Font(name='TkFixedFont', exists=True, root=root)) + self.assertEqual( + conf.GetFont(root, 'main', 'EditorWindow'), + (f['family'], 10 if f['size'] <= 0 else f['size'], f['weight'])) + + # Cleanup root + root.destroy() + del root + + def test_get_core_keys(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetCoreKeys()['<>'], ['']) + eq(conf.GetCoreKeys()['<>'], ['', '']) + eq(conf.GetCoreKeys()['<>'], ['']) + eq(conf.GetCoreKeys('IDLE Classic Windows')['<>'], + ['', '']) + eq(conf.GetCoreKeys('IDLE Classic OSX')['<>'], ['']) + eq(conf.GetCoreKeys('IDLE Classic Unix')['<>'], + ['', '']) + eq(conf.GetCoreKeys('IDLE Modern Unix')['<>'], + ['', '']) + + class CurrentColorKeysTest(unittest.TestCase): """ Test colorkeys function with user config [Theme] and [Keys] patterns. diff --git a/Misc/NEWS.d/next/IDLE/2017-07-17-23-35-57.bpo-30917.hSiuuO.rst b/Misc/NEWS.d/next/IDLE/2017-07-17-23-35-57.bpo-30917.hSiuuO.rst new file mode 100644 index 00000000000000..c2cc9ccdd117a8 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-17-23-35-57.bpo-30917.hSiuuO.rst @@ -0,0 +1,3 @@ +Add tests for idlelib.config.IdleConf. +Increase coverage from 46% to 96%. +Patch by Louie Lu. From e0d306f682816df352b903c9b785d684ac8a87a5 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 20 Jul 2017 04:19:22 -0400 Subject: [PATCH 0576/1345] Update doc download files size estimates. (#2771) (#2772) --- Doc/tools/templates/download.html | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 3a05cb697938fd..d49ebdd14ba8a1 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -18,23 +18,23 @@

    Download Python {{ release }} Documentation

    - - + + - - + + - - + + - - + + - +
    FormatPacked as .zipPacked as .tar.bz2
    PDF (US-Letter paper size)Download (ca. 8 MB)Download (ca. 8 MB)Download (ca. 13 MB)Download (ca. 13 MB)
    PDF (A4 paper size)Download (ca. 8 MB)Download (ca. 8 MB)Download (ca. 13 MB)Download (ca. 13 MB)
    HTMLDownload (ca. 6 MB)Download (ca. 4 MB)Download (ca. 9 MB)Download (ca. 6 MB)
    Plain TextDownload (ca. 2 MB)Download (ca. 1.5 MB)Download (ca. 3 MB)Download (ca. 2 MB)
    EPUBDownload (ca. 4.5 MB)Download (ca. 5.5 MB)
    From 7f58994842d930ed5f92386579997871d2a3bb48 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 20 Jul 2017 07:14:05 -0700 Subject: [PATCH 0577/1345] [3.6] Fix typo in Turtle Docs: yingyang -> yinyang (GH-2770) (GH-2778) (cherry picked from commit fff2a21057b98732562098e3bdd65980551f0135) --- Doc/library/turtle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 31761be02b78ee..2b5d3e567d6cf2 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2368,7 +2368,7 @@ The demo scripts are: | wikipedia | a pattern from the wikipedia | :func:`clone`, | | | article on turtle graphics | :func:`undo` | +----------------+------------------------------+-----------------------+ -| yingyang | another elementary example | :func:`circle` | +| yinyang | another elementary example | :func:`circle` | +----------------+------------------------------+-----------------------+ Have fun! From 96ef06f3979023fd69ed8796d0b1d88885d3ae4d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 20 Jul 2017 17:08:48 +0200 Subject: [PATCH 0578/1345] bpo-30822: Exclude tzdata from regrtest --all (#2775) (#2777) When running the test suite using --use=all / -u all, exclude tzdata since it makes test_datetime too slow (15-20 min on some buildbots) which then times out on some buildbots. -u tzdata must now be enabled explicitly, -u tzdata or -u all,tzdata, to run all test_datetime tests. Fix also regrtest command line parser to allow passing -u extralargefile to run test_zipfile64. Travis CI: remove -tzdata. Replace -u all,-tzdata,-cpu with -u all,-cpu since tzdata is now excluded from -u all. (cherry picked from commit 5b392bbaeb9d9b1db961ecfc7315d8c8662c27f6) --- .travis.yml | 4 ++-- Lib/test/libregrtest/__init__.py | 2 +- Lib/test/libregrtest/cmdline.py | 15 ++++++++++++--- Lib/test/test_regrtest.py | 13 ++++++++++++- .../2017-07-20-14-29-54.bpo-30822.X0wREo.rst | 5 +++++ 5 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst diff --git a/.travis.yml b/.travis.yml index e7b78db48bd829..cf59d965a871d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,7 +61,7 @@ matrix: ./venv/bin/python -m pip install -U coverage script: # Skip tests that re-run the entire test suite. - - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu,-tzdata -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn after_script: # Probably should be after_success once test suite updated to run under coverage.py. # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate @@ -94,7 +94,7 @@ script: # Only run on Linux as the check only needs to be run once. - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi # `-r -w` implicitly provided through `make buildbottest`. - - make buildbottest TESTOPTS="-j4 -uall,-cpu,-tzdata" + - make buildbottest TESTOPTS="-j4 -uall,-cpu" notifications: email: false diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 7ba0e6e5356166..3427b51b60af86 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,5 +1,5 @@ # We import importlib *ASAP* in order to test #15386 import importlib -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2315cd59b4e679..4999fa734114a1 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -127,8 +127,17 @@ """ -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'tzdata') +ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +# Other resources excluded from --use=all: +# +# - extralagefile (ex: test_zipfile64): really too slow to be enabled +# "by default" +# - tzdata: while needed to validate fully test_datetime, it makes +# test_datetime too slow (15-20 min on some buildbots) and so is disabled by +# default (see bpo-30822). +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') class _ArgParser(argparse.ArgumentParser): @@ -344,7 +353,7 @@ def _parse_args(args, **kwargs): for a in ns.use: for r in a: if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES + ns.use_resources[:] = ALL_RESOURCES continue if r == 'none': del ns.use_resources[:] diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a544f880d2be01..f63ed647f80e0b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -191,15 +191,26 @@ def test_use(self): with self.subTest(opt=opt): ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.RESOURCE_NAMES) + + expected = list(libregrtest.ALL_RESOURCES) expected.remove('gui') ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') + # all + a resource not part of "all" + ns = libregrtest._parse_args([opt, 'all,tzdata']) + self.assertEqual(ns.use_resources, + list(libregrtest.ALL_RESOURCES) + ['tzdata']) + + # test another resource which is not part of "all" + ns = libregrtest._parse_args([opt, 'extralargefile']) + self.assertEqual(ns.use_resources, ['extralargefile']) + def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): diff --git a/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst b/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst new file mode 100644 index 00000000000000..53557f66feffa9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst @@ -0,0 +1,5 @@ +regrtest: Exclude tzdata from regrtest --all. When running the test suite +using --use=all / -u all, exclude tzdata since it makes test_datetime too +slow (15-20 min on some buildbots) which then times out on some buildbots. +Fix also regrtest command line parser to allow passing -u extralargefile to +run test_zipfile64. From ef4231adb7aa6b323c94e854ac3dc3bebc841115 Mon Sep 17 00:00:00 2001 From: Trey Hunner Date: Thu, 20 Jul 2017 20:10:54 -0700 Subject: [PATCH 0579/1345] bpo-30466: Add brief explanation of classes to tutorial (GH-1804) (GH-2700) --- Doc/tutorial/classes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 073444cf8b3985..95141f94d6accb 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -4,6 +4,12 @@ Classes ******* +Classes provide a means of bundling data and functionality together. Creating +a new class creates a new *type* of object, allowing new *instances* of that +type to be made. Each class instance can have attributes attached to it for +maintaining its state. Class instances can also have methods (defined by its +class) for modifying its state. + Compared with other programming languages, Python's class mechanism adds classes with a minimum of new syntax and semantics. It is a mixture of the class mechanisms found in C++ and Modula-3. Python classes provide all the standard From ac5c1e2ea311d11714f89476c86936f9388d19a4 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 21 Jul 2017 01:29:09 -0400 Subject: [PATCH 0580/1345] [3.6] bpo-28523: IDLE: Use 'color' instead of 'colour'. (GH-2787) (#2791) (cherry picked from commit a54a8f1) --- Lib/idlelib/configdialog.py | 108 +++++++++--------- .../2017-07-21-00-54-52.bpo-28523.OPcqYJ.rst | 1 + 2 files changed, 55 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-21-00-54-52.bpo-28523.OPcqYJ.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index c36f0f3f67bed0..e1c3923caed43f 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -261,7 +261,7 @@ def create_page_highlight(self): """Return frame of widgets for Highlighting tab. Tk Variables: - colour: Color of selected target. + color: Color of selected target. builtin_theme: Menu variable for built-in theme. custom_theme: Menu variable for custom theme. fg_bg_toggle: Toggle for foreground/background color. @@ -276,11 +276,11 @@ def create_page_highlight(self): Methods [attachment]: load_theme_cfg: Load current highlight colors. - get_colour: Invoke colorchooser [button_set_colour]. - set_colour_sample_binding: Call set_colour_sample [fg_bg_toggle]. + get_color: Invoke colorchooser [button_set_color]. + set_color_sample_binding: Call set_color_sample [fg_bg_toggle]. set_highlight_target: set fg_bg_toggle, set_color_sample(). - set_colour_sample: Set frame background to target. - on_new_colour_set: Set new color and add option. + set_color_sample: Set frame background to target. + on_new_color_set: Set new color and add option. paint_theme_sample: Recolor sample. get_new_theme_name: Get from popup. create_new_theme: Combine theme with changes and save. @@ -293,8 +293,8 @@ def create_page_highlight(self): frame frame_custom: LabelFrame (*)text_highlight_sample: Text - (*)frame_colour_set: Frame - button_set_colour: Button + (*)frame_color_set: Frame + button_set_color: Button (*)opt_menu_highlight_target: DynOptionMenu - highlight_target frame_fg_bg_toggle: Frame (*)radio_fg: Radiobutton - fg_bg_toggle @@ -329,7 +329,7 @@ def create_page_highlight(self): self.builtin_theme = StringVar(parent) self.custom_theme = StringVar(parent) self.fg_bg_toggle = BooleanVar(parent) - self.colour = StringVar(parent) + self.color = StringVar(parent) self.is_builtin_theme = BooleanVar(parent) self.highlight_target = StringVar(parent) @@ -374,20 +374,20 @@ def tem(event, elem=element): text.tag_bind( self.theme_elements[element][0], '', tem) text.config(state=DISABLED) - self.frame_colour_set = Frame(frame_custom, relief=SOLID, borderwidth=1) + self.frame_color_set = Frame(frame_custom, relief=SOLID, borderwidth=1) frame_fg_bg_toggle = Frame(frame_custom) - button_set_colour = Button( - self.frame_colour_set, text='Choose Colour for :', - command=self.get_colour, highlightthickness=0) + button_set_color = Button( + self.frame_color_set, text='Choose Color for :', + command=self.get_color, highlightthickness=0) self.opt_menu_highlight_target = DynOptionMenu( - self.frame_colour_set, self.highlight_target, None, + self.frame_color_set, self.highlight_target, None, highlightthickness=0) #, command=self.set_highlight_targetBinding self.radio_fg = Radiobutton( frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=1, - text='Foreground', command=self.set_colour_sample_binding) + text='Foreground', command=self.set_color_sample_binding) self.radio_bg=Radiobutton( frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=0, - text='Background', command=self.set_colour_sample_binding) + text='Background', command=self.set_color_sample_binding) self.fg_bg_toggle.set(1) button_save_custom_theme = Button( frame_custom, text='Save as New Custom Theme', @@ -414,11 +414,11 @@ def tem(event, elem=element): frame_custom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_theme.pack(side=LEFT, padx=5, pady=5, fill=Y) #frame_custom - self.frame_colour_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) + self.frame_color_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) frame_fg_bg_toggle.pack(side=TOP, padx=5, pady=0) self.text_highlight_sample.pack( side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - button_set_colour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) + button_set_color.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) self.opt_menu_highlight_target.pack( side=TOP, expand=TRUE, fill=X, padx=8, pady=3) self.radio_fg.pack(side=LEFT, anchor=E) @@ -694,7 +694,7 @@ def attach_var_callbacks(self): self.font_name.trace_add('write', self.var_changed_font) self.font_bold.trace_add('write', self.var_changed_font) self.space_num.trace_add('write', self.var_changed_space_num) - self.colour.trace_add('write', self.var_changed_colour) + self.color.trace_add('write', self.var_changed_color) self.builtin_theme.trace_add('write', self.var_changed_builtin_theme) self.custom_theme.trace_add('write', self.var_changed_custom_theme) self.is_builtin_theme.trace_add('write', self.var_changed_is_builtin_theme) @@ -712,7 +712,7 @@ def remove_var_callbacks(self): "Remove callbacks to prevent memory leaks." for var in ( self.font_size, self.font_name, self.font_bold, - self.space_num, self.colour, self.builtin_theme, + self.space_num, self.color, self.builtin_theme, self.custom_theme, self.is_builtin_theme, self.highlight_target, self.keybinding, self.builtin_keys, self.custom_keys, self.are_keys_builtin, self.win_width, self.win_height, @@ -738,9 +738,9 @@ def var_changed_space_num(self, *params): value = self.space_num.get() changes.add_option('main', 'Indent', 'num-spaces', value) - def var_changed_colour(self, *params): + def var_changed_color(self, *params): "Process change to color choice." - self.on_new_colour_set() + self.on_new_color_set() def var_changed_builtin_theme(self, *params): """Process new builtin theme selection. @@ -1099,7 +1099,7 @@ def delete_custom_theme(self): self.activate_config_changes() self.set_theme_type() - def get_colour(self): + def get_color(self): """Handle button to select a new color for the target tag. If a new color is selected while using a builtin theme, a @@ -1107,23 +1107,23 @@ def get_colour(self): Attributes accessed: highlight_target - frame_colour_set + frame_color_set is_builtin_theme Attributes updated: - colour + color Methods: get_new_theme_name create_new_theme """ target = self.highlight_target.get() - prev_colour = self.frame_colour_set.cget('bg') - rgbTuplet, colour_string = tkColorChooser.askcolor( - parent=self, title='Pick new colour for : '+target, - initialcolor=prev_colour) - if colour_string and (colour_string != prev_colour): - # User didn't cancel and they chose a new colour. + prev_color = self.frame_color_set.cget('bg') + rgbTuplet, color_string = tkColorChooser.askcolor( + parent=self, title='Pick new color for : '+target, + initialcolor=prev_color) + if color_string and (color_string != prev_color): + # User didn't cancel and they chose a new color. if self.is_builtin_theme.get(): # Current theme is a built-in. message = ('Your changes will be saved as a new Custom Theme. ' 'Enter a name for your new Custom Theme below.') @@ -1132,20 +1132,20 @@ def get_colour(self): return else: # Create new custom theme based on previously active theme. self.create_new_theme(new_theme) - self.colour.set(colour_string) + self.color.set(color_string) else: # Current theme is user defined. - self.colour.set(colour_string) + self.color.set(color_string) - def on_new_colour_set(self): + def on_new_color_set(self): "Display sample of new color selection on the dialog." - new_colour=self.colour.get() - self.frame_colour_set.config(bg=new_colour) # Set sample. + new_color=self.color.get() + self.frame_color_set.config(bg=new_color) # Set sample. plane ='foreground' if self.fg_bg_toggle.get() else 'background' sample_element = self.theme_elements[self.highlight_target.get()][0] - self.text_highlight_sample.tag_config(sample_element, **{plane:new_colour}) + self.text_highlight_sample.tag_config(sample_element, **{plane:new_color}) theme = self.custom_theme.get() theme_element = sample_element + '-' + plane - changes.add_option('highlight', theme, theme_element, new_colour) + changes.add_option('highlight', theme, theme_element, new_color) def get_new_theme_name(self, message): "Return name of new theme from query popup." @@ -1258,7 +1258,7 @@ def set_highlight_target(self): fg_bg_toggle Methods: - set_colour_sample + set_color_sample Called from: var_changed_highlight_target @@ -1272,17 +1272,17 @@ def set_highlight_target(self): self.radio_fg.config(state=NORMAL) self.radio_bg.config(state=NORMAL) self.fg_bg_toggle.set(1) - self.set_colour_sample() + self.set_color_sample() - def set_colour_sample_binding(self, *args): + def set_color_sample_binding(self, *args): """Change color sample based on foreground/background toggle. Methods: - set_colour_sample + set_color_sample """ - self.set_colour_sample() + self.set_color_sample() - def set_colour_sample(self): + def set_color_sample(self): """Set the color of the frame background to reflect the selected target. Instance variables accessed: @@ -1292,13 +1292,13 @@ def set_colour_sample(self): text_highlight_sample Attributes updated: - frame_colour_set + frame_color_set """ - # Set the colour sample area. + # Set the color sample area. tag = self.theme_elements[self.highlight_target.get()][0] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - colour = self.text_highlight_sample.tag_cget(tag, plane) - self.frame_colour_set.config(bg=colour) + color = self.text_highlight_sample.tag_cget(tag, plane) + self.frame_color_set.config(bg=color) def paint_theme_sample(self): """Apply the theme colors to each element tag in the sample text. @@ -1313,7 +1313,7 @@ def paint_theme_sample(self): text_highlight_sample: Set the tag elements to the theme. Methods: - set_colour_sample + set_color_sample Called from: var_changed_builtin_theme @@ -1326,19 +1326,19 @@ def paint_theme_sample(self): theme = self.custom_theme.get() for element_title in self.theme_elements: element = self.theme_elements[element_title][0] - colours = idleConf.GetHighlight(theme, element) + colors = idleConf.GetHighlight(theme, element) if element == 'cursor': # Cursor sample needs special painting. - colours['background'] = idleConf.GetHighlight( + colors['background'] = idleConf.GetHighlight( theme, 'normal', fgBg='bg') # Handle any unsaved changes to this theme. if theme in changes['highlight']: theme_dict = changes['highlight'][theme] if element + '-foreground' in theme_dict: - colours['foreground'] = theme_dict[element + '-foreground'] + colors['foreground'] = theme_dict[element + '-foreground'] if element + '-background' in theme_dict: - colours['background'] = theme_dict[element + '-background'] - self.text_highlight_sample.tag_config(element, **colours) - self.set_colour_sample() + colors['background'] = theme_dict[element + '-background'] + self.text_highlight_sample.tag_config(element, **colors) + self.set_color_sample() def help_source_selected(self, event): "Handle event for selecting additional help." diff --git a/Misc/NEWS.d/next/IDLE/2017-07-21-00-54-52.bpo-28523.OPcqYJ.rst b/Misc/NEWS.d/next/IDLE/2017-07-21-00-54-52.bpo-28523.OPcqYJ.rst new file mode 100644 index 00000000000000..d7c5f54c7c322e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-21-00-54-52.bpo-28523.OPcqYJ.rst @@ -0,0 +1 @@ +IDLE: replace 'colour' with 'color' in configdialog. From df5837b5a916aac645087ae02c44e88ff77024ef Mon Sep 17 00:00:00 2001 From: Nicholas Date: Thu, 20 Jul 2017 23:44:43 -0700 Subject: [PATCH 0581/1345] [3.6] bpo-30964: Mention ensurepip in package installation docs (GH-2797) Adds a new 'Pip not installed' section that covers running `ensurepip` manually, and also references the relevant section of the Python Packaging User Guide. (cherry picked from commit b3527bfefd7a0188d43a2d7515ac6addd97a8202) --- Doc/installing/index.rst | 11 +++++++++++ Misc/ACKS | 1 + 2 files changed, 12 insertions(+) diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index b22465df2910eb..09bb8251c35d58 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -211,6 +211,17 @@ On such systems, it is often better to use a virtual environment or a per-user installation when installing packages with ``pip``. +Pip not installed +----------------- + +It is possible that ``pip`` does not get installed by default. One potential fix is:: + + python -m ensurepip --default-pip + +There are also additional resources for `installing pip. +`__ + + Installing binary extensions ---------------------------- diff --git a/Misc/ACKS b/Misc/ACKS index 940e07e6d6e667..f4ccdb88cdc962 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -802,6 +802,7 @@ Kim Knapp Lenny Kneler Pat Knight Jeff Knupp +Nicholas Kobald Kubilay Kocak Greg Kochanski Manvisha Kodali From bb323b261d0f41ad6998d3a16c41b29b3bb8e982 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Jul 2017 13:28:55 +0200 Subject: [PATCH 0582/1345] bpo-30891: Fix again importlib _find_and_load() (#2665) (#2801) Use sys.modules.get() in the "with _ModuleLockManager(name):" block to protect the dictionary key with the module lock and use an atomic get to prevent race condition. Remove also _bootstrap._POPULATE since it was unused (_bootstrap_external now has its own _POPULATE object), add a new _SENTINEL object instead. (cherry picked from commit e72b1359f81d1dd42bd8a5c5cc2b3928b74f8023) --- Lib/importlib/_bootstrap.py | 10 +- Python/importlib.h | 683 ++++++++++++++++++------------------ 2 files changed, 347 insertions(+), 346 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index bf13e2aed017b7..92cf14f9504a94 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -446,9 +446,6 @@ def spec_from_loader(name, loader, *, origin=None, is_package=None): return ModuleSpec(name, loader, origin=origin, is_package=is_package) -_POPULATE = object() - - def _spec_from_module(module, loader=None, origin=None): # This function is meant for use in _setup(). try: @@ -953,13 +950,16 @@ def _find_and_load_unlocked(name, import_): return module +_NEEDS_LOADING = object() + + def _find_and_load(name, import_): """Find and load the module.""" with _ModuleLockManager(name): - if name not in sys.modules: + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: return _find_and_load_unlocked(name, import_) - module = sys.modules[name] if module is None: message = ('import of {} halted; ' 'None in sys.modules'.format(name)) diff --git a/Python/importlib.h b/Python/importlib.h index 30c554241fe1d2..eef1fababe967e 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -14,18 +14,18 @@ const unsigned char _Py_M__importlib[] = { 90,17,100,30,100,31,132,0,90,18,71,0,100,32,100,33, 132,0,100,33,131,2,90,19,71,0,100,34,100,35,132,0, 100,35,131,2,90,20,100,1,100,1,100,36,156,2,100,37, - 100,38,132,2,90,21,101,22,131,0,90,23,100,92,100,39, - 100,40,132,1,90,24,100,41,100,42,156,1,100,43,100,44, - 132,2,90,25,100,45,100,46,132,0,90,26,100,47,100,48, - 132,0,90,27,100,49,100,50,132,0,90,28,100,51,100,52, - 132,0,90,29,100,53,100,54,132,0,90,30,100,55,100,56, - 132,0,90,31,71,0,100,57,100,58,132,0,100,58,131,2, - 90,32,71,0,100,59,100,60,132,0,100,60,131,2,90,33, - 71,0,100,61,100,62,132,0,100,62,131,2,90,34,100,63, - 100,64,132,0,90,35,100,65,100,66,132,0,90,36,100,93, - 100,67,100,68,132,1,90,37,100,69,100,70,132,0,90,38, - 100,71,90,39,101,39,100,72,23,0,90,40,100,73,100,74, - 132,0,90,41,100,75,100,76,132,0,90,42,100,94,100,78, + 100,38,132,2,90,21,100,92,100,39,100,40,132,1,90,22, + 100,41,100,42,156,1,100,43,100,44,132,2,90,23,100,45, + 100,46,132,0,90,24,100,47,100,48,132,0,90,25,100,49, + 100,50,132,0,90,26,100,51,100,52,132,0,90,27,100,53, + 100,54,132,0,90,28,100,55,100,56,132,0,90,29,71,0, + 100,57,100,58,132,0,100,58,131,2,90,30,71,0,100,59, + 100,60,132,0,100,60,131,2,90,31,71,0,100,61,100,62, + 132,0,100,62,131,2,90,32,100,63,100,64,132,0,90,33, + 100,65,100,66,132,0,90,34,100,93,100,67,100,68,132,1, + 90,35,100,69,100,70,132,0,90,36,100,71,90,37,101,37, + 100,72,23,0,90,38,100,73,100,74,132,0,90,39,101,40, + 131,0,90,41,100,75,100,76,132,0,90,42,100,94,100,78, 100,79,132,1,90,43,100,80,100,81,132,0,90,44,100,82, 100,83,132,0,90,45,100,1,100,1,102,0,100,77,102,4, 100,84,100,85,132,1,90,46,100,86,100,87,132,0,90,47, @@ -850,7 +850,7 @@ const unsigned char _Py_M__importlib[] = { 0,114,15,0,0,0,90,8,108,111,99,97,116,105,111,110, 114,113,0,0,0,114,107,0,0,0,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,17,95,115,112,101,99, - 95,102,114,111,109,95,109,111,100,117,108,101,196,1,0,0, + 95,102,114,111,109,95,109,111,100,117,108,101,193,1,0,0, 115,72,0,0,0,0,2,2,1,10,1,14,1,6,2,8, 1,4,2,6,1,8,1,2,1,10,1,14,2,6,1,2, 1,10,1,14,1,10,1,8,1,8,1,2,1,10,1,14, @@ -902,7 +902,7 @@ const unsigned char _Py_M__importlib[] = { 114,130,0,0,0,114,94,0,0,0,114,132,0,0,0,114, 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,18, 95,105,110,105,116,95,109,111,100,117,108,101,95,97,116,116, - 114,115,241,1,0,0,115,92,0,0,0,0,4,20,1,2, + 114,115,238,1,0,0,115,92,0,0,0,0,4,20,1,2, 1,12,1,14,1,6,2,20,1,6,1,8,2,10,1,8, 1,4,1,6,2,10,1,8,1,2,1,10,1,14,1,6, 2,20,1,2,1,12,1,14,1,6,2,2,1,10,1,14, @@ -929,7 +929,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,15,0,0,0,114,134,0,0,0,41,2,114, 83,0,0,0,114,84,0,0,0,114,10,0,0,0,114,10, 0,0,0,114,11,0,0,0,218,16,109,111,100,117,108,101, - 95,102,114,111,109,95,115,112,101,99,45,2,0,0,115,18, + 95,102,114,111,109,95,115,112,101,99,42,2,0,0,115,18, 0,0,0,0,3,4,1,12,3,14,1,12,1,8,2,8, 1,10,1,10,1,114,137,0,0,0,99,1,0,0,0,0, 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, @@ -951,7 +951,7 @@ const unsigned char _Py_M__importlib[] = { 41,5,114,15,0,0,0,114,104,0,0,0,114,94,0,0, 0,114,38,0,0,0,114,114,0,0,0,41,2,114,83,0, 0,0,114,15,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,92,0,0,0,62,2,0,0,115, + 0,114,11,0,0,0,114,92,0,0,0,59,2,0,0,115, 16,0,0,0,0,3,20,1,10,1,10,1,10,2,16,2, 6,1,14,2,114,92,0,0,0,99,2,0,0,0,0,0, 0,0,4,0,0,0,12,0,0,0,67,0,0,0,115,178, @@ -982,7 +982,7 @@ const unsigned char _Py_M__importlib[] = { 111,97,100,95,109,111,100,117,108,101,114,136,0,0,0,41, 4,114,83,0,0,0,114,84,0,0,0,114,15,0,0,0, 218,3,109,115,103,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,114,81,0,0,0,79,2,0,0,115,30,0, + 11,0,0,0,114,81,0,0,0,76,2,0,0,115,30,0, 0,0,0,2,6,1,10,1,16,1,10,1,12,1,10,1, 10,1,14,2,14,1,4,1,14,1,12,4,14,2,22,1, 114,81,0,0,0,99,1,0,0,0,0,0,0,0,2,0, @@ -1009,7 +1009,7 @@ const unsigned char _Py_M__importlib[] = { 0,41,2,114,83,0,0,0,114,84,0,0,0,114,10,0, 0,0,114,10,0,0,0,114,11,0,0,0,218,25,95,108, 111,97,100,95,98,97,99,107,119,97,114,100,95,99,111,109, - 112,97,116,105,98,108,101,103,2,0,0,115,40,0,0,0, + 112,97,116,105,98,108,101,100,2,0,0,115,40,0,0,0, 0,4,14,2,12,1,16,1,2,1,12,1,14,1,6,1, 16,1,2,4,8,1,10,1,22,1,14,1,6,1,16,1, 2,1,10,1,14,1,6,1,114,140,0,0,0,99,1,0, @@ -1030,7 +1030,7 @@ const unsigned char _Py_M__importlib[] = { 114,80,0,0,0,41,2,114,83,0,0,0,114,84,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,14,95,108,111,97,100,95,117,110,108,111,99,107,101,100, - 132,2,0,0,115,20,0,0,0,0,2,10,2,12,1,8, + 129,2,0,0,115,20,0,0,0,0,2,10,2,12,1,8, 2,8,1,10,1,10,1,10,1,16,3,22,5,114,141,0, 0,0,99,1,0,0,0,0,0,0,0,1,0,0,0,9, 0,0,0,67,0,0,0,115,30,0,0,0,116,0,124,0, @@ -1050,7 +1050,7 @@ const unsigned char _Py_M__importlib[] = { 98,98,101,114,101,100,46,10,10,32,32,32,32,78,41,3, 114,42,0,0,0,114,15,0,0,0,114,141,0,0,0,41, 1,114,83,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,82,0,0,0,155,2,0,0,115,4, + 114,11,0,0,0,114,82,0,0,0,152,2,0,0,115,4, 0,0,0,0,9,12,1,114,82,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,4,0,0,0,64,0,0, 0,115,136,0,0,0,101,0,90,1,100,0,90,2,100,1, @@ -1086,7 +1086,7 @@ const unsigned char _Py_M__importlib[] = { 98,117,105,108,116,45,105,110,41,62,41,2,114,38,0,0, 0,114,1,0,0,0,41,1,114,84,0,0,0,114,10,0, 0,0,114,10,0,0,0,114,11,0,0,0,114,87,0,0, - 0,179,2,0,0,115,2,0,0,0,0,7,122,27,66,117, + 0,176,2,0,0,115,2,0,0,0,0,7,122,27,66,117, 105,108,116,105,110,73,109,112,111,114,116,101,114,46,109,111, 100,117,108,101,95,114,101,112,114,78,99,4,0,0,0,0, 0,0,0,4,0,0,0,5,0,0,0,67,0,0,0,115, @@ -1099,7 +1099,7 @@ const unsigned char _Py_M__importlib[] = { 99,108,115,114,72,0,0,0,218,4,112,97,116,104,218,6, 116,97,114,103,101,116,114,10,0,0,0,114,10,0,0,0, 114,11,0,0,0,218,9,102,105,110,100,95,115,112,101,99, - 188,2,0,0,115,10,0,0,0,0,2,8,1,4,1,10, + 185,2,0,0,115,10,0,0,0,0,2,8,1,4,1,10, 1,14,2,122,25,66,117,105,108,116,105,110,73,109,112,111, 114,116,101,114,46,102,105,110,100,95,115,112,101,99,99,3, 0,0,0,0,0,0,0,4,0,0,0,3,0,0,0,67, @@ -1120,7 +1120,7 @@ const unsigned char _Py_M__importlib[] = { 0,114,94,0,0,0,41,4,114,143,0,0,0,114,72,0, 0,0,114,144,0,0,0,114,83,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,218,11,102,105,110, - 100,95,109,111,100,117,108,101,197,2,0,0,115,4,0,0, + 100,95,109,111,100,117,108,101,194,2,0,0,115,4,0,0, 0,0,9,12,1,122,27,66,117,105,108,116,105,110,73,109, 112,111,114,116,101,114,46,102,105,110,100,95,109,111,100,117, 108,101,99,2,0,0,0,0,0,0,0,2,0,0,0,4, @@ -1137,7 +1137,7 @@ const unsigned char _Py_M__importlib[] = { 0,90,14,99,114,101,97,116,101,95,98,117,105,108,116,105, 110,41,2,114,26,0,0,0,114,83,0,0,0,114,10,0, 0,0,114,10,0,0,0,114,11,0,0,0,114,135,0,0, - 0,209,2,0,0,115,8,0,0,0,0,3,12,1,12,1, + 0,206,2,0,0,115,8,0,0,0,0,3,12,1,12,1, 10,1,122,29,66,117,105,108,116,105,110,73,109,112,111,114, 116,101,114,46,99,114,101,97,116,101,95,109,111,100,117,108, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,3,0, @@ -1148,7 +1148,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,90,12,101,120,101,99,95,98,117,105,108,116,105,110, 41,2,114,26,0,0,0,114,84,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,114,136,0,0,0, - 217,2,0,0,115,2,0,0,0,0,3,122,27,66,117,105, + 214,2,0,0,115,2,0,0,0,0,3,122,27,66,117,105, 108,116,105,110,73,109,112,111,114,116,101,114,46,101,120,101, 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, @@ -1158,7 +1158,7 @@ const unsigned char _Py_M__importlib[] = { 32,104,97,118,101,32,99,111,100,101,32,111,98,106,101,99, 116,115,46,78,114,10,0,0,0,41,2,114,143,0,0,0, 114,72,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,8,103,101,116,95,99,111,100,101,222,2, + 11,0,0,0,218,8,103,101,116,95,99,111,100,101,219,2, 0,0,115,2,0,0,0,0,4,122,24,66,117,105,108,116, 105,110,73,109,112,111,114,116,101,114,46,103,101,116,95,99, 111,100,101,99,2,0,0,0,0,0,0,0,2,0,0,0, @@ -1169,7 +1169,7 @@ const unsigned char _Py_M__importlib[] = { 32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,10, 0,0,0,41,2,114,143,0,0,0,114,72,0,0,0,114, 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,10, - 103,101,116,95,115,111,117,114,99,101,228,2,0,0,115,2, + 103,101,116,95,115,111,117,114,99,101,225,2,0,0,115,2, 0,0,0,0,4,122,26,66,117,105,108,116,105,110,73,109, 112,111,114,116,101,114,46,103,101,116,95,115,111,117,114,99, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,1,0, @@ -1179,7 +1179,7 @@ const unsigned char _Py_M__importlib[] = { 108,101,115,32,97,114,101,32,110,101,118,101,114,32,112,97, 99,107,97,103,101,115,46,70,114,10,0,0,0,41,2,114, 143,0,0,0,114,72,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,106,0,0,0,234,2,0, + 0,0,0,114,11,0,0,0,114,106,0,0,0,231,2,0, 0,115,2,0,0,0,0,4,122,26,66,117,105,108,116,105, 110,73,109,112,111,114,116,101,114,46,105,115,95,112,97,99, 107,97,103,101,41,2,78,78,41,1,78,41,17,114,1,0, @@ -1190,7 +1190,7 @@ const unsigned char _Py_M__importlib[] = { 114,136,0,0,0,114,75,0,0,0,114,148,0,0,0,114, 149,0,0,0,114,106,0,0,0,114,85,0,0,0,114,138, 0,0,0,114,10,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,114,142,0,0,0,170,2,0,0, + 0,0,114,11,0,0,0,114,142,0,0,0,167,2,0,0, 115,30,0,0,0,8,7,4,2,12,9,2,1,12,8,2, 1,12,11,12,8,12,5,2,1,14,5,2,1,14,5,2, 1,14,5,114,142,0,0,0,99,0,0,0,0,0,0,0, @@ -1227,7 +1227,7 @@ const unsigned char _Py_M__importlib[] = { 109,111,100,117,108,101,32,123,33,114,125,32,40,102,114,111, 122,101,110,41,62,41,2,114,38,0,0,0,114,1,0,0, 0,41,1,218,1,109,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,87,0,0,0,252,2,0,0,115,2, + 114,11,0,0,0,114,87,0,0,0,249,2,0,0,115,2, 0,0,0,0,7,122,26,70,114,111,122,101,110,73,109,112, 111,114,116,101,114,46,109,111,100,117,108,101,95,114,101,112, 114,78,99,4,0,0,0,0,0,0,0,4,0,0,0,5, @@ -1238,7 +1238,7 @@ const unsigned char _Py_M__importlib[] = { 114,52,0,0,0,114,76,0,0,0,114,79,0,0,0,41, 4,114,143,0,0,0,114,72,0,0,0,114,144,0,0,0, 114,145,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,114,146,0,0,0,5,3,0,0,115,6,0, + 11,0,0,0,114,146,0,0,0,2,3,0,0,115,6,0, 0,0,0,2,10,1,14,2,122,24,70,114,111,122,101,110, 73,109,112,111,114,116,101,114,46,102,105,110,100,95,115,112, 101,99,99,3,0,0,0,0,0,0,0,3,0,0,0,2, @@ -1253,7 +1253,7 @@ const unsigned char _Py_M__importlib[] = { 41,2,114,52,0,0,0,114,76,0,0,0,41,3,114,143, 0,0,0,114,72,0,0,0,114,144,0,0,0,114,10,0, 0,0,114,10,0,0,0,114,11,0,0,0,114,147,0,0, - 0,12,3,0,0,115,2,0,0,0,0,7,122,26,70,114, + 0,9,3,0,0,115,2,0,0,0,0,7,122,26,70,114, 111,122,101,110,73,109,112,111,114,116,101,114,46,102,105,110, 100,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, @@ -1262,7 +1262,7 @@ const unsigned char _Py_M__importlib[] = { 102,111,114,32,109,111,100,117,108,101,32,99,114,101,97,116, 105,111,110,46,78,114,10,0,0,0,41,2,114,143,0,0, 0,114,83,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,135,0,0,0,21,3,0,0,115,0, + 114,11,0,0,0,114,135,0,0,0,18,3,0,0,115,0, 0,0,0,122,28,70,114,111,122,101,110,73,109,112,111,114, 116,101,114,46,99,114,101,97,116,101,95,109,111,100,117,108, 101,99,1,0,0,0,0,0,0,0,3,0,0,0,4,0, @@ -1279,7 +1279,7 @@ const unsigned char _Py_M__importlib[] = { 122,101,110,95,111,98,106,101,99,116,218,4,101,120,101,99, 114,7,0,0,0,41,3,114,84,0,0,0,114,15,0,0, 0,218,4,99,111,100,101,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,136,0,0,0,25,3,0,0,115, + 0,114,11,0,0,0,114,136,0,0,0,22,3,0,0,115, 12,0,0,0,0,2,8,1,10,1,10,1,8,1,12,1, 122,26,70,114,111,122,101,110,73,109,112,111,114,116,101,114, 46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,0, @@ -1293,7 +1293,7 @@ const unsigned char _Py_M__importlib[] = { 32,105,110,115,116,101,97,100,46,10,10,32,32,32,32,32, 32,32,32,41,1,114,85,0,0,0,41,2,114,143,0,0, 0,114,72,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,138,0,0,0,34,3,0,0,115,2, + 114,11,0,0,0,114,138,0,0,0,31,3,0,0,115,2, 0,0,0,0,7,122,26,70,114,111,122,101,110,73,109,112, 111,114,116,101,114,46,108,111,97,100,95,109,111,100,117,108, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, @@ -1304,7 +1304,7 @@ const unsigned char _Py_M__importlib[] = { 111,100,117,108,101,46,41,2,114,52,0,0,0,114,154,0, 0,0,41,2,114,143,0,0,0,114,72,0,0,0,114,10, 0,0,0,114,10,0,0,0,114,11,0,0,0,114,148,0, - 0,0,43,3,0,0,115,2,0,0,0,0,4,122,23,70, + 0,0,40,3,0,0,115,2,0,0,0,0,4,122,23,70, 114,111,122,101,110,73,109,112,111,114,116,101,114,46,103,101, 116,95,99,111,100,101,99,2,0,0,0,0,0,0,0,2, 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, @@ -1314,7 +1314,7 @@ const unsigned char _Py_M__importlib[] = { 101,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 10,0,0,0,41,2,114,143,0,0,0,114,72,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 149,0,0,0,49,3,0,0,115,2,0,0,0,0,4,122, + 149,0,0,0,46,3,0,0,115,2,0,0,0,0,4,122, 25,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, @@ -1325,7 +1325,7 @@ const unsigned char _Py_M__importlib[] = { 41,2,114,52,0,0,0,90,17,105,115,95,102,114,111,122, 101,110,95,112,97,99,107,97,103,101,41,2,114,143,0,0, 0,114,72,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,106,0,0,0,55,3,0,0,115,2, + 114,11,0,0,0,114,106,0,0,0,52,3,0,0,115,2, 0,0,0,0,4,122,25,70,114,111,122,101,110,73,109,112, 111,114,116,101,114,46,105,115,95,112,97,99,107,97,103,101, 41,2,78,78,41,1,78,41,16,114,1,0,0,0,114,0, @@ -1335,7 +1335,7 @@ const unsigned char _Py_M__importlib[] = { 114,138,0,0,0,114,78,0,0,0,114,148,0,0,0,114, 149,0,0,0,114,106,0,0,0,114,10,0,0,0,114,10, 0,0,0,114,10,0,0,0,114,11,0,0,0,114,152,0, - 0,0,243,2,0,0,115,30,0,0,0,8,7,4,2,12, + 0,0,240,2,0,0,115,30,0,0,0,8,7,4,2,12, 9,2,1,12,6,2,1,12,8,12,4,12,9,12,9,2, 1,14,5,2,1,14,5,2,1,114,152,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, @@ -1351,7 +1351,7 @@ const unsigned char _Py_M__importlib[] = { 113,117,105,114,101,32,116,104,101,32,105,109,112,111,114,116, 32,108,111,99,107,46,78,41,2,114,52,0,0,0,114,53, 0,0,0,41,1,114,26,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,46,0,0,0,68,3, + 10,0,0,0,114,11,0,0,0,114,46,0,0,0,65,3, 0,0,115,2,0,0,0,0,2,122,28,95,73,109,112,111, 114,116,76,111,99,107,67,111,110,116,101,120,116,46,95,95, 101,110,116,101,114,95,95,99,4,0,0,0,0,0,0,0, @@ -1365,13 +1365,13 @@ const unsigned char _Py_M__importlib[] = { 0,90,8,101,120,99,95,116,121,112,101,90,9,101,120,99, 95,118,97,108,117,101,90,13,101,120,99,95,116,114,97,99, 101,98,97,99,107,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,114,48,0,0,0,72,3,0,0,115,2,0, + 11,0,0,0,114,48,0,0,0,69,3,0,0,115,2,0, 0,0,0,2,122,27,95,73,109,112,111,114,116,76,111,99, 107,67,111,110,116,101,120,116,46,95,95,101,120,105,116,95, 95,78,41,6,114,1,0,0,0,114,0,0,0,0,114,2, 0,0,0,114,3,0,0,0,114,46,0,0,0,114,48,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,157,0,0,0,64,3,0,0,115, + 0,114,11,0,0,0,114,157,0,0,0,61,3,0,0,115, 6,0,0,0,8,2,4,2,8,4,114,157,0,0,0,99, 3,0,0,0,0,0,0,0,5,0,0,0,4,0,0,0, 67,0,0,0,115,64,0,0,0,124,1,106,0,100,1,124, @@ -1392,7 +1392,7 @@ const unsigned char _Py_M__importlib[] = { 218,7,112,97,99,107,97,103,101,218,5,108,101,118,101,108, 90,4,98,105,116,115,90,4,98,97,115,101,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,218,13,95,114,101, - 115,111,108,118,101,95,110,97,109,101,77,3,0,0,115,10, + 115,111,108,118,101,95,110,97,109,101,74,3,0,0,115,10, 0,0,0,0,2,16,1,12,1,8,1,8,1,114,163,0, 0,0,99,3,0,0,0,0,0,0,0,4,0,0,0,3, 0,0,0,67,0,0,0,115,34,0,0,0,124,0,106,0, @@ -1402,7 +1402,7 @@ const unsigned char _Py_M__importlib[] = { 6,102,105,110,100,101,114,114,15,0,0,0,114,144,0,0, 0,114,94,0,0,0,114,10,0,0,0,114,10,0,0,0, 114,11,0,0,0,218,17,95,102,105,110,100,95,115,112,101, - 99,95,108,101,103,97,99,121,86,3,0,0,115,8,0,0, + 99,95,108,101,103,97,99,121,83,3,0,0,115,8,0,0, 0,0,3,12,1,8,1,4,1,114,165,0,0,0,99,3, 0,0,0,0,0,0,0,10,0,0,0,27,0,0,0,67, 0,0,0,115,242,0,0,0,116,0,106,1,125,3,124,3, @@ -1437,7 +1437,7 @@ const unsigned char _Py_M__importlib[] = { 105,115,95,114,101,108,111,97,100,114,164,0,0,0,114,146, 0,0,0,114,83,0,0,0,114,84,0,0,0,114,90,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,10,95,102,105,110,100,95,115,112,101,99,95,3,0, + 0,218,10,95,102,105,110,100,95,115,112,101,99,92,3,0, 0,115,54,0,0,0,0,2,6,1,8,2,8,3,4,1, 12,5,10,1,10,1,8,1,2,1,10,1,14,1,12,1, 8,1,8,2,22,1,8,2,16,1,10,1,2,1,10,1, @@ -1469,7 +1469,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,41,3,114,15,0,0,0,114,161,0,0,0,114,162, 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, 0,0,218,13,95,115,97,110,105,116,121,95,99,104,101,99, - 107,142,3,0,0,115,22,0,0,0,0,2,10,1,18,1, + 107,139,3,0,0,115,22,0,0,0,0,2,10,1,18,1, 8,1,8,1,8,1,10,1,10,1,4,1,8,2,14,1, 114,174,0,0,0,122,16,78,111,32,109,111,100,117,108,101, 32,110,97,109,101,100,32,122,4,123,33,114,125,99,2,0, @@ -1502,303 +1502,304 @@ const unsigned char _Py_M__importlib[] = { 101,114,139,0,0,0,114,83,0,0,0,114,84,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, 23,95,102,105,110,100,95,97,110,100,95,108,111,97,100,95, - 117,110,108,111,99,107,101,100,161,3,0,0,115,42,0,0, + 117,110,108,111,99,107,101,100,158,3,0,0,115,42,0,0, 0,0,1,4,1,14,1,4,1,10,1,10,2,10,1,10, 1,10,1,2,1,10,1,14,1,16,1,20,1,10,1,8, 1,20,2,8,1,4,2,10,1,22,1,114,179,0,0,0, 99,2,0,0,0,0,0,0,0,4,0,0,0,11,0,0, - 0,67,0,0,0,115,92,0,0,0,116,0,124,0,131,1, - 143,26,1,0,124,0,116,1,106,2,107,7,114,30,116,3, - 124,0,124,1,131,2,83,0,87,0,100,1,81,0,82,0, - 88,0,116,1,106,2,124,0,25,0,125,2,124,2,100,1, - 107,8,114,80,100,2,106,4,124,0,131,1,125,3,116,5, - 124,3,124,0,100,3,141,2,130,1,116,6,124,0,131,1, - 1,0,124,2,83,0,41,4,122,25,70,105,110,100,32,97, - 110,100,32,108,111,97,100,32,116,104,101,32,109,111,100,117, - 108,101,46,78,122,40,105,109,112,111,114,116,32,111,102,32, - 123,125,32,104,97,108,116,101,100,59,32,78,111,110,101,32, - 105,110,32,115,121,115,46,109,111,100,117,108,101,115,41,1, - 114,15,0,0,0,41,7,114,42,0,0,0,114,14,0,0, - 0,114,80,0,0,0,114,179,0,0,0,114,38,0,0,0, - 114,177,0,0,0,114,57,0,0,0,41,4,114,15,0,0, - 0,114,178,0,0,0,114,84,0,0,0,114,68,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, - 14,95,102,105,110,100,95,97,110,100,95,108,111,97,100,188, - 3,0,0,115,20,0,0,0,0,2,10,1,10,1,20,2, - 10,1,8,1,4,1,6,1,12,2,8,1,114,180,0,0, - 0,114,19,0,0,0,99,3,0,0,0,0,0,0,0,3, - 0,0,0,4,0,0,0,67,0,0,0,115,42,0,0,0, - 116,0,124,0,124,1,124,2,131,3,1,0,124,2,100,1, - 107,4,114,32,116,1,124,0,124,1,124,2,131,3,125,0, - 116,2,124,0,116,3,131,2,83,0,41,2,97,50,1,0, - 0,73,109,112,111,114,116,32,97,110,100,32,114,101,116,117, - 114,110,32,116,104,101,32,109,111,100,117,108,101,32,98,97, - 115,101,100,32,111,110,32,105,116,115,32,110,97,109,101,44, - 32,116,104,101,32,112,97,99,107,97,103,101,32,116,104,101, - 32,99,97,108,108,32,105,115,10,32,32,32,32,98,101,105, - 110,103,32,109,97,100,101,32,102,114,111,109,44,32,97,110, - 100,32,116,104,101,32,108,101,118,101,108,32,97,100,106,117, - 115,116,109,101,110,116,46,10,10,32,32,32,32,84,104,105, - 115,32,102,117,110,99,116,105,111,110,32,114,101,112,114,101, - 115,101,110,116,115,32,116,104,101,32,103,114,101,97,116,101, - 115,116,32,99,111,109,109,111,110,32,100,101,110,111,109,105, - 110,97,116,111,114,32,111,102,32,102,117,110,99,116,105,111, - 110,97,108,105,116,121,10,32,32,32,32,98,101,116,119,101, - 101,110,32,105,109,112,111,114,116,95,109,111,100,117,108,101, - 32,97,110,100,32,95,95,105,109,112,111,114,116,95,95,46, - 32,84,104,105,115,32,105,110,99,108,117,100,101,115,32,115, - 101,116,116,105,110,103,32,95,95,112,97,99,107,97,103,101, - 95,95,32,105,102,10,32,32,32,32,116,104,101,32,108,111, - 97,100,101,114,32,100,105,100,32,110,111,116,46,10,10,32, - 32,32,32,114,19,0,0,0,41,4,114,174,0,0,0,114, - 163,0,0,0,114,180,0,0,0,218,11,95,103,99,100,95, - 105,109,112,111,114,116,41,3,114,15,0,0,0,114,161,0, - 0,0,114,162,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,11,0,0,0,114,181,0,0,0,204,3,0,0,115, - 8,0,0,0,0,9,12,1,8,1,12,1,114,181,0,0, - 0,99,3,0,0,0,0,0,0,0,6,0,0,0,17,0, - 0,0,67,0,0,0,115,164,0,0,0,116,0,124,0,100, - 1,131,2,114,160,100,2,124,1,107,6,114,58,116,1,124, - 1,131,1,125,1,124,1,106,2,100,2,131,1,1,0,116, - 0,124,0,100,3,131,2,114,58,124,1,106,3,124,0,106, - 4,131,1,1,0,120,100,124,1,68,0,93,92,125,3,116, - 0,124,0,124,3,131,2,115,64,100,4,106,5,124,0,106, - 6,124,3,131,2,125,4,121,14,116,7,124,2,124,4,131, - 2,1,0,87,0,113,64,4,0,116,8,107,10,114,154,1, - 0,125,5,1,0,122,20,124,5,106,9,124,4,107,2,114, - 136,119,64,130,0,87,0,89,0,100,5,100,5,125,5,126, - 5,88,0,113,64,88,0,113,64,87,0,124,0,83,0,41, - 6,122,238,70,105,103,117,114,101,32,111,117,116,32,119,104, - 97,116,32,95,95,105,109,112,111,114,116,95,95,32,115,104, - 111,117,108,100,32,114,101,116,117,114,110,46,10,10,32,32, - 32,32,84,104,101,32,105,109,112,111,114,116,95,32,112,97, - 114,97,109,101,116,101,114,32,105,115,32,97,32,99,97,108, - 108,97,98,108,101,32,119,104,105,99,104,32,116,97,107,101, - 115,32,116,104,101,32,110,97,109,101,32,111,102,32,109,111, - 100,117,108,101,32,116,111,10,32,32,32,32,105,109,112,111, - 114,116,46,32,73,116,32,105,115,32,114,101,113,117,105,114, - 101,100,32,116,111,32,100,101,99,111,117,112,108,101,32,116, - 104,101,32,102,117,110,99,116,105,111,110,32,102,114,111,109, - 32,97,115,115,117,109,105,110,103,32,105,109,112,111,114,116, - 108,105,98,39,115,10,32,32,32,32,105,109,112,111,114,116, - 32,105,109,112,108,101,109,101,110,116,97,116,105,111,110,32, - 105,115,32,100,101,115,105,114,101,100,46,10,10,32,32,32, - 32,114,128,0,0,0,250,1,42,218,7,95,95,97,108,108, - 95,95,122,5,123,125,46,123,125,78,41,10,114,4,0,0, - 0,114,127,0,0,0,218,6,114,101,109,111,118,101,218,6, - 101,120,116,101,110,100,114,183,0,0,0,114,38,0,0,0, - 114,1,0,0,0,114,59,0,0,0,114,177,0,0,0,114, - 15,0,0,0,41,6,114,84,0,0,0,218,8,102,114,111, - 109,108,105,115,116,114,178,0,0,0,218,1,120,90,9,102, - 114,111,109,95,110,97,109,101,90,3,101,120,99,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,16,95,104, - 97,110,100,108,101,95,102,114,111,109,108,105,115,116,219,3, - 0,0,115,32,0,0,0,0,10,10,1,8,1,8,1,10, - 1,10,1,12,1,10,1,10,1,14,1,2,1,14,1,16, - 4,10,1,2,1,24,1,114,188,0,0,0,99,1,0,0, - 0,0,0,0,0,3,0,0,0,6,0,0,0,67,0,0, - 0,115,146,0,0,0,124,0,106,0,100,1,131,1,125,1, - 124,0,106,0,100,2,131,1,125,2,124,1,100,3,107,9, - 114,82,124,2,100,3,107,9,114,78,124,1,124,2,106,1, - 107,3,114,78,116,2,106,3,100,4,124,1,155,2,100,5, - 124,2,106,1,155,2,100,6,157,5,116,4,100,7,100,8, - 141,3,1,0,124,1,83,0,124,2,100,3,107,9,114,96, - 124,2,106,1,83,0,116,2,106,3,100,9,116,4,100,7, - 100,8,141,3,1,0,124,0,100,10,25,0,125,1,100,11, - 124,0,107,7,114,142,124,1,106,5,100,12,131,1,100,13, - 25,0,125,1,124,1,83,0,41,14,122,167,67,97,108,99, - 117,108,97,116,101,32,119,104,97,116,32,95,95,112,97,99, - 107,97,103,101,95,95,32,115,104,111,117,108,100,32,98,101, - 46,10,10,32,32,32,32,95,95,112,97,99,107,97,103,101, - 95,95,32,105,115,32,110,111,116,32,103,117,97,114,97,110, - 116,101,101,100,32,116,111,32,98,101,32,100,101,102,105,110, - 101,100,32,111,114,32,99,111,117,108,100,32,98,101,32,115, - 101,116,32,116,111,32,78,111,110,101,10,32,32,32,32,116, - 111,32,114,101,112,114,101,115,101,110,116,32,116,104,97,116, - 32,105,116,115,32,112,114,111,112,101,114,32,118,97,108,117, - 101,32,105,115,32,117,110,107,110,111,119,110,46,10,10,32, - 32,32,32,114,131,0,0,0,114,90,0,0,0,78,122,32, - 95,95,112,97,99,107,97,103,101,95,95,32,33,61,32,95, - 95,115,112,101,99,95,95,46,112,97,114,101,110,116,32,40, - 122,4,32,33,61,32,250,1,41,233,3,0,0,0,41,1, - 90,10,115,116,97,99,107,108,101,118,101,108,122,89,99,97, - 110,39,116,32,114,101,115,111,108,118,101,32,112,97,99,107, - 97,103,101,32,102,114,111,109,32,95,95,115,112,101,99,95, - 95,32,111,114,32,95,95,112,97,99,107,97,103,101,95,95, - 44,32,102,97,108,108,105,110,103,32,98,97,99,107,32,111, - 110,32,95,95,110,97,109,101,95,95,32,97,110,100,32,95, - 95,112,97,116,104,95,95,114,1,0,0,0,114,128,0,0, - 0,114,118,0,0,0,114,19,0,0,0,41,6,114,30,0, - 0,0,114,120,0,0,0,114,167,0,0,0,114,168,0,0, - 0,114,169,0,0,0,114,119,0,0,0,41,3,218,7,103, - 108,111,98,97,108,115,114,161,0,0,0,114,83,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, - 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, - 95,95,250,3,0,0,115,30,0,0,0,0,7,10,1,10, - 1,8,1,18,1,22,2,10,1,4,1,8,1,6,2,6, - 2,10,1,8,1,8,1,14,1,114,192,0,0,0,99,5, - 0,0,0,0,0,0,0,9,0,0,0,5,0,0,0,67, - 0,0,0,115,166,0,0,0,124,4,100,1,107,2,114,18, - 116,0,124,0,131,1,125,5,110,36,124,1,100,2,107,9, - 114,30,124,1,110,2,105,0,125,6,116,1,124,6,131,1, - 125,7,116,0,124,0,124,7,124,4,131,3,125,5,124,3, - 115,150,124,4,100,1,107,2,114,84,116,0,124,0,106,2, - 100,3,131,1,100,1,25,0,131,1,83,0,124,0,115,92, - 124,5,83,0,116,3,124,0,131,1,116,3,124,0,106,2, - 100,3,131,1,100,1,25,0,131,1,24,0,125,8,116,4, - 106,5,124,5,106,6,100,2,116,3,124,5,106,6,131,1, - 124,8,24,0,133,2,25,0,25,0,83,0,110,12,116,7, - 124,5,124,3,116,0,131,3,83,0,100,2,83,0,41,4, - 97,215,1,0,0,73,109,112,111,114,116,32,97,32,109,111, - 100,117,108,101,46,10,10,32,32,32,32,84,104,101,32,39, - 103,108,111,98,97,108,115,39,32,97,114,103,117,109,101,110, - 116,32,105,115,32,117,115,101,100,32,116,111,32,105,110,102, - 101,114,32,119,104,101,114,101,32,116,104,101,32,105,109,112, - 111,114,116,32,105,115,32,111,99,99,117,114,114,105,110,103, - 32,102,114,111,109,10,32,32,32,32,116,111,32,104,97,110, - 100,108,101,32,114,101,108,97,116,105,118,101,32,105,109,112, - 111,114,116,115,46,32,84,104,101,32,39,108,111,99,97,108, - 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,105, - 103,110,111,114,101,100,46,32,84,104,101,10,32,32,32,32, - 39,102,114,111,109,108,105,115,116,39,32,97,114,103,117,109, - 101,110,116,32,115,112,101,99,105,102,105,101,115,32,119,104, - 97,116,32,115,104,111,117,108,100,32,101,120,105,115,116,32, - 97,115,32,97,116,116,114,105,98,117,116,101,115,32,111,110, - 32,116,104,101,32,109,111,100,117,108,101,10,32,32,32,32, - 98,101,105,110,103,32,105,109,112,111,114,116,101,100,32,40, - 101,46,103,46,32,96,96,102,114,111,109,32,109,111,100,117, - 108,101,32,105,109,112,111,114,116,32,60,102,114,111,109,108, - 105,115,116,62,96,96,41,46,32,32,84,104,101,32,39,108, - 101,118,101,108,39,10,32,32,32,32,97,114,103,117,109,101, - 110,116,32,114,101,112,114,101,115,101,110,116,115,32,116,104, - 101,32,112,97,99,107,97,103,101,32,108,111,99,97,116,105, - 111,110,32,116,111,32,105,109,112,111,114,116,32,102,114,111, - 109,32,105,110,32,97,32,114,101,108,97,116,105,118,101,10, - 32,32,32,32,105,109,112,111,114,116,32,40,101,46,103,46, - 32,96,96,102,114,111,109,32,46,46,112,107,103,32,105,109, - 112,111,114,116,32,109,111,100,96,96,32,119,111,117,108,100, - 32,104,97,118,101,32,97,32,39,108,101,118,101,108,39,32, - 111,102,32,50,41,46,10,10,32,32,32,32,114,19,0,0, - 0,78,114,118,0,0,0,41,8,114,181,0,0,0,114,192, - 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,159, - 0,0,0,114,14,0,0,0,114,80,0,0,0,114,1,0, - 0,0,114,188,0,0,0,41,9,114,15,0,0,0,114,191, - 0,0,0,218,6,108,111,99,97,108,115,114,186,0,0,0, - 114,162,0,0,0,114,84,0,0,0,90,8,103,108,111,98, - 97,108,115,95,114,161,0,0,0,90,7,99,117,116,95,111, - 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,10,95,95,105,109,112,111,114,116,95,95,21,4,0, - 0,115,26,0,0,0,0,11,8,1,10,2,16,1,8,1, - 12,1,4,3,8,1,18,1,4,1,4,4,26,3,32,2, - 114,195,0,0,0,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,38,0,0,0,116, - 0,106,1,124,0,131,1,125,1,124,1,100,0,107,8,114, - 30,116,2,100,1,124,0,23,0,131,1,130,1,116,3,124, - 1,131,1,83,0,41,2,78,122,25,110,111,32,98,117,105, - 108,116,45,105,110,32,109,111,100,117,108,101,32,110,97,109, - 101,100,32,41,4,114,142,0,0,0,114,146,0,0,0,114, - 71,0,0,0,114,141,0,0,0,41,2,114,15,0,0,0, - 114,83,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,18,95,98,117,105,108,116,105,110,95,102, - 114,111,109,95,110,97,109,101,56,4,0,0,115,8,0,0, - 0,0,1,10,1,8,1,12,1,114,196,0,0,0,99,2, - 0,0,0,0,0,0,0,12,0,0,0,12,0,0,0,67, - 0,0,0,115,244,0,0,0,124,1,97,0,124,0,97,1, - 116,2,116,1,131,1,125,2,120,86,116,1,106,3,106,4, - 131,0,68,0,93,72,92,2,125,3,125,4,116,5,124,4, - 124,2,131,2,114,28,124,3,116,1,106,6,107,6,114,62, - 116,7,125,5,110,18,116,0,106,8,124,3,131,1,114,28, - 116,9,125,5,110,2,113,28,116,10,124,4,124,5,131,2, - 125,6,116,11,124,6,124,4,131,2,1,0,113,28,87,0, - 116,1,106,3,116,12,25,0,125,7,120,54,100,5,68,0, - 93,46,125,8,124,8,116,1,106,3,107,7,114,144,116,13, - 124,8,131,1,125,9,110,10,116,1,106,3,124,8,25,0, - 125,9,116,14,124,7,124,8,124,9,131,3,1,0,113,120, - 87,0,121,12,116,13,100,2,131,1,125,10,87,0,110,24, - 4,0,116,15,107,10,114,206,1,0,1,0,1,0,100,3, - 125,10,89,0,110,2,88,0,116,14,124,7,100,2,124,10, - 131,3,1,0,116,13,100,4,131,1,125,11,116,14,124,7, - 100,4,124,11,131,3,1,0,100,3,83,0,41,6,122,250, - 83,101,116,117,112,32,105,109,112,111,114,116,108,105,98,32, - 98,121,32,105,109,112,111,114,116,105,110,103,32,110,101,101, - 100,101,100,32,98,117,105,108,116,45,105,110,32,109,111,100, - 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, - 110,103,32,116,104,101,109,10,32,32,32,32,105,110,116,111, - 32,116,104,101,32,103,108,111,98,97,108,32,110,97,109,101, - 115,112,97,99,101,46,10,10,32,32,32,32,65,115,32,115, - 121,115,32,105,115,32,110,101,101,100,101,100,32,102,111,114, - 32,115,121,115,46,109,111,100,117,108,101,115,32,97,99,99, - 101,115,115,32,97,110,100,32,95,105,109,112,32,105,115,32, - 110,101,101,100,101,100,32,116,111,32,108,111,97,100,32,98, - 117,105,108,116,45,105,110,10,32,32,32,32,109,111,100,117, - 108,101,115,44,32,116,104,111,115,101,32,116,119,111,32,109, - 111,100,117,108,101,115,32,109,117,115,116,32,98,101,32,101, - 120,112,108,105,99,105,116,108,121,32,112,97,115,115,101,100, - 32,105,110,46,10,10,32,32,32,32,114,167,0,0,0,114, - 20,0,0,0,78,114,55,0,0,0,41,1,114,167,0,0, - 0,41,16,114,52,0,0,0,114,14,0,0,0,114,13,0, - 0,0,114,80,0,0,0,218,5,105,116,101,109,115,114,171, - 0,0,0,114,70,0,0,0,114,142,0,0,0,114,76,0, - 0,0,114,152,0,0,0,114,129,0,0,0,114,134,0,0, - 0,114,1,0,0,0,114,196,0,0,0,114,5,0,0,0, - 114,71,0,0,0,41,12,218,10,115,121,115,95,109,111,100, - 117,108,101,218,11,95,105,109,112,95,109,111,100,117,108,101, - 90,11,109,111,100,117,108,101,95,116,121,112,101,114,15,0, - 0,0,114,84,0,0,0,114,94,0,0,0,114,83,0,0, - 0,90,11,115,101,108,102,95,109,111,100,117,108,101,90,12, - 98,117,105,108,116,105,110,95,110,97,109,101,90,14,98,117, - 105,108,116,105,110,95,109,111,100,117,108,101,90,13,116,104, - 114,101,97,100,95,109,111,100,117,108,101,90,14,119,101,97, - 107,114,101,102,95,109,111,100,117,108,101,114,10,0,0,0, - 114,10,0,0,0,114,11,0,0,0,218,6,95,115,101,116, - 117,112,63,4,0,0,115,50,0,0,0,0,9,4,1,4, - 3,8,1,20,1,10,1,10,1,6,1,10,1,6,2,2, - 1,10,1,14,3,10,1,10,1,10,1,10,2,10,1,16, - 3,2,1,12,1,14,2,10,1,12,3,8,1,114,200,0, - 0,0,99,2,0,0,0,0,0,0,0,3,0,0,0,3, - 0,0,0,67,0,0,0,115,66,0,0,0,116,0,124,0, - 124,1,131,2,1,0,116,1,106,2,106,3,116,4,131,1, - 1,0,116,1,106,2,106,3,116,5,131,1,1,0,100,1, - 100,2,108,6,125,2,124,2,97,7,124,2,106,8,116,1, - 106,9,116,10,25,0,131,1,1,0,100,2,83,0,41,3, - 122,50,73,110,115,116,97,108,108,32,105,109,112,111,114,116, - 108,105,98,32,97,115,32,116,104,101,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, - 111,114,116,46,114,19,0,0,0,78,41,11,114,200,0,0, - 0,114,14,0,0,0,114,166,0,0,0,114,110,0,0,0, - 114,142,0,0,0,114,152,0,0,0,218,26,95,102,114,111, - 122,101,110,95,105,109,112,111,114,116,108,105,98,95,101,120, - 116,101,114,110,97,108,114,116,0,0,0,218,8,95,105,110, - 115,116,97,108,108,114,80,0,0,0,114,1,0,0,0,41, - 3,114,198,0,0,0,114,199,0,0,0,114,201,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 202,0,0,0,110,4,0,0,115,12,0,0,0,0,2,10, - 2,12,1,12,3,8,1,4,1,114,202,0,0,0,41,2, - 78,78,41,1,78,41,2,78,114,19,0,0,0,41,50,114, - 3,0,0,0,114,116,0,0,0,114,12,0,0,0,114,16, - 0,0,0,114,49,0,0,0,114,29,0,0,0,114,36,0, - 0,0,114,17,0,0,0,114,18,0,0,0,114,41,0,0, - 0,114,42,0,0,0,114,45,0,0,0,114,57,0,0,0, - 114,59,0,0,0,114,69,0,0,0,114,75,0,0,0,114, - 78,0,0,0,114,85,0,0,0,114,96,0,0,0,114,97, - 0,0,0,114,103,0,0,0,114,79,0,0,0,218,6,111, - 98,106,101,99,116,90,9,95,80,79,80,85,76,65,84,69, - 114,129,0,0,0,114,134,0,0,0,114,137,0,0,0,114, - 92,0,0,0,114,81,0,0,0,114,140,0,0,0,114,141, - 0,0,0,114,82,0,0,0,114,142,0,0,0,114,152,0, - 0,0,114,157,0,0,0,114,163,0,0,0,114,165,0,0, - 0,114,170,0,0,0,114,174,0,0,0,90,15,95,69,82, - 82,95,77,83,71,95,80,82,69,70,73,88,114,176,0,0, - 0,114,179,0,0,0,114,180,0,0,0,114,181,0,0,0, - 114,188,0,0,0,114,192,0,0,0,114,195,0,0,0,114, - 196,0,0,0,114,200,0,0,0,114,202,0,0,0,114,10, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,8,60,109,111,100,117,108,101,62,8,0,0,0, - 115,94,0,0,0,4,17,4,2,8,8,8,8,4,2,4, - 3,16,4,14,68,14,21,14,16,8,27,8,17,8,11,14, - 8,8,11,8,12,8,16,8,36,14,27,14,101,16,26,6, - 3,10,45,14,60,8,17,8,17,8,24,8,29,8,23,8, - 15,14,73,14,77,14,13,8,9,8,9,10,47,8,16,4, - 1,8,2,8,27,8,16,10,15,8,31,8,27,18,35,8, - 7,8,47, + 0,67,0,0,0,115,94,0,0,0,116,0,124,0,131,1, + 143,38,1,0,116,1,106,2,106,3,124,0,116,4,131,2, + 125,2,124,2,116,4,107,8,114,42,116,5,124,0,124,1, + 131,2,83,0,87,0,100,1,81,0,82,0,88,0,124,2, + 100,1,107,8,114,82,100,2,106,6,124,0,131,1,125,3, + 116,7,124,3,124,0,100,3,141,2,130,1,116,8,124,0, + 131,1,1,0,124,2,83,0,41,4,122,25,70,105,110,100, + 32,97,110,100,32,108,111,97,100,32,116,104,101,32,109,111, + 100,117,108,101,46,78,122,40,105,109,112,111,114,116,32,111, + 102,32,123,125,32,104,97,108,116,101,100,59,32,78,111,110, + 101,32,105,110,32,115,121,115,46,109,111,100,117,108,101,115, + 41,1,114,15,0,0,0,41,9,114,42,0,0,0,114,14, + 0,0,0,114,80,0,0,0,114,30,0,0,0,218,14,95, + 78,69,69,68,83,95,76,79,65,68,73,78,71,114,179,0, + 0,0,114,38,0,0,0,114,177,0,0,0,114,57,0,0, + 0,41,4,114,15,0,0,0,114,178,0,0,0,114,84,0, + 0,0,114,68,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,218,14,95,102,105,110,100,95,97,110, + 100,95,108,111,97,100,188,3,0,0,115,20,0,0,0,0, + 2,10,1,14,1,8,1,20,2,8,1,4,1,6,1,12, + 2,8,1,114,181,0,0,0,114,19,0,0,0,99,3,0, + 0,0,0,0,0,0,3,0,0,0,4,0,0,0,67,0, + 0,0,115,42,0,0,0,116,0,124,0,124,1,124,2,131, + 3,1,0,124,2,100,1,107,4,114,32,116,1,124,0,124, + 1,124,2,131,3,125,0,116,2,124,0,116,3,131,2,83, + 0,41,2,97,50,1,0,0,73,109,112,111,114,116,32,97, + 110,100,32,114,101,116,117,114,110,32,116,104,101,32,109,111, + 100,117,108,101,32,98,97,115,101,100,32,111,110,32,105,116, + 115,32,110,97,109,101,44,32,116,104,101,32,112,97,99,107, + 97,103,101,32,116,104,101,32,99,97,108,108,32,105,115,10, + 32,32,32,32,98,101,105,110,103,32,109,97,100,101,32,102, + 114,111,109,44,32,97,110,100,32,116,104,101,32,108,101,118, + 101,108,32,97,100,106,117,115,116,109,101,110,116,46,10,10, + 32,32,32,32,84,104,105,115,32,102,117,110,99,116,105,111, + 110,32,114,101,112,114,101,115,101,110,116,115,32,116,104,101, + 32,103,114,101,97,116,101,115,116,32,99,111,109,109,111,110, + 32,100,101,110,111,109,105,110,97,116,111,114,32,111,102,32, + 102,117,110,99,116,105,111,110,97,108,105,116,121,10,32,32, + 32,32,98,101,116,119,101,101,110,32,105,109,112,111,114,116, + 95,109,111,100,117,108,101,32,97,110,100,32,95,95,105,109, + 112,111,114,116,95,95,46,32,84,104,105,115,32,105,110,99, + 108,117,100,101,115,32,115,101,116,116,105,110,103,32,95,95, + 112,97,99,107,97,103,101,95,95,32,105,102,10,32,32,32, + 32,116,104,101,32,108,111,97,100,101,114,32,100,105,100,32, + 110,111,116,46,10,10,32,32,32,32,114,19,0,0,0,41, + 4,114,174,0,0,0,114,163,0,0,0,114,181,0,0,0, + 218,11,95,103,99,100,95,105,109,112,111,114,116,41,3,114, + 15,0,0,0,114,161,0,0,0,114,162,0,0,0,114,10, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,182,0, + 0,0,204,3,0,0,115,8,0,0,0,0,9,12,1,8, + 1,12,1,114,182,0,0,0,99,3,0,0,0,0,0,0, + 0,6,0,0,0,17,0,0,0,67,0,0,0,115,164,0, + 0,0,116,0,124,0,100,1,131,2,114,160,100,2,124,1, + 107,6,114,58,116,1,124,1,131,1,125,1,124,1,106,2, + 100,2,131,1,1,0,116,0,124,0,100,3,131,2,114,58, + 124,1,106,3,124,0,106,4,131,1,1,0,120,100,124,1, + 68,0,93,92,125,3,116,0,124,0,124,3,131,2,115,64, + 100,4,106,5,124,0,106,6,124,3,131,2,125,4,121,14, + 116,7,124,2,124,4,131,2,1,0,87,0,113,64,4,0, + 116,8,107,10,114,154,1,0,125,5,1,0,122,20,124,5, + 106,9,124,4,107,2,114,136,119,64,130,0,87,0,89,0, + 100,5,100,5,125,5,126,5,88,0,113,64,88,0,113,64, + 87,0,124,0,83,0,41,6,122,238,70,105,103,117,114,101, + 32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,111, + 114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,117, + 114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,112, + 111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,105, + 115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,105, + 99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,109, + 101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,32, + 32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,115, + 32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,99, + 111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,105, + 111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,103, + 32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,32, + 32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,110, + 116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,101, + 100,46,10,10,32,32,32,32,114,128,0,0,0,250,1,42, + 218,7,95,95,97,108,108,95,95,122,5,123,125,46,123,125, + 78,41,10,114,4,0,0,0,114,127,0,0,0,218,6,114, + 101,109,111,118,101,218,6,101,120,116,101,110,100,114,184,0, + 0,0,114,38,0,0,0,114,1,0,0,0,114,59,0,0, + 0,114,177,0,0,0,114,15,0,0,0,41,6,114,84,0, + 0,0,218,8,102,114,111,109,108,105,115,116,114,178,0,0, + 0,218,1,120,90,9,102,114,111,109,95,110,97,109,101,90, + 3,101,120,99,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,16,95,104,97,110,100,108,101,95,102,114,111, + 109,108,105,115,116,219,3,0,0,115,32,0,0,0,0,10, + 10,1,8,1,8,1,10,1,10,1,12,1,10,1,10,1, + 14,1,2,1,14,1,16,4,10,1,2,1,24,1,114,189, + 0,0,0,99,1,0,0,0,0,0,0,0,3,0,0,0, + 6,0,0,0,67,0,0,0,115,146,0,0,0,124,0,106, + 0,100,1,131,1,125,1,124,0,106,0,100,2,131,1,125, + 2,124,1,100,3,107,9,114,82,124,2,100,3,107,9,114, + 78,124,1,124,2,106,1,107,3,114,78,116,2,106,3,100, + 4,124,1,155,2,100,5,124,2,106,1,155,2,100,6,157, + 5,116,4,100,7,100,8,141,3,1,0,124,1,83,0,124, + 2,100,3,107,9,114,96,124,2,106,1,83,0,116,2,106, + 3,100,9,116,4,100,7,100,8,141,3,1,0,124,0,100, + 10,25,0,125,1,100,11,124,0,107,7,114,142,124,1,106, + 5,100,12,131,1,100,13,25,0,125,1,124,1,83,0,41, + 14,122,167,67,97,108,99,117,108,97,116,101,32,119,104,97, + 116,32,95,95,112,97,99,107,97,103,101,95,95,32,115,104, + 111,117,108,100,32,98,101,46,10,10,32,32,32,32,95,95, + 112,97,99,107,97,103,101,95,95,32,105,115,32,110,111,116, + 32,103,117,97,114,97,110,116,101,101,100,32,116,111,32,98, + 101,32,100,101,102,105,110,101,100,32,111,114,32,99,111,117, + 108,100,32,98,101,32,115,101,116,32,116,111,32,78,111,110, + 101,10,32,32,32,32,116,111,32,114,101,112,114,101,115,101, + 110,116,32,116,104,97,116,32,105,116,115,32,112,114,111,112, + 101,114,32,118,97,108,117,101,32,105,115,32,117,110,107,110, + 111,119,110,46,10,10,32,32,32,32,114,131,0,0,0,114, + 90,0,0,0,78,122,32,95,95,112,97,99,107,97,103,101, + 95,95,32,33,61,32,95,95,115,112,101,99,95,95,46,112, + 97,114,101,110,116,32,40,122,4,32,33,61,32,250,1,41, + 233,3,0,0,0,41,1,90,10,115,116,97,99,107,108,101, + 118,101,108,122,89,99,97,110,39,116,32,114,101,115,111,108, + 118,101,32,112,97,99,107,97,103,101,32,102,114,111,109,32, + 95,95,115,112,101,99,95,95,32,111,114,32,95,95,112,97, + 99,107,97,103,101,95,95,44,32,102,97,108,108,105,110,103, + 32,98,97,99,107,32,111,110,32,95,95,110,97,109,101,95, + 95,32,97,110,100,32,95,95,112,97,116,104,95,95,114,1, + 0,0,0,114,128,0,0,0,114,118,0,0,0,114,19,0, + 0,0,41,6,114,30,0,0,0,114,120,0,0,0,114,167, + 0,0,0,114,168,0,0,0,114,169,0,0,0,114,119,0, + 0,0,41,3,218,7,103,108,111,98,97,108,115,114,161,0, + 0,0,114,83,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,218,17,95,99,97,108,99,95,95,95, + 112,97,99,107,97,103,101,95,95,250,3,0,0,115,30,0, + 0,0,0,7,10,1,10,1,8,1,18,1,22,2,10,1, + 4,1,8,1,6,2,6,2,10,1,8,1,8,1,14,1, + 114,193,0,0,0,99,5,0,0,0,0,0,0,0,9,0, + 0,0,5,0,0,0,67,0,0,0,115,166,0,0,0,124, + 4,100,1,107,2,114,18,116,0,124,0,131,1,125,5,110, + 36,124,1,100,2,107,9,114,30,124,1,110,2,105,0,125, + 6,116,1,124,6,131,1,125,7,116,0,124,0,124,7,124, + 4,131,3,125,5,124,3,115,150,124,4,100,1,107,2,114, + 84,116,0,124,0,106,2,100,3,131,1,100,1,25,0,131, + 1,83,0,124,0,115,92,124,5,83,0,116,3,124,0,131, + 1,116,3,124,0,106,2,100,3,131,1,100,1,25,0,131, + 1,24,0,125,8,116,4,106,5,124,5,106,6,100,2,116, + 3,124,5,106,6,131,1,124,8,24,0,133,2,25,0,25, + 0,83,0,110,12,116,7,124,5,124,3,116,0,131,3,83, + 0,100,2,83,0,41,4,97,215,1,0,0,73,109,112,111, + 114,116,32,97,32,109,111,100,117,108,101,46,10,10,32,32, + 32,32,84,104,101,32,39,103,108,111,98,97,108,115,39,32, + 97,114,103,117,109,101,110,116,32,105,115,32,117,115,101,100, + 32,116,111,32,105,110,102,101,114,32,119,104,101,114,101,32, + 116,104,101,32,105,109,112,111,114,116,32,105,115,32,111,99, + 99,117,114,114,105,110,103,32,102,114,111,109,10,32,32,32, + 32,116,111,32,104,97,110,100,108,101,32,114,101,108,97,116, + 105,118,101,32,105,109,112,111,114,116,115,46,32,84,104,101, + 32,39,108,111,99,97,108,115,39,32,97,114,103,117,109,101, + 110,116,32,105,115,32,105,103,110,111,114,101,100,46,32,84, + 104,101,10,32,32,32,32,39,102,114,111,109,108,105,115,116, + 39,32,97,114,103,117,109,101,110,116,32,115,112,101,99,105, + 102,105,101,115,32,119,104,97,116,32,115,104,111,117,108,100, + 32,101,120,105,115,116,32,97,115,32,97,116,116,114,105,98, + 117,116,101,115,32,111,110,32,116,104,101,32,109,111,100,117, + 108,101,10,32,32,32,32,98,101,105,110,103,32,105,109,112, + 111,114,116,101,100,32,40,101,46,103,46,32,96,96,102,114, + 111,109,32,109,111,100,117,108,101,32,105,109,112,111,114,116, + 32,60,102,114,111,109,108,105,115,116,62,96,96,41,46,32, + 32,84,104,101,32,39,108,101,118,101,108,39,10,32,32,32, + 32,97,114,103,117,109,101,110,116,32,114,101,112,114,101,115, + 101,110,116,115,32,116,104,101,32,112,97,99,107,97,103,101, + 32,108,111,99,97,116,105,111,110,32,116,111,32,105,109,112, + 111,114,116,32,102,114,111,109,32,105,110,32,97,32,114,101, + 108,97,116,105,118,101,10,32,32,32,32,105,109,112,111,114, + 116,32,40,101,46,103,46,32,96,96,102,114,111,109,32,46, + 46,112,107,103,32,105,109,112,111,114,116,32,109,111,100,96, + 96,32,119,111,117,108,100,32,104,97,118,101,32,97,32,39, + 108,101,118,101,108,39,32,111,102,32,50,41,46,10,10,32, + 32,32,32,114,19,0,0,0,78,114,118,0,0,0,41,8, + 114,182,0,0,0,114,193,0,0,0,218,9,112,97,114,116, + 105,116,105,111,110,114,159,0,0,0,114,14,0,0,0,114, + 80,0,0,0,114,1,0,0,0,114,189,0,0,0,41,9, + 114,15,0,0,0,114,192,0,0,0,218,6,108,111,99,97, + 108,115,114,187,0,0,0,114,162,0,0,0,114,84,0,0, + 0,90,8,103,108,111,98,97,108,115,95,114,161,0,0,0, + 90,7,99,117,116,95,111,102,102,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,218,10,95,95,105,109,112,111, + 114,116,95,95,21,4,0,0,115,26,0,0,0,0,11,8, + 1,10,2,16,1,8,1,12,1,4,3,8,1,18,1,4, + 1,4,4,26,3,32,2,114,196,0,0,0,99,1,0,0, + 0,0,0,0,0,2,0,0,0,3,0,0,0,67,0,0, + 0,115,38,0,0,0,116,0,106,1,124,0,131,1,125,1, + 124,1,100,0,107,8,114,30,116,2,100,1,124,0,23,0, + 131,1,130,1,116,3,124,1,131,1,83,0,41,2,78,122, + 25,110,111,32,98,117,105,108,116,45,105,110,32,109,111,100, + 117,108,101,32,110,97,109,101,100,32,41,4,114,142,0,0, + 0,114,146,0,0,0,114,71,0,0,0,114,141,0,0,0, + 41,2,114,15,0,0,0,114,83,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,218,18,95,98,117, + 105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,56, + 4,0,0,115,8,0,0,0,0,1,10,1,8,1,12,1, + 114,197,0,0,0,99,2,0,0,0,0,0,0,0,12,0, + 0,0,12,0,0,0,67,0,0,0,115,244,0,0,0,124, + 1,97,0,124,0,97,1,116,2,116,1,131,1,125,2,120, + 86,116,1,106,3,106,4,131,0,68,0,93,72,92,2,125, + 3,125,4,116,5,124,4,124,2,131,2,114,28,124,3,116, + 1,106,6,107,6,114,62,116,7,125,5,110,18,116,0,106, + 8,124,3,131,1,114,28,116,9,125,5,110,2,113,28,116, + 10,124,4,124,5,131,2,125,6,116,11,124,6,124,4,131, + 2,1,0,113,28,87,0,116,1,106,3,116,12,25,0,125, + 7,120,54,100,5,68,0,93,46,125,8,124,8,116,1,106, + 3,107,7,114,144,116,13,124,8,131,1,125,9,110,10,116, + 1,106,3,124,8,25,0,125,9,116,14,124,7,124,8,124, + 9,131,3,1,0,113,120,87,0,121,12,116,13,100,2,131, + 1,125,10,87,0,110,24,4,0,116,15,107,10,114,206,1, + 0,1,0,1,0,100,3,125,10,89,0,110,2,88,0,116, + 14,124,7,100,2,124,10,131,3,1,0,116,13,100,4,131, + 1,125,11,116,14,124,7,100,4,124,11,131,3,1,0,100, + 3,83,0,41,6,122,250,83,101,116,117,112,32,105,109,112, + 111,114,116,108,105,98,32,98,121,32,105,109,112,111,114,116, + 105,110,103,32,110,101,101,100,101,100,32,98,117,105,108,116, + 45,105,110,32,109,111,100,117,108,101,115,32,97,110,100,32, + 105,110,106,101,99,116,105,110,103,32,116,104,101,109,10,32, + 32,32,32,105,110,116,111,32,116,104,101,32,103,108,111,98, + 97,108,32,110,97,109,101,115,112,97,99,101,46,10,10,32, + 32,32,32,65,115,32,115,121,115,32,105,115,32,110,101,101, + 100,101,100,32,102,111,114,32,115,121,115,46,109,111,100,117, + 108,101,115,32,97,99,99,101,115,115,32,97,110,100,32,95, + 105,109,112,32,105,115,32,110,101,101,100,101,100,32,116,111, + 32,108,111,97,100,32,98,117,105,108,116,45,105,110,10,32, + 32,32,32,109,111,100,117,108,101,115,44,32,116,104,111,115, + 101,32,116,119,111,32,109,111,100,117,108,101,115,32,109,117, + 115,116,32,98,101,32,101,120,112,108,105,99,105,116,108,121, + 32,112,97,115,115,101,100,32,105,110,46,10,10,32,32,32, + 32,114,167,0,0,0,114,20,0,0,0,78,114,55,0,0, + 0,41,1,114,167,0,0,0,41,16,114,52,0,0,0,114, + 14,0,0,0,114,13,0,0,0,114,80,0,0,0,218,5, + 105,116,101,109,115,114,171,0,0,0,114,70,0,0,0,114, + 142,0,0,0,114,76,0,0,0,114,152,0,0,0,114,129, + 0,0,0,114,134,0,0,0,114,1,0,0,0,114,197,0, + 0,0,114,5,0,0,0,114,71,0,0,0,41,12,218,10, + 115,121,115,95,109,111,100,117,108,101,218,11,95,105,109,112, + 95,109,111,100,117,108,101,90,11,109,111,100,117,108,101,95, + 116,121,112,101,114,15,0,0,0,114,84,0,0,0,114,94, + 0,0,0,114,83,0,0,0,90,11,115,101,108,102,95,109, + 111,100,117,108,101,90,12,98,117,105,108,116,105,110,95,110, + 97,109,101,90,14,98,117,105,108,116,105,110,95,109,111,100, + 117,108,101,90,13,116,104,114,101,97,100,95,109,111,100,117, + 108,101,90,14,119,101,97,107,114,101,102,95,109,111,100,117, + 108,101,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,218,6,95,115,101,116,117,112,63,4,0,0,115,50,0, + 0,0,0,9,4,1,4,3,8,1,20,1,10,1,10,1, + 6,1,10,1,6,2,2,1,10,1,14,3,10,1,10,1, + 10,1,10,2,10,1,16,3,2,1,12,1,14,2,10,1, + 12,3,8,1,114,201,0,0,0,99,2,0,0,0,0,0, + 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,66, + 0,0,0,116,0,124,0,124,1,131,2,1,0,116,1,106, + 2,106,3,116,4,131,1,1,0,116,1,106,2,106,3,116, + 5,131,1,1,0,100,1,100,2,108,6,125,2,124,2,97, + 7,124,2,106,8,116,1,106,9,116,10,25,0,131,1,1, + 0,100,2,83,0,41,3,122,50,73,110,115,116,97,108,108, + 32,105,109,112,111,114,116,108,105,98,32,97,115,32,116,104, + 101,32,105,109,112,108,101,109,101,110,116,97,116,105,111,110, + 32,111,102,32,105,109,112,111,114,116,46,114,19,0,0,0, + 78,41,11,114,201,0,0,0,114,14,0,0,0,114,166,0, + 0,0,114,110,0,0,0,114,142,0,0,0,114,152,0,0, + 0,218,26,95,102,114,111,122,101,110,95,105,109,112,111,114, + 116,108,105,98,95,101,120,116,101,114,110,97,108,114,116,0, + 0,0,218,8,95,105,110,115,116,97,108,108,114,80,0,0, + 0,114,1,0,0,0,41,3,114,199,0,0,0,114,200,0, + 0,0,114,202,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,203,0,0,0,110,4,0,0,115, + 12,0,0,0,0,2,10,2,12,1,12,3,8,1,4,1, + 114,203,0,0,0,41,2,78,78,41,1,78,41,2,78,114, + 19,0,0,0,41,50,114,3,0,0,0,114,116,0,0,0, + 114,12,0,0,0,114,16,0,0,0,114,49,0,0,0,114, + 29,0,0,0,114,36,0,0,0,114,17,0,0,0,114,18, + 0,0,0,114,41,0,0,0,114,42,0,0,0,114,45,0, + 0,0,114,57,0,0,0,114,59,0,0,0,114,69,0,0, + 0,114,75,0,0,0,114,78,0,0,0,114,85,0,0,0, + 114,96,0,0,0,114,97,0,0,0,114,103,0,0,0,114, + 79,0,0,0,114,129,0,0,0,114,134,0,0,0,114,137, + 0,0,0,114,92,0,0,0,114,81,0,0,0,114,140,0, + 0,0,114,141,0,0,0,114,82,0,0,0,114,142,0,0, + 0,114,152,0,0,0,114,157,0,0,0,114,163,0,0,0, + 114,165,0,0,0,114,170,0,0,0,114,174,0,0,0,90, + 15,95,69,82,82,95,77,83,71,95,80,82,69,70,73,88, + 114,176,0,0,0,114,179,0,0,0,218,6,111,98,106,101, + 99,116,114,180,0,0,0,114,181,0,0,0,114,182,0,0, + 0,114,189,0,0,0,114,193,0,0,0,114,196,0,0,0, + 114,197,0,0,0,114,201,0,0,0,114,203,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,8,60,109,111,100,117,108,101,62,8,0,0, + 0,115,94,0,0,0,4,17,4,2,8,8,8,8,4,2, + 4,3,16,4,14,68,14,21,14,16,8,27,8,17,8,11, + 14,8,8,11,8,12,8,16,8,36,14,27,14,101,16,26, + 10,45,14,60,8,17,8,17,8,24,8,29,8,23,8,15, + 14,73,14,77,14,13,8,9,8,9,10,47,8,16,4,1, + 8,2,8,27,6,3,8,16,10,15,8,31,8,27,18,35, + 8,7,8,47, }; From 64b9a15886840df422c5203fad25c9801b4cf61e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 22 Jul 2017 01:31:04 +0300 Subject: [PATCH 0583/1345] [3.6] bpo-29403: Fix mock's broken autospec behavior on method-bound builtin functions (GH-3) Cython will, in the right circumstances, offer a MethodType instance where im_func is a builtin function. Any instance of MethodType is automatically assumed to be a Python-defined function (more specifically, a function that has an inspectable signature), but _set_signature was still conservative in its assumptions. As a result _set_signature would return early with None instead of a mock since the im_func had no inspectable signature. This causes problems deeper inside mock, as _set_signature is assumed to _always_ return a mock, and nothing checked its return value. In similar corner cases, autospec will simply not check the spec of the function, so _set_signature is amended to now return early with the original, not-wrapped mock object. Patch by Aaron Gallagher. (cherry picked from commit 856cbcc12f2e4cca93af5dc7ed6bcea4dd942f10) --- Lib/unittest/mock.py | 2 +- Lib/unittest/test/testmock/testhelpers.py | 15 +++++++++++++++ .../2017-07-20-02-29-49.bpo-29403.3RinCV.rst | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-20-02-29-49.bpo-29403.3RinCV.rst diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 5f97728de28c59..6be7bb4f065e81 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -165,7 +165,7 @@ def _set_signature(mock, original, instance=False): skipfirst = isinstance(original, type) result = _get_signature_object(original, instance, skipfirst) if result is None: - return + return mock func, sig = result def checksig(*args, **kwargs): sig.bind(*args, **kwargs) diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d2202a7b4132e9..7919482ae99c7f 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -1,3 +1,5 @@ +import time +import types import unittest from unittest.mock import ( @@ -856,6 +858,19 @@ def check_data_descriptor(mock_attr): check_data_descriptor(foo.desc) + def test_autospec_on_bound_builtin_function(self): + meth = types.MethodType(time.ctime, time.time()) + self.assertIsInstance(meth(), str) + mocked = create_autospec(meth) + + # no signature, so no spec to check against + mocked() + mocked.assert_called_once_with() + mocked.reset_mock() + mocked(4, 5, 6) + mocked.assert_called_once_with(4, 5, 6) + + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/Misc/NEWS.d/next/Library/2017-07-20-02-29-49.bpo-29403.3RinCV.rst b/Misc/NEWS.d/next/Library/2017-07-20-02-29-49.bpo-29403.3RinCV.rst new file mode 100644 index 00000000000000..95bcd1007de05a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-20-02-29-49.bpo-29403.3RinCV.rst @@ -0,0 +1,2 @@ +Fix ``unittest.mock``'s autospec to not fail on method-bound builtin +functions. Patch by Aaron Gallagher. From 04864b491e3ce0f022e918ad344fb53a99ce2a1b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 22 Jul 2017 00:56:18 -0400 Subject: [PATCH 0584/1345] =?UTF-8?q?[3.6]=20bpo-30981:=20IDLE=20--=20Add?= =?UTF-8?q?=20more=20configdialog=20font=20page=20tests.=20=20(GH-?= =?UTF-8?q?=E2=80=A6=20(#2796)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verify that clicking the bold checkbutton and calling its command, set_samples, changes the bold setting of both samples. Simplify some names in configdialog. (cherry picked from commit d0969d6) (Incorporates changes and fixes from PRs 2798, 7c5798e, and 2810, 616ecf1) * Fix broken test with PR2798 and PR2810 changes. --- Lib/idlelib/configdialog.py | 83 +++++++++---------- Lib/idlelib/idle_test/test_configdialog.py | 57 ++++++++----- .../2017-07-21-01-55-14.bpo-30981.ZFvQPt.rst | 1 + 3 files changed, 76 insertions(+), 65 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-21-01-55-14.bpo-30981.ZFvQPt.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index e1c3923caed43f..c1db76817c17c7 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -153,22 +153,29 @@ def create_action_buttons(self): def create_page_font_tab(self): """Return frame of widgets for Font/Tabs tab. + Enable users to provisionally change font face, size, or + boldness and to see the consequence of proposed choices. Each + action set 3 options in changes structuree and changes the + corresponding aspect of the font sample on this page and + highlight sample on highlight page. + + Enable users to change spaces entered for indent tabs. + Tk Variables: + font_name: Font face. font_size: Font size. font_bold: Select font bold or not. - font_name: Font face. Note: these 3 share var_changed_font callback. space_num: Indentation width. Data Attribute: - edit_font: Font widget with default font name, size, and weight. + edit_font: Font with default font name, size, and weight. Methods: load_font_cfg: Set vars and fontlist. on_fontlist_select: Bound to fontlist button release or key release. - set_font_sample: Command for opt_menu_font_size and - check_font_bold. + set_samples: Notify both samples of any font change. load_tab_cfg: Get current. Widget Structure: (*) widgets bound to self @@ -181,7 +188,7 @@ def create_page_font_tab(self): frame_font_param: Frame font_size_title: Label (*)opt_menu_font_size: DynOptionMenu - font_size - check_font_bold: Checkbutton - font_bold + (*)bold_toggle: Checkbutton - font_bold frame_font_sample: Frame (*)font_sample: Label frame_indent: LabelFrame @@ -190,9 +197,9 @@ def create_page_font_tab(self): (*)scale_indent_size: Scale - space_num """ parent = self.parent + self.font_name = StringVar(parent) self.font_size = StringVar(parent) self.font_bold = BooleanVar(parent) - self.font_name = StringVar(parent) self.space_num = IntVar(parent) self.edit_font = tkFont.Font(parent, ('courier', 10, 'normal')) @@ -218,10 +225,10 @@ def create_page_font_tab(self): self.fontlist.config(yscrollcommand=scroll_font.set) font_size_title = Label(frame_font_param, text='Size :') self.opt_menu_font_size = DynOptionMenu( - frame_font_param, self.font_size, None, command=self.set_font_sample) - check_font_bold = Checkbutton( + frame_font_param, self.font_size, None, command=self.set_samples) + self.bold_toggle = Checkbutton( frame_font_param, variable=self.font_bold, onvalue=1, - offvalue=0, text='Bold', command=self.set_font_sample) + offvalue=0, text='Bold', command=self.set_samples) frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) self.font_sample = Label( frame_font_sample, justify=LEFT, font=self.edit_font, @@ -247,7 +254,7 @@ def create_page_font_tab(self): scroll_font.pack(side=LEFT, fill=Y) font_size_title.pack(side=LEFT, anchor=W) self.opt_menu_font_size.pack(side=LEFT, anchor=W) - check_font_bold.pack(side=LEFT, anchor=W, padx=20) + self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) self.font_sample.pack(expand=TRUE, fill=BOTH) # frame_indent @@ -292,7 +299,7 @@ def create_page_highlight(self): Widget Structure: (*) widgets bound to self frame frame_custom: LabelFrame - (*)text_highlight_sample: Text + (*)highlight_sample: Text (*)frame_color_set: Frame button_set_color: Button (*)opt_menu_highlight_target: DynOptionMenu - highlight_target @@ -342,11 +349,11 @@ def create_page_highlight(self): frame_theme = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Highlighting Theme ') #frame_custom - self.text_highlight_sample=Text( + self.highlight_sample=Text( frame_custom, relief=SOLID, borderwidth=1, font=('courier', 12, ''), cursor='hand2', width=21, height=11, takefocus=FALSE, highlightthickness=0, wrap=NONE) - text=self.text_highlight_sample + text=self.highlight_sample text.bind('', lambda e: 'break') text.bind('', lambda e: 'break') text_and_tags=( @@ -416,7 +423,7 @@ def tem(event, elem=element): #frame_custom self.frame_color_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) frame_fg_bg_toggle.pack(side=TOP, padx=5, pady=0) - self.text_highlight_sample.pack( + self.highlight_sample.pack( side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) button_set_color.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) self.opt_menu_highlight_target.pack( @@ -1142,7 +1149,7 @@ def on_new_color_set(self): self.frame_color_set.config(bg=new_color) # Set sample. plane ='foreground' if self.fg_bg_toggle.get() else 'background' sample_element = self.theme_elements[self.highlight_target.get()][0] - self.text_highlight_sample.tag_config(sample_element, **{plane:new_color}) + self.highlight_sample.tag_config(sample_element, **{plane:new_color}) theme = self.custom_theme.get() theme_element = sample_element + '-' + plane changes.add_option('highlight', theme, theme_element, new_color) @@ -1210,41 +1217,25 @@ def on_fontlist_select(self, event): """Handle selecting a font from the list. Event can result from either mouse click or Up or Down key. - Set font_name and example display to selection. - - Attributes updated: - font_name: Set to name selected from fontlist. - - Methods: - set_font_sample + Set font_name and example displays to selection. """ font = self.fontlist.get( ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) self.font_name.set(font.lower()) - self.set_font_sample() - - def set_font_sample(self, event=None): - """Update the screen samples with the font settings from the dialog. - - Attributes accessed: - font_name - font_bold - font_size + self.set_samples() - Attributes updated: - font_sample: Set to selected font name, size, and weight. - text_highlight_sample: Set to selected font name, size, and weight. + def set_samples(self, event=None): + """Update update both screen samples with the font settings. - Called from: - handler for opt_menu_font_size and check_font_bold - on_fontlist_select - load_font_cfg + Called on font initialization and change events. + Accesses font_name, font_size, and font_bold Variables. + Updates font_sample and hightlight page highlight_sample. """ font_name = self.font_name.get() font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) - self.font_sample.config(font=new_font) - self.text_highlight_sample.configure(font=new_font) + self.font_sample['font'] = new_font + self.highlight_sample['font'] = new_font def set_highlight_target(self): """Set fg/bg toggle and color based on highlight tag target. @@ -1289,7 +1280,7 @@ def set_color_sample(self): theme_elements highlight_target fg_bg_toggle - text_highlight_sample + highlight_sample Attributes updated: frame_color_set @@ -1297,7 +1288,7 @@ def set_color_sample(self): # Set the color sample area. tag = self.theme_elements[self.highlight_target.get()][0] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - color = self.text_highlight_sample.tag_cget(tag, plane) + color = self.highlight_sample.tag_cget(tag, plane) self.frame_color_set.config(bg=color) def paint_theme_sample(self): @@ -1310,7 +1301,7 @@ def paint_theme_sample(self): custom_theme Attributes updated: - text_highlight_sample: Set the tag elements to the theme. + highlight_sample: Set the tag elements to the theme. Methods: set_color_sample @@ -1337,7 +1328,7 @@ def paint_theme_sample(self): colors['foreground'] = theme_dict[element + '-foreground'] if element + '-background' in theme_dict: colors['background'] = theme_dict[element + '-background'] - self.text_highlight_sample.tag_config(element, **colors) + self.highlight_sample.tag_config(element, **colors) self.set_color_sample() def help_source_selected(self, event): @@ -1424,7 +1415,7 @@ def load_font_cfg(self): font_bold: Set to current font weight. Methods: - set_font_sample + set_samples """ # Set base editor font selection list. fonts = list(tkFont.families(self)) @@ -1452,7 +1443,7 @@ def load_font_cfg(self): # Set font weight. self.font_bold.set(font_bold) # Set font sample. - self.set_font_sample() + self.set_samples() def load_tab_cfg(self): """Load current configuration settings for the tab options. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 4f1f9af180abb1..171fa3d309c44f 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -28,18 +28,12 @@ highpage = changes['highlight'] keyspage = changes['keys'] - -class TestDialog(ConfigDialog): - pass # Delete? - - def setUpModule(): global root, dialog idleConf.userCfg = testcfg root = Tk() # root.withdraw() # Comment out, see issue 30870 - dialog = TestDialog(root, 'Test', _utest=True) - + dialog = ConfigDialog(root, 'Test', _utest=True) def tearDownModule(): global root, dialog @@ -52,12 +46,16 @@ def tearDownModule(): class FontTabTest(unittest.TestCase): + "Test that font widget enable users to make font changes." + def setUp(self): changes.clear() - def test_font(self): - # Set values guaranteed not to be defaults. + def test_font_set(self): + # Test that setting a font Variable results in 3 provisional + # change entries. Use values sure to not be defaults. + # Other font tests verify that user actions set Variables. default_font = idleConf.GetFont(root, 'main', 'EditorWindow') default_size = str(default_font[1]) default_bold = default_font[2] == 'bold' @@ -79,9 +77,30 @@ def test_font(self): 'font-bold': str(not default_bold)}} self.assertEqual(mainpage, expected) - def test_set_sample(self): - # Set_font_sample also sets highlight_sample. - pass + def test_set_samples_bold_toggle(self): + # Set up. + d = dialog + d.font_sample, d.highlight_sample = {}, {} # Must undo this. + d.font_name.set('test') + d.font_size.set('5') + d.font_bold.set(1) + expected0 = {'font': ('test', '5', 'normal')} + expected1 = {'font': ('test', '5', 'bold')} + + # Test set_samples. + d.set_samples() + self.assertTrue(d.font_sample == d.highlight_sample == expected1) + + # Test bold_toggle. + d.bold_toggle.invoke() + self.assertFalse(d.font_bold.get()) + self.assertTrue(d.font_sample == d.highlight_sample == expected0) + d.bold_toggle.invoke() + self.assertTrue(d.font_bold.get()) + self.assertTrue(d.font_sample == d.highlight_sample == expected1) + + # Clean up. + del d.font_sample, d.highlight_sample def test_tabspace(self): dialog.space_num.set(6) @@ -90,7 +109,7 @@ def test_tabspace(self): class FontSelectTest(unittest.TestCase): # These two functions test that selecting a new font in the - # list of fonts changes font_name and calls set_font_sample. + # list of fonts changes font_name and calls set_samples. # The fontlist widget and on_fontlist_select event handler # are tested here together. @@ -98,14 +117,14 @@ class FontSelectTest(unittest.TestCase): def setUpClass(cls): if dialog.fontlist.size() < 2: cls.skipTest('need at least 2 fonts') - dialog.set_font_sample = Func() # Mask instance method. + dialog.set_samples = Func() # Mask instance method. @classmethod def tearDownClass(cls): - del dialog.set_font_sample # Unmask instance method. + del dialog.set_samples # Unmask instance method. def setUp(self): - dialog.set_font_sample.called = 0 + dialog.set_samples.called = 0 changes.clear() def test_select_font_key(self): @@ -124,7 +143,7 @@ def test_select_font_key(self): down_font = fontlist.get('active') self.assertNotEqual(down_font, font) self.assertIn(dialog.font_name.get(), down_font.lower()) - self.assertEqual(dialog.set_font_sample.called, 1) + self.assertEqual(dialog.set_samples.called, 1) # Test Up key. fontlist.focus_force() @@ -135,7 +154,7 @@ def test_select_font_key(self): up_font = fontlist.get('active') self.assertEqual(up_font, font) self.assertIn(dialog.font_name.get(), up_font.lower()) - self.assertEqual(dialog.set_font_sample.called, 2) + self.assertEqual(dialog.set_samples.called, 2) def test_select_font_mouse(self): # Click on item should select that item. @@ -157,7 +176,7 @@ def test_select_font_mouse(self): select_font = fontlist.get('anchor') self.assertEqual(select_font, font1) self.assertIn(dialog.font_name.get(), font1.lower()) - self.assertEqual(dialog.set_font_sample.called, 1) + self.assertEqual(dialog.set_samples.called, 1) class HighlightTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-21-01-55-14.bpo-30981.ZFvQPt.rst b/Misc/NEWS.d/next/IDLE/2017-07-21-01-55-14.bpo-30981.ZFvQPt.rst new file mode 100644 index 00000000000000..48d477ed2e1d0b --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-21-01-55-14.bpo-30981.ZFvQPt.rst @@ -0,0 +1 @@ +IDLE -- Add more configdialog font page tests. From 8de48fe04633974318d48d0abf29e552e7c742c0 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 22 Jul 2017 23:00:39 -0700 Subject: [PATCH 0585/1345] [3.6] Make test_shutil test_disk_usage not depend on the cwd fs (GH-2597) (#2820) Make test_shutil test_disk_usage not depend on the current working directory's filesystem. (cherry picked from commit 529746c90584069270cd122920270bd7be38bca3) --- Lib/test/test_shutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 2ad3a21c86d2c1..b2ab1af296d682 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1294,7 +1294,7 @@ def _boo(filename, extract_dir, extra): @unittest.skipUnless(hasattr(shutil, 'disk_usage'), "disk_usage not available on this platform") def test_disk_usage(self): - usage = shutil.disk_usage(os.getcwd()) + usage = shutil.disk_usage(os.path.dirname(__file__)) self.assertGreater(usage.total, 0) self.assertGreater(usage.used, 0) self.assertGreaterEqual(usage.free, 0) From 5aa3bf041de5ee90ccbfcff103dcf3e54c5af237 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 23 Jul 2017 14:18:27 -0400 Subject: [PATCH 0586/1345] [3.6] bpo-30993: IDLE - Improve configdialog font page and tests. (GH-2818) (#2826) * Document causal event pathways in docstring. * Simplify some attribute names. * Rename test_bold_toggle_set_samples to make test_font_set fail. * Fix test_font_set so not order dependent. * Fix renamed test_indent_scale so it tests the widget. (cherry picked from commit 07ba305) --- Lib/idlelib/configdialog.py | 99 ++++++++++--------- Lib/idlelib/idle_test/test_configdialog.py | 20 ++-- .../2017-07-22-18-08-41.bpo-30993.34vJkB.rst | 6 ++ 3 files changed, 69 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index c1db76817c17c7..d51eca6f533276 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -153,70 +153,73 @@ def create_action_buttons(self): def create_page_font_tab(self): """Return frame of widgets for Font/Tabs tab. - Enable users to provisionally change font face, size, or + Fonts: Enable users to provisionally change font face, size, or boldness and to see the consequence of proposed choices. Each action set 3 options in changes structuree and changes the corresponding aspect of the font sample on this page and highlight sample on highlight page. - Enable users to change spaces entered for indent tabs. + Fontlist: mouse button 1 click or up or down key invoke + on_fontlist_select(), which sets Var font_name and calls + set_samples. - Tk Variables: - font_name: Font face. - font_size: Font size. - font_bold: Select font bold or not. - Note: these 3 share var_changed_font callback. - space_num: Indentation width. + Sizelist: clicking the menubutton opens the dropdown menu. A + mouse button 1 click or return key invokes an internal command + which sets Var font_size and calls set_samples. - Data Attribute: - edit_font: Font with default font name, size, and weight. + Bold_toggle, clicking the box toggles font_bold and calls + set_samples. - Methods: - load_font_cfg: Set vars and fontlist. - on_fontlist_select: Bound to fontlist button release - or key release. - set_samples: Notify both samples of any font change. - load_tab_cfg: Get current. + Setting any of the font vars invokes var_changed_font, which + adds all 3 font options to changes. Set_samples applies a new + font constructed from the font vars to font_sample and + highlight_sample on the hightlight page. + + Tabs: Enable users to change spaces entered for indent tabs. + Changing indent_scale value with the mouse sets Var space_num, + which invokes var_changed_space_num, which adds an entry to + changes. + + Load_font_cfg and load_tab_cfg initialize vars and widgets from + idleConf entries. Widget Structure: (*) widgets bound to self - frame + frame (of tab_pages) frame_font: LabelFrame frame_font_name: Frame font_name_title: Label - (*)fontlist: ListBox + (*)fontlist: ListBox - font_name scroll_font: Scrollbar frame_font_param: Frame font_size_title: Label - (*)opt_menu_font_size: DynOptionMenu - font_size + (*)sizelist: DynOptionMenu - font_size (*)bold_toggle: Checkbutton - font_bold frame_font_sample: Frame (*)font_sample: Label frame_indent: LabelFrame - frame_indent_size: Frame - indent_size_title: Label - (*)scale_indent_size: Scale - space_num + indent_title: Label + (*)indent_scale: Scale - space_num """ parent = self.parent self.font_name = StringVar(parent) self.font_size = StringVar(parent) self.font_bold = BooleanVar(parent) self.space_num = IntVar(parent) - self.edit_font = tkFont.Font(parent, ('courier', 10, 'normal')) - # Create widgets. + # Create widgets: # body and body section frames. frame = self.tab_pages.pages['Fonts/Tabs'].frame frame_font = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') frame_indent = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') - # frame_font + # frame_font. frame_font_name = Frame(frame_font) frame_font_param = Frame(frame_font) font_name_title = Label( frame_font_name, justify=LEFT, text='Font Face :') - self.fontlist = Listbox( - frame_font_name, height=5, takefocus=FALSE, exportselection=FALSE) + self.fontlist = Listbox(frame_font_name, height=5, + takefocus=FALSE, exportselection=FALSE) self.fontlist.bind('', self.on_fontlist_select) self.fontlist.bind('', self.on_fontlist_select) self.fontlist.bind('', self.on_fontlist_select) @@ -224,43 +227,43 @@ def create_page_font_tab(self): scroll_font.config(command=self.fontlist.yview) self.fontlist.config(yscrollcommand=scroll_font.set) font_size_title = Label(frame_font_param, text='Size :') - self.opt_menu_font_size = DynOptionMenu( - frame_font_param, self.font_size, None, command=self.set_samples) + self.sizelist = DynOptionMenu(frame_font_param, self.font_size, + None, command=self.set_samples) self.bold_toggle = Checkbutton( frame_font_param, variable=self.font_bold, onvalue=1, offvalue=0, text='Bold', command=self.set_samples) frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) + temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) self.font_sample = Label( - frame_font_sample, justify=LEFT, font=self.edit_font, + frame_font_sample, justify=LEFT, font=temp_font, text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') - # frame_indent - frame_indent_size = Frame(frame_indent) - indent_size_title = Label( - frame_indent_size, justify=LEFT, + # frame_indent. + indent_title = Label( + frame_indent, justify=LEFT, text='Python Standard: 4 Spaces!') - self.scale_indent_size = Scale( - frame_indent_size, variable=self.space_num, + self.indent_scale = Scale( + frame_indent, variable=self.space_num, orient='horizontal', tickinterval=2, from_=2, to=16) - # Pack widgets. - # body + # Pack widgets: + # body. frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) - # frame_font + # frame_font. frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) font_name_title.pack(side=TOP, anchor=W) self.fontlist.pack(side=LEFT, expand=TRUE, fill=X) scroll_font.pack(side=LEFT, fill=Y) font_size_title.pack(side=LEFT, anchor=W) - self.opt_menu_font_size.pack(side=LEFT, anchor=W) + self.sizelist.pack(side=LEFT, anchor=W) self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) self.font_sample.pack(expand=TRUE, fill=BOTH) - # frame_indent - frame_indent_size.pack(side=TOP, fill=X) - indent_size_title.pack(side=TOP, anchor=W, padx=5) - self.scale_indent_size.pack(side=TOP, padx=5, fill=X) + # frame_indent. + frame_indent.pack(side=TOP, fill=X) + indent_title.pack(side=TOP, anchor=W, padx=5) + self.indent_scale.pack(side=TOP, padx=5, fill=X) return frame @@ -1410,7 +1413,7 @@ def load_font_cfg(self): Attributes updated: fontlist: Populate with fonts from tkinter.font. font_name: Set to current font. - opt_menu_font_size: Populate valid options tuple and set + sizelist: Populate valid options tuple and set to current size. font_bold: Set to current font weight. @@ -1437,9 +1440,9 @@ def load_font_cfg(self): except ValueError: pass # Set font size dropdown. - self.opt_menu_font_size.SetMenu(('7', '8', '9', '10', '11', '12', '13', - '14', '16', '18', '20', '22', - '25', '29', '34', '40'), font_size ) + self.sizelist.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', + '16', '18', '20', '22', '25', '29', '34', '40'), + font_size) # Set font weight. self.font_bold.set(font_bold) # Set font sample. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 171fa3d309c44f..b6d5df3c5b4579 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -59,25 +59,28 @@ def test_font_set(self): default_font = idleConf.GetFont(root, 'main', 'EditorWindow') default_size = str(default_font[1]) default_bold = default_font[2] == 'bold' - dialog.font_name.set('Test Font') + d = dialog + d.font_name.set('Test Font') + d.font_size.set(default_size) + d.font_bold.set(default_bold) expected = {'EditorWindow': {'font': 'Test Font', 'font-size': default_size, 'font-bold': str(default_bold)}} self.assertEqual(mainpage, expected) changes.clear() - dialog.font_size.set(20) + d.font_size.set(20) expected = {'EditorWindow': {'font': 'Test Font', 'font-size': '20', 'font-bold': str(default_bold)}} self.assertEqual(mainpage, expected) changes.clear() - dialog.font_bold.set(not default_bold) + d.font_bold.set(not default_bold) expected = {'EditorWindow': {'font': 'Test Font', 'font-size': '20', 'font-bold': str(not default_bold)}} self.assertEqual(mainpage, expected) - def test_set_samples_bold_toggle(self): + def test_bold_toggle_set_samples(self): # Set up. d = dialog d.font_sample, d.highlight_sample = {}, {} # Must undo this. @@ -91,7 +94,7 @@ def test_set_samples_bold_toggle(self): d.set_samples() self.assertTrue(d.font_sample == d.highlight_sample == expected1) - # Test bold_toggle. + # Test bold_toggle. If this fails, problem precedes set_samples. d.bold_toggle.invoke() self.assertFalse(d.font_bold.get()) self.assertTrue(d.font_sample == d.highlight_sample == expected0) @@ -102,9 +105,10 @@ def test_set_samples_bold_toggle(self): # Clean up. del d.font_sample, d.highlight_sample - def test_tabspace(self): - dialog.space_num.set(6) - self.assertEqual(mainpage, {'Indent': {'num-spaces': '6'}}) + def test_indent_scale(self): + dialog.indent_scale.set(26) + self.assertEqual(dialog.space_num.get(), 16) + self.assertEqual(mainpage, {'Indent': {'num-spaces': '16'}}) class FontSelectTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst b/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst new file mode 100644 index 00000000000000..8eab703c5264e5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst @@ -0,0 +1,6 @@ +IDLE - Improve configdialog font page and tests. + +* Document causal pathways in docstring. * Simplify some attribute names. * +Rename test_bold_toggle_set_samples to make test_font_set fail. * Fix +test_font_set so not order dependent. * Fix renamed test_indent_scale so it +tests the widget. From f76d1a0819899c4aa6fd3fa5834c7677faa01b12 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 23 Jul 2017 17:10:40 -0400 Subject: [PATCH 0587/1345] Mac installer build now needs venv for docs build (#2828) (#2830) (cherry picked from commit 4c7532e96da19f2d5543bdab2b972797699c37d5) --- Mac/BuildScript/build-installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 1fff052e2489b2..4cb7e3214cd40a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -1088,10 +1088,10 @@ def buildPythonDocs(): docdir = os.path.join(rootDir, 'pydocs') curDir = os.getcwd() os.chdir(buildDir) - # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9 runCommand('make clean') - # Assume sphinx-build is on our PATH, checked in checkEnvironment - runCommand('make html') + # Create virtual environment for docs builds with blurb and sphinx + runCommand('make venv') + runCommand('make html PYTHON=venv/bin/python') os.chdir(curDir) if not os.path.exists(docdir): os.mkdir(docdir) From 1daeb259799d0664c9453a3bd8e80411e65b52c9 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 24 Jul 2017 02:50:28 -0400 Subject: [PATCH 0588/1345] [3.6] bpo-30993: IDLE - Improve configdialog font page and tests. (GH-2831) (#2834) In configdialog: Document causal pathways in create_font_tab docstring. Simplify some attribute names. Move set_samples calls to var_changed_font (idea from Cheryl Sabella). Move related functions to positions after the create widgets function. In test_configdialog: Fix test_font_set so not order dependent. Fix renamed test_indent_scale so it tests the widget. Adjust tests for movement of set_samples call. Add tests for load functions. Put all font tests in one class and tab indent tests in another. Except for two lines, these tests completely cover the related functions. (cherry picked from commit 77e97ca) --- Lib/idlelib/configdialog.py | 221 +++++++++--------- Lib/idlelib/idle_test/test_configdialog.py | 189 ++++++++------- .../2017-07-22-18-08-41.bpo-30993.34vJkB.rst | 7 +- 3 files changed, 216 insertions(+), 201 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index d51eca6f533276..1832e156dc6fac 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -150,6 +150,7 @@ def create_action_buttons(self): buttons.pack(side=BOTTOM) return outer + def create_page_font_tab(self): """Return frame of widgets for Font/Tabs tab. @@ -159,29 +160,26 @@ def create_page_font_tab(self): corresponding aspect of the font sample on this page and highlight sample on highlight page. + Load_font_cfg initializes font vars and widgets from + idleConf entries and tk. + Fontlist: mouse button 1 click or up or down key invoke - on_fontlist_select(), which sets Var font_name and calls - set_samples. + on_fontlist_select(), which sets var font_name. Sizelist: clicking the menubutton opens the dropdown menu. A - mouse button 1 click or return key invokes an internal command - which sets Var font_size and calls set_samples. + mouse button 1 click or return key sets var font_size. - Bold_toggle, clicking the box toggles font_bold and calls - set_samples. + Bold_toggle: clicking the box toggles var font_bold. - Setting any of the font vars invokes var_changed_font, which - adds all 3 font options to changes. Set_samples applies a new - font constructed from the font vars to font_sample and - highlight_sample on the hightlight page. + Changing any of the font vars invokes var_changed_font, which + adds all 3 font options to changes and calls set_samples. + Set_samples applies a new font constructed from the font vars to + font_sample and to highlight_sample on the hightlight page. Tabs: Enable users to change spaces entered for indent tabs. Changing indent_scale value with the mouse sets Var space_num, which invokes var_changed_space_num, which adds an entry to - changes. - - Load_font_cfg and load_tab_cfg initialize vars and widgets from - idleConf entries. + changes. Load_tab_cfg initializes space_num to default. Widget Structure: (*) widgets bound to self frame (of tab_pages) @@ -227,11 +225,10 @@ def create_page_font_tab(self): scroll_font.config(command=self.fontlist.yview) self.fontlist.config(yscrollcommand=scroll_font.set) font_size_title = Label(frame_font_param, text='Size :') - self.sizelist = DynOptionMenu(frame_font_param, self.font_size, - None, command=self.set_samples) + self.sizelist = DynOptionMenu(frame_font_param, self.font_size, None) self.bold_toggle = Checkbutton( - frame_font_param, variable=self.font_bold, onvalue=1, - offvalue=0, text='Bold', command=self.set_samples) + frame_font_param, variable=self.font_bold, + onvalue=1, offvalue=0, text='Bold') frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) self.font_sample = Label( @@ -267,6 +264,96 @@ def create_page_font_tab(self): return frame + def load_font_cfg(self): + """Load current configuration settings for the font options. + + Retrieve current font with idleConf.GetFont and font families + from tk. Setup fontlist and set font_name. Setup sizelist, + which sets font_size. Set font_bold. Setting font variables + calls set_samples (thrice). + """ + configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') + font_name = configured_font[0].lower() + font_size = configured_font[1] + font_bold = configured_font[2]=='bold' + + # Set editor font selection list and font_name. + fonts = list(tkFont.families(self)) + fonts.sort() + for font in fonts: + self.fontlist.insert(END, font) + self.font_name.set(font_name) + lc_fonts = [s.lower() for s in fonts] + try: + current_font_index = lc_fonts.index(font_name) + self.fontlist.see(current_font_index) + self.fontlist.select_set(current_font_index) + self.fontlist.select_anchor(current_font_index) + self.fontlist.activate(current_font_index) + except ValueError: + pass + # Set font size dropdown. + self.sizelist.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', + '16', '18', '20', '22', '25', '29', '34', '40'), + font_size) + # Set font weight. + self.font_bold.set(font_bold) + + def on_fontlist_select(self, event): + """Handle selecting a font from the list. + + Event can result from either mouse click or Up or Down key. + Set font_name and example displays to selection. + """ + font = self.fontlist.get( + ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) + self.font_name.set(font.lower()) + + def var_changed_font(self, *params): + """Store changes to font attributes. + + When one font attribute changes, save them all, as they are + not independent from each other. In particular, when we are + overriding the default font, we need to write out everything. + """ + value = self.font_name.get() + changes.add_option('main', 'EditorWindow', 'font', value) + value = self.font_size.get() + changes.add_option('main', 'EditorWindow', 'font-size', value) + value = self.font_bold.get() + changes.add_option('main', 'EditorWindow', 'font-bold', value) + self.set_samples() + + def set_samples(self, event=None): + """Update update both screen samples with the font settings. + + Called on font initialization and change events. + Accesses font_name, font_size, and font_bold Variables. + Updates font_sample and hightlight page highlight_sample. + """ + font_name = self.font_name.get() + font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + new_font = (font_name, self.font_size.get(), font_weight) + self.font_sample['font'] = new_font + self.highlight_sample['font'] = new_font + + def load_tab_cfg(self): + """Load current configuration settings for the tab options. + + Attributes updated: + space_num: Set to value from idleConf. + """ + # Set indent sizes. + space_num = idleConf.GetOption( + 'main', 'Indent', 'num-spaces', default=4, type='int') + self.space_num.set(space_num) + + def var_changed_space_num(self, *params): + "Store change to indentation size." + value = self.space_num.get() + changes.add_option('main', 'Indent', 'num-spaces', value) + + def create_page_highlight(self): """Return frame of widgets for Highlighting tab. @@ -729,25 +816,6 @@ def remove_var_callbacks(self): self.startup_edit, self.autosave,): var.trace_remove('write', var.trace_info()[0][1]) - def var_changed_font(self, *params): - """Store changes to font attributes. - - When one font attribute changes, save them all, as they are - not independent from each other. In particular, when we are - overriding the default font, we need to write out everything. - """ - value = self.font_name.get() - changes.add_option('main', 'EditorWindow', 'font', value) - value = self.font_size.get() - changes.add_option('main', 'EditorWindow', 'font-size', value) - value = self.font_bold.get() - changes.add_option('main', 'EditorWindow', 'font-bold', value) - - def var_changed_space_num(self, *params): - "Store change to indentation size." - value = self.space_num.get() - changes.add_option('main', 'Indent', 'num-spaces', value) - def var_changed_color(self, *params): "Process change to color choice." self.on_new_color_set() @@ -1216,30 +1284,6 @@ def create_new_theme(self, new_theme_name): self.is_builtin_theme.set(0) self.set_theme_type() - def on_fontlist_select(self, event): - """Handle selecting a font from the list. - - Event can result from either mouse click or Up or Down key. - Set font_name and example displays to selection. - """ - font = self.fontlist.get( - ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) - self.font_name.set(font.lower()) - self.set_samples() - - def set_samples(self, event=None): - """Update update both screen samples with the font settings. - - Called on font initialization and change events. - Accesses font_name, font_size, and font_bold Variables. - Updates font_sample and hightlight page highlight_sample. - """ - font_name = self.font_name.get() - font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL - new_font = (font_name, self.font_size.get(), font_weight) - self.font_sample['font'] = new_font - self.highlight_sample['font'] = new_font - def set_highlight_target(self): """Set fg/bg toggle and color based on highlight tag target. @@ -1404,61 +1448,6 @@ def update_user_help_changed_items(self): 'main', 'HelpFiles', str(num), ';'.join(self.user_helplist[num-1][:2])) - def load_font_cfg(self): - """Load current configuration settings for the font options. - - Retrieve current font values from idleConf.GetFont to set - as initial values for font widgets. - - Attributes updated: - fontlist: Populate with fonts from tkinter.font. - font_name: Set to current font. - sizelist: Populate valid options tuple and set - to current size. - font_bold: Set to current font weight. - - Methods: - set_samples - """ - # Set base editor font selection list. - fonts = list(tkFont.families(self)) - fonts.sort() - for font in fonts: - self.fontlist.insert(END, font) - configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') - font_name = configured_font[0].lower() - font_size = configured_font[1] - font_bold = configured_font[2]=='bold' - self.font_name.set(font_name) - lc_fonts = [s.lower() for s in fonts] - try: - current_font_index = lc_fonts.index(font_name) - self.fontlist.see(current_font_index) - self.fontlist.select_set(current_font_index) - self.fontlist.select_anchor(current_font_index) - self.fontlist.activate(current_font_index) - except ValueError: - pass - # Set font size dropdown. - self.sizelist.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', - '16', '18', '20', '22', '25', '29', '34', '40'), - font_size) - # Set font weight. - self.font_bold.set(font_bold) - # Set font sample. - self.set_samples() - - def load_tab_cfg(self): - """Load current configuration settings for the tab options. - - Attributes updated: - space_num: Set to value from idleConf. - """ - # Set indent sizes. - space_num = idleConf.GetOption( - 'main', 'Indent', 'num-spaces', default=4, type='int') - self.space_num.set(space_num) - def load_theme_cfg(self): """Load current configuration settings for the theme options. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index b6d5df3c5b4579..54b2d78d667fae 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -45,82 +45,15 @@ def tearDownModule(): del root -class FontTabTest(unittest.TestCase): - "Test that font widget enable users to make font changes." - - - def setUp(self): - changes.clear() - - def test_font_set(self): - # Test that setting a font Variable results in 3 provisional - # change entries. Use values sure to not be defaults. - # Other font tests verify that user actions set Variables. - default_font = idleConf.GetFont(root, 'main', 'EditorWindow') - default_size = str(default_font[1]) - default_bold = default_font[2] == 'bold' - d = dialog - d.font_name.set('Test Font') - d.font_size.set(default_size) - d.font_bold.set(default_bold) - expected = {'EditorWindow': {'font': 'Test Font', - 'font-size': default_size, - 'font-bold': str(default_bold)}} - self.assertEqual(mainpage, expected) - changes.clear() - d.font_size.set(20) - expected = {'EditorWindow': {'font': 'Test Font', - 'font-size': '20', - 'font-bold': str(default_bold)}} - self.assertEqual(mainpage, expected) - changes.clear() - d.font_bold.set(not default_bold) - expected = {'EditorWindow': {'font': 'Test Font', - 'font-size': '20', - 'font-bold': str(not default_bold)}} - self.assertEqual(mainpage, expected) - - def test_bold_toggle_set_samples(self): - # Set up. - d = dialog - d.font_sample, d.highlight_sample = {}, {} # Must undo this. - d.font_name.set('test') - d.font_size.set('5') - d.font_bold.set(1) - expected0 = {'font': ('test', '5', 'normal')} - expected1 = {'font': ('test', '5', 'bold')} - - # Test set_samples. - d.set_samples() - self.assertTrue(d.font_sample == d.highlight_sample == expected1) - - # Test bold_toggle. If this fails, problem precedes set_samples. - d.bold_toggle.invoke() - self.assertFalse(d.font_bold.get()) - self.assertTrue(d.font_sample == d.highlight_sample == expected0) - d.bold_toggle.invoke() - self.assertTrue(d.font_bold.get()) - self.assertTrue(d.font_sample == d.highlight_sample == expected1) - - # Clean up. - del d.font_sample, d.highlight_sample - - def test_indent_scale(self): - dialog.indent_scale.set(26) - self.assertEqual(dialog.space_num.get(), 16) - self.assertEqual(mainpage, {'Indent': {'num-spaces': '16'}}) - - -class FontSelectTest(unittest.TestCase): - # These two functions test that selecting a new font in the - # list of fonts changes font_name and calls set_samples. - # The fontlist widget and on_fontlist_select event handler - # are tested here together. +class FontTest(unittest.TestCase): + """Test that font widgets enable users to make font changes. + Test that widget actions set vars, that var changes add three + options to changes and call set_samples, and that set_samples + changes the font of both sample boxes. + """ @classmethod def setUpClass(cls): - if dialog.fontlist.size() < 2: - cls.skipTest('need at least 2 fonts') dialog.set_samples = Func() # Mask instance method. @classmethod @@ -128,12 +61,27 @@ def tearDownClass(cls): del dialog.set_samples # Unmask instance method. def setUp(self): - dialog.set_samples.called = 0 changes.clear() - def test_select_font_key(self): + def test_load_font_cfg(self): + # Leave widget load test to human visual check. + # TODO Improve checks when add IdleConf.get_font_values. + d = dialog + d.font_name.set('Fake') + d.font_size.set('1') + d.font_bold.set(True) + d.set_samples.called = 0 + d.load_font_cfg() + self.assertNotEqual(d.font_name.get(), 'Fake') + self.assertNotEqual(d.font_size.get(), '1') + self.assertFalse(d.font_bold.get()) + self.assertEqual(d.set_samples.called, 3) + + def test_fontlist_key(self): # Up and Down keys should select a new font. + if dialog.fontlist.size() < 2: + cls.skipTest('need at least 2 fonts') fontlist = dialog.fontlist fontlist.activate(0) font = dialog.fontlist.get('active') @@ -147,7 +95,6 @@ def test_select_font_key(self): down_font = fontlist.get('active') self.assertNotEqual(down_font, font) self.assertIn(dialog.font_name.get(), down_font.lower()) - self.assertEqual(dialog.set_samples.called, 1) # Test Up key. fontlist.focus_force() @@ -158,11 +105,12 @@ def test_select_font_key(self): up_font = fontlist.get('active') self.assertEqual(up_font, font) self.assertIn(dialog.font_name.get(), up_font.lower()) - self.assertEqual(dialog.set_samples.called, 2) - def test_select_font_mouse(self): + def test_fontlist_mouse(self): # Click on item should select that item. + if dialog.fontlist.size() < 2: + cls.skipTest('need at least 2 fonts') fontlist = dialog.fontlist fontlist.activate(0) @@ -180,7 +128,88 @@ def test_select_font_mouse(self): select_font = fontlist.get('anchor') self.assertEqual(select_font, font1) self.assertIn(dialog.font_name.get(), font1.lower()) - self.assertEqual(dialog.set_samples.called, 1) + + def test_sizelist(self): + # Click on number shouod select that number + d = dialog + d.sizelist.variable.set(40) + self.assertEqual(d.font_size.get(), '40') + + def test_bold_toggle(self): + # Click on checkbutton should invert it. + d = dialog + d.font_bold.set(False) + d.bold_toggle.invoke() + self.assertTrue(d.font_bold.get()) + d.bold_toggle.invoke() + self.assertFalse(d.font_bold.get()) + + def test_font_set(self): + # Test that setting a font Variable results in 3 provisional + # change entries and a call to set_samples. Use values sure to + # not be defaults. + + default_font = idleConf.GetFont(root, 'main', 'EditorWindow') + default_size = str(default_font[1]) + default_bold = default_font[2] == 'bold' + d = dialog + d.font_size.set(default_size) + d.font_bold.set(default_bold) + d.set_samples.called = 0 + + d.font_name.set('Test Font') + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': default_size, + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 1) + changes.clear() + + d.font_size.set('20') + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 2) + changes.clear() + + d.font_bold.set(not default_bold) + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(not default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 3) + + def test_set_samples(self): + d = dialog + del d.set_samples # Unmask method for test + d.font_sample, d.highlight_sample = {}, {} + d.font_name.set('test') + d.font_size.set('5') + d.font_bold.set(1) + expected = {'font': ('test', '5', 'bold')} + + # Test set_samples. + d.set_samples() + self.assertTrue(d.font_sample == d.highlight_sample == expected) + + del d.font_sample, d.highlight_sample + d.set_samples = Func() # Re-mask for other tests. + + +class IndentTest(unittest.TestCase): + + def test_load_tab_cfg(self): + d = dialog + d.space_num.set(16) + d.load_tab_cfg() + self.assertEqual(d.space_num.get(), 4) + + def test_indent_scale(self): + changes.clear() + dialog.indent_scale.set(26) + self.assertEqual(dialog.space_num.get(), 16) + self.assertEqual(mainpage, {'Indent': {'num-spaces': '16'}}) class HighlightTest(unittest.TestCase): @@ -188,8 +217,6 @@ class HighlightTest(unittest.TestCase): def setUp(self): changes.clear() - #def test_colorchoose(self): pass # TODO - class KeysTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst b/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst index 8eab703c5264e5..04c94349165a51 100644 --- a/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst +++ b/Misc/NEWS.d/next/IDLE/2017-07-22-18-08-41.bpo-30993.34vJkB.rst @@ -1,6 +1,5 @@ IDLE - Improve configdialog font page and tests. -* Document causal pathways in docstring. * Simplify some attribute names. * -Rename test_bold_toggle_set_samples to make test_font_set fail. * Fix -test_font_set so not order dependent. * Fix renamed test_indent_scale so it -tests the widget. +In configdialog: Document causal pathways in create_font_tab docstring. Simplify some attribute names. Move set_samples calls to var_changed_font (idea from Cheryl Sabella). Move related functions to positions after the create widgets function. + +In test_configdialog: Fix test_font_set so not order dependent. Fix renamed test_indent_scale so it tests the widget. Adjust tests for movement of set_samples call. Add tests for load functions. Put all font tests in one class and tab indent tests in another. Except for two lines, these tests completely cover the related functions. From bb33ccfc5a216eff753b10a8dc59ec0277f1e1fb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 24 Jul 2017 17:40:50 +0200 Subject: [PATCH 0589/1345] bpo-30908: Fix dangling thread in test_os.TestSendfile (#2680) (#2844) tearDown() now clears explicitly the self.server variable to make sure that the thread is completely cleared when tearDownClass() checks if all threads have been cleaned up. Fix the following warning: $ ./python -m test --fail-env-changed -m test.test_os.TestSendfile.test_keywords -R 3:1 test_os (...) Warning -- threading_cleanup() failed to cleanup 0 threads after 3 sec (count: 0, dangling: 2) (...) Tests result: ENV CHANGED (cherry picked from commit d1cc037d1442cc35d1b194ec8e50901514360949) --- Lib/test/test_os.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e4951b44ce4955..b65ccb71591da5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2621,6 +2621,7 @@ def tearDown(self): self.client.close() if self.server.running: self.server.stop() + self.server = None def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): """A higher level wrapper representing how an application is From 548a6889f4d5cbc3c470b118fae1316adc75826b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 24 Jul 2017 18:54:53 -0400 Subject: [PATCH 0590/1345] [3.6] Minor typo in curses.rst (GH-2763) (#2853) I found a tiny typo in the curses how-to: changed 'Thy' to 'The' on line 68. (cherry picked from commit d439d3e) --- Doc/howto/curses.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 188a5cf2231b87..cbff84160ffcb6 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -65,7 +65,7 @@ and full support for mouse and keyboard input. The Python curses module ------------------------ -Thy Python module is a fairly simple wrapper over the C functions provided by +The Python module is a fairly simple wrapper over the C functions provided by curses; if you're already familiar with curses programming in C, it's really easy to transfer that knowledge to Python. The biggest difference is that the Python interface makes things simpler by merging different C functions such as From d8b5216f998bbb0f921815165f46918b86620ab7 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Tue, 25 Jul 2017 20:20:38 +0300 Subject: [PATCH 0591/1345] [3.6] bpo-31018: Switch to GH-pragma pack from __declspec(align) (GH-2848) (#2868) (cherry picked from commit 39243779f4c97ec6f700fa73c007ca66dfa9408e) --- Modules/_tracemalloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 0ce38ffc78c53f..4f3bb5463f0386 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -77,7 +77,7 @@ typedef struct #ifdef __GNUC__ __attribute__((packed)) #elif defined(_MSC_VER) -_declspec(align(4)) +#pragma pack(push, 4) #endif { /* filename cannot be NULL: "" is used if the Python frame @@ -85,6 +85,9 @@ _declspec(align(4)) PyObject *filename; unsigned int lineno; } frame_t; +#ifdef _MSC_VER +#pragma pack(pop) +#endif typedef struct { From 59e6ab15e47d496ac4e5f9d53aac0fae0c708da4 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 25 Jul 2017 11:04:08 -0700 Subject: [PATCH 0592/1345] bpo-26506: hex() documentation: mention %x % int (GH-2525) (GH-2870) (cherry picked from commit 67ba4fa467ffff825d6a0c0a21cc54ff1df2ed1b) --- Doc/library/functions.rst | 62 ++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 916e3528d79dac..bd4c94fcae19af 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -81,9 +81,24 @@ are always available. They are listed here in alphabetical order. .. function:: bin(x) - Convert an integer number to a binary string. The result is a valid Python - expression. If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. + Convert an integer number to a binary string prefixed with "0b". The result + is a valid Python expression. If *x* is not a Python :class:`int` object, it + has to define an :meth:`__index__` method that returns an integer. Some + examples: + + >>> bin(3) + '0b11' + >>> bin(-10) + '-0b1010' + + If prefix "0b" is desired or not, you can use either of the following ways. + + >>> format(14, '#b'), format(14, 'b') + ('0b1110', '1110') + >>> f'{14:#b}', f'{14:b}' + ('0b1110', '1110') + + See also :func:`format` for more information. .. class:: bool([x]) @@ -635,16 +650,26 @@ are always available. They are listed here in alphabetical order. .. function:: hex(x) - Convert an integer number to a lowercase hexadecimal string - prefixed with "0x", for example: + Convert an integer number to a lowercase hexadecimal string prefixed with + "0x". If x is not a Python :class:`int` object, it has to define an + __index__() method that returns an integer. Some examples: >>> hex(255) '0xff' >>> hex(-42) '-0x2a' - If x is not a Python :class:`int` object, it has to define an __index__() - method that returns an integer. + If you want to convert an integer number to an uppercase or lower hexadecimal + string with prefix or not, you can use either of the following ways: + + >>> '%#x' % 255, '%x' % 255, '%X' % 255 + ('0xff', 'ff', 'FF') + >>> format(255, '#x'), format(255, 'x'), format(255, 'X') + ('0xff', 'ff', 'FF') + >>> f'{255:#x}', f'{255:x}', f'{255:X}' + ('0xff', 'ff', 'FF') + + See also :func:`format` for more information. See also :func:`int` for converting a hexadecimal string to an integer using a base of 16. @@ -878,10 +903,27 @@ are always available. They are listed here in alphabetical order. .. function:: oct(x) - Convert an integer number to an octal string. The result is a valid Python - expression. If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. + Convert an integer number to an octal string prefixed with "0o". The result + is a valid Python expression. If *x* is not a Python :class:`int` object, it + has to define an :meth:`__index__` method that returns an integer. For + example: + + >>> oct(8) + '0o10' + >>> oct(-56) + '-0o70' + + If you want to convert an integer number to octal string either with prefix + "0o" or not, you can use either of the following ways. + + >>> '%#o' % 10, '%o' % 10 + ('0o12', '12') + >>> format(10, '#o'), format(10, 'o') + ('0o12', '12') + >>> f'{10:#o}', f'{10:o}' + ('0o12', '12') + See also :func:`format` for more information. .. index:: single: file object; open() built-in function From efe9fcbd2ca85ba9d6af6d95cc530a9c332f37c5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 26 Jul 2017 02:33:47 +0200 Subject: [PATCH 0593/1345] bpo-31034: Reliable signal handler for test_asyncio (#2867) (#2876) * bpo-31034: Reliable signal handler for test_asyncio Don't rely on the current SIGHUP signal handler, make sure that it's set to the "default" signal handler: SIG_DFL. * Add comments (cherry picked from commit 830080913c22a9834d310294b9f7653234dc6a59) --- Lib/test/test_asyncio/test_events.py | 33 +++++++++++------- Lib/test/test_asyncio/test_subprocess.py | 43 ++++++++++++++---------- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 492a84a2313baf..9746678607c936 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1980,19 +1980,26 @@ def test_subprocess_terminate(self): @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") def test_subprocess_send_signal(self): - prog = os.path.join(os.path.dirname(__file__), 'echo.py') - - connect = self.loop.subprocess_exec( - functools.partial(MySubprocessProtocol, self.loop), - sys.executable, prog) - transp, proto = self.loop.run_until_complete(connect) - self.assertIsInstance(proto, MySubprocessProtocol) - self.loop.run_until_complete(proto.connected) - - transp.send_signal(signal.SIGHUP) - self.loop.run_until_complete(proto.completed) - self.assertEqual(-signal.SIGHUP, proto.returncode) - transp.close() + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.send_signal(signal.SIGHUP) + self.loop.run_until_complete(proto.completed) + self.assertEqual(-signal.SIGHUP, proto.returncode) + transp.close() + finally: + signal.signal(signal.SIGHUP, old_handler) def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 2e14a8a9735535..e8822c36698200 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -166,25 +166,32 @@ def test_terminate(self): @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") def test_send_signal(self): - code = 'import time; print("sleeping", flush=True); time.sleep(3600)' - args = [sys.executable, '-c', code] - create = asyncio.create_subprocess_exec(*args, - stdout=subprocess.PIPE, - loop=self.loop) - proc = self.loop.run_until_complete(create) - - @asyncio.coroutine - def send_signal(proc): - # basic synchronization to wait until the program is sleeping - line = yield from proc.stdout.readline() - self.assertEqual(line, b'sleeping\n') + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + code = 'import time; print("sleeping", flush=True); time.sleep(3600)' + args = [sys.executable, '-c', code] + create = asyncio.create_subprocess_exec(*args, + stdout=subprocess.PIPE, + loop=self.loop) + proc = self.loop.run_until_complete(create) - proc.send_signal(signal.SIGHUP) - returncode = (yield from proc.wait()) - return returncode - - returncode = self.loop.run_until_complete(send_signal(proc)) - self.assertEqual(-signal.SIGHUP, returncode) + @asyncio.coroutine + def send_signal(proc): + # basic synchronization to wait until the program is sleeping + line = yield from proc.stdout.readline() + self.assertEqual(line, b'sleeping\n') + + proc.send_signal(signal.SIGHUP) + returncode = (yield from proc.wait()) + return returncode + + returncode = self.loop.run_until_complete(send_signal(proc)) + self.assertEqual(-signal.SIGHUP, returncode) + finally: + signal.signal(signal.SIGHUP, old_handler) def prepare_broken_pipe_test(self): # buffer large enough to feed the whole pipe buffer From d0adfb25c5082046133a18fd185375508c1c334f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 26 Jul 2017 04:48:56 +0200 Subject: [PATCH 0594/1345] [3.6] bpo-26762, bpo-31019: Backport multiprocessing fixes from master to 3.6 (#2879) * bpo-26762: Avoid daemon process in _test_multiprocessing (#2842) test_level() of _test_multiprocessing._TestLogging now uses regular processes rather than daemon processes to prevent zombi processes (to not "leak" processes). (cherry picked from commit 06634950c553f8df83330ed468c11483b857b7dc) * test_multiprocessing: Fix dangling process/thread (#2850) bpo-26762: Fix more dangling processes and threads in test_multiprocessing: * Queue: call close() followed by join_thread() * Process: call join() or self.addCleanup(p.join) (cherry picked from commit d7e64d9934d86aa6173229de5af5fe908662a33a) * test_multiprocessing detects dangling per test case (#2841) bpo-26762: test_multiprocessing now detects dangling processes and threads per test case classes: * setUpClass()/tearDownClass() of mixin classes now check if multiprocessing.process._dangling or threading._dangling was modified to detect "dangling" processses and threads. * ManagerMixin.tearDownClass() now also emits a warning if it still has more than one active child process after 5 seconds. * tearDownModule() now checks for dangling processes and threads before sleep 500 ms. And it now only sleeps if there is a least one dangling process or thread. (cherry picked from commit ffb49408f0780ae80a553208aa133bc5bb3ba129) * bpo-26762: test_multiprocessing close more queues (#2855) * Close explicitly queues to make sure that we don't leave dangling threads * test_queue_in_process(): remove unused queue * test_access() joins also the process to fix a random warning (cherry picked from commit b4c52966c810b5c5e088fceff403247f610b7d13) * bpo-31019: Fix multiprocessing.Process.is_alive() (#2875) multiprocessing.Process.is_alive() now removes the process from the _children set if the process completed. The change prevents leaking "dangling" processes. (cherry picked from commit 2db64823c20538a6cfc6033661fab5711d2d4585) --- Lib/multiprocessing/process.py | 10 ++- Lib/test/_test_multiprocessing.py | 125 ++++++++++++++++++++++++------ 2 files changed, 111 insertions(+), 24 deletions(-) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index f9c22703df29cb..1d26b5e521e7e7 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -132,10 +132,16 @@ def is_alive(self): if self is _current_process: return True assert self._parent_pid == os.getpid(), 'can only test a child process' + if self._popen is None: return False - self._popen.poll() - return self._popen.returncode is None + + returncode = self._popen.poll() + if returncode is None: + return True + else: + _children.discard(self) + return False @property def name(self): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 871a34e4b26f12..4d3c65583744e1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -32,11 +32,12 @@ # without thread support. import threading -import multiprocessing.dummy import multiprocessing.connection -import multiprocessing.managers +import multiprocessing.dummy import multiprocessing.heap +import multiprocessing.managers import multiprocessing.pool +import multiprocessing.queues from multiprocessing import util @@ -64,6 +65,13 @@ def latin(s): return s.encode('latin') + +def close_queue(queue): + if isinstance(queue, multiprocessing.queues.Queue): + queue.close() + queue.join_thread() + + # # Constants # @@ -275,6 +283,7 @@ def test_process(self): self.assertEqual(p.exitcode, 0) self.assertEqual(p.is_alive(), False) self.assertNotIn(p, self.active_children()) + close_queue(q) @classmethod def _test_terminate(cls): @@ -414,6 +423,7 @@ def test_lose_target_ref(self): p.join() self.assertIs(wr(), None) self.assertEqual(q.get(), 5) + close_queue(q) # @@ -600,6 +610,7 @@ def test_put(self): self.assertEqual(queue_full(queue, MAXSIZE), False) proc.join() + close_queue(queue) @classmethod def _test_get(cls, queue, child_can_start, parent_can_continue): @@ -662,6 +673,7 @@ def test_get(self): self.assertTimingAlmostEqual(get.elapsed, TIMEOUT3) proc.join() + close_queue(queue) @classmethod def _test_fork(cls, queue): @@ -697,6 +709,7 @@ def test_fork(self): self.assertRaises(pyqueue.Empty, queue.get, False) p.join() + close_queue(queue) def test_qsize(self): q = self.Queue() @@ -712,6 +725,7 @@ def test_qsize(self): self.assertEqual(q.qsize(), 1) q.get() self.assertEqual(q.qsize(), 0) + close_queue(q) @classmethod def _test_task_done(cls, q): @@ -739,6 +753,7 @@ def test_task_done(self): for p in workers: p.join() + close_queue(queue) def test_no_import_lock_contention(self): with test.support.temp_cwd(): @@ -769,6 +784,7 @@ def test_timeout(self): # Tolerate a delta of 30 ms because of the bad clock resolution on # Windows (usually 15.6 ms) self.assertGreaterEqual(delta, 0.170) + close_queue(q) def test_queue_feeder_donot_stop_onexc(self): # bpo-30414: verify feeder handles exceptions correctly @@ -782,7 +798,9 @@ def __reduce__(self): q = self.Queue() q.put(NotSerializable()) q.put(True) - self.assertTrue(q.get(timeout=0.1)) + # bpo-30595: use a timeout of 1 second for slow buildbots + self.assertTrue(q.get(timeout=1.0)) + close_queue(q) # # @@ -895,10 +913,12 @@ def test_notify(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) p = threading.Thread(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) # wait for both children to start sleeping sleeping.acquire() @@ -941,11 +961,13 @@ def test_notify_all(self): args=(cond, sleeping, woken, TIMEOUT1)) p.daemon = True p.start() + self.addCleanup(p.join) t = threading.Thread(target=self.f, args=(cond, sleeping, woken, TIMEOUT1)) t.daemon = True t.start() + self.addCleanup(t.join) # wait for them all to sleep for i in range(6): @@ -964,10 +986,12 @@ def test_notify_all(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) t.daemon = True t.start() + self.addCleanup(t.join) # wait for them to all sleep for i in range(6): @@ -1143,6 +1167,7 @@ def test_event(self): p.daemon = True p.start() self.assertEqual(wait(), True) + p.join() # # Tests for Barrier - adapted from tests in test/lock_tests.py @@ -1318,6 +1343,7 @@ def test_wait_return(self): self.run_threads(self._test_wait_return_f, (self.barrier, queue)) results = [queue.get() for i in range(self.N)] self.assertEqual(results.count(0), 1) + close_queue(queue) @classmethod def _test_action_f(cls, barrier, results): @@ -1488,6 +1514,7 @@ def test_thousand(self): p = self.Process(target=self._test_thousand_f, args=(self.barrier, passes, child_conn, lock)) p.start() + self.addCleanup(p.join) for i in range(passes): for j in range(self.N): @@ -2971,6 +2998,8 @@ def test_access(self): w.close() self.assertEqual(conn.recv(), 'foobar'*2) + p.join() + # # # @@ -3296,16 +3325,16 @@ def test_level(self): logger.setLevel(LEVEL1) p = self.Process(target=self._test_level, args=(writer,)) - p.daemon = True p.start() self.assertEqual(LEVEL1, reader.recv()) + p.join() logger.setLevel(logging.NOTSET) root_logger.setLevel(LEVEL2) p = self.Process(target=self._test_level, args=(writer,)) - p.daemon = True p.start() self.assertEqual(LEVEL2, reader.recv()) + p.join() root_logger.setLevel(root_level) logger.setLevel(level=LOG_LEVEL) @@ -3459,7 +3488,7 @@ def _this_sub_process(q): except pyqueue.Empty: pass -def _test_process(q): +def _test_process(): queue = multiprocessing.Queue() subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,)) subProc.daemon = True @@ -3499,8 +3528,7 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self): - queue = multiprocessing.Queue() - proc = multiprocessing.Process(target=_test_process, args=(queue,)) + proc = multiprocessing.Process(target=_test_process) proc.start() proc.join() @@ -4108,7 +4136,32 @@ def test_empty(self): # Mixins # -class ProcessesMixin(object): +class BaseMixin(object): + @classmethod + def setUpClass(cls): + cls.dangling = (multiprocessing.process._dangling.copy(), + threading._dangling.copy()) + + @classmethod + def tearDownClass(cls): + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + + processes = set(multiprocessing.process._dangling) - set(cls.dangling[0]) + if processes: + print('Warning -- Dangling processes: %s' % processes, + file=sys.stderr) + processes = None + + threads = set(threading._dangling) - set(cls.dangling[1]) + if threads: + print('Warning -- Dangling threads: %s' % threads, + file=sys.stderr) + threads = None + + +class ProcessesMixin(BaseMixin): TYPE = 'processes' Process = multiprocessing.Process connection = multiprocessing.connection @@ -4131,7 +4184,7 @@ class ProcessesMixin(object): RawArray = staticmethod(multiprocessing.RawArray) -class ManagerMixin(object): +class ManagerMixin(BaseMixin): TYPE = 'manager' Process = multiprocessing.Process Queue = property(operator.attrgetter('manager.Queue')) @@ -4155,6 +4208,7 @@ def Pool(cls, *args, **kwds): @classmethod def setUpClass(cls): + super().setUpClass() cls.manager = multiprocessing.Manager() @classmethod @@ -4162,23 +4216,35 @@ def tearDownClass(cls): # only the manager process should be returned by active_children() # but this can take a bit on slow machines, so wait a few seconds # if there are other children too (see #17395) + start_time = time.monotonic() t = 0.01 - while len(multiprocessing.active_children()) > 1 and t < 5: + while len(multiprocessing.active_children()) > 1: time.sleep(t) t *= 2 + dt = time.monotonic() - start_time + if dt >= 5.0: + print("Warning -- multiprocessing.Manager still has %s active " + "children after %s seconds" + % (multiprocessing.active_children(), dt), + file=sys.stderr) + break + gc.collect() # do garbage collection if cls.manager._number_of_objects() != 0: # This is not really an error since some tests do not # ensure that all processes which hold a reference to a # managed object have been joined. - print('Shared objects which still exist at manager shutdown:') + print('Warning -- Shared objects which still exist at manager ' + 'shutdown:') print(cls.manager._debug_info()) cls.manager.shutdown() cls.manager.join() cls.manager = None + super().tearDownClass() + -class ThreadsMixin(object): +class ThreadsMixin(BaseMixin): TYPE = 'threads' Process = multiprocessing.dummy.Process connection = multiprocessing.dummy.connection @@ -4255,18 +4321,33 @@ def setUpModule(): multiprocessing.get_logger().setLevel(LOG_LEVEL) def tearDownModule(): + need_sleep = False + + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + multiprocessing.set_start_method(old_start_method[0], force=True) # pause a bit so we don't get warning about dangling threads/processes - time.sleep(0.5) + processes = set(multiprocessing.process._dangling) - set(dangling[0]) + if processes: + need_sleep = True + print('Warning -- Dangling processes: %s' % processes, + file=sys.stderr) + processes = None + + threads = set(threading._dangling) - set(dangling[1]) + if threads: + need_sleep = True + print('Warning -- Dangling threads: %s' % threads, + file=sys.stderr) + threads = None + + # Sleep 500 ms to give time to child processes to complete. + if need_sleep: + time.sleep(0.5) multiprocessing.process._cleanup() - gc.collect() - tmp = set(multiprocessing.process._dangling) - set(dangling[0]) - if tmp: - print('Dangling processes:', tmp, file=sys.stderr) - del tmp - tmp = set(threading._dangling) - set(dangling[1]) - if tmp: - print('Dangling threads:', tmp, file=sys.stderr) + test.support.gc_collect() remote_globs['setUpModule'] = setUpModule remote_globs['tearDownModule'] = tearDownModule From a6f045224a03f9b5f940c80ee142a2a4b4a9c9d8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 26 Jul 2017 05:05:09 +0200 Subject: [PATCH 0595/1345] bpo-30845: Enhance test_concurrent_futures cleanup (#2564) (#2880) * bpo-30845: reap_children() now logs warnings * bpo-30845: Enhance test_concurrent_futures cleanup In setUp() and tearDown() methods of test_concurrent_futures tests, make sure that tests don't leak threads nor processes. Clear explicitly the reference to the executor to make it that it's destroyed (to prevent "dangling threads" warning). (cherry picked from commit 3df9dec425b0254df1cdf41922fd8d6b08bf47e4) --- Lib/test/support/__init__.py | 3 ++- Lib/test/test_concurrent_futures.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 677743bb389802..a5b8c46f9495b0 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2053,7 +2053,6 @@ def reap_children(): stick around to hog resources and create problems when looking for refleaks. """ - # Reap all our dead child processes so we don't leave zombies around. # These hog resources and might be causing some of the buildbots to die. if hasattr(os, 'waitpid'): @@ -2064,6 +2063,8 @@ def reap_children(): pid, status = os.waitpid(any_process, os.WNOHANG) if pid == 0: break + print("Warning -- reap_children() reaped child process %s" + % pid, file=sys.stderr) except: break diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 23e95b212447c8..92a3ebdd886c12 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -63,6 +63,8 @@ class ExecutorMixin: worker_count = 5 def setUp(self): + self._thread_cleanup = test.support.threading_setup() + self.t1 = time.time() try: self.executor = self.executor_type(max_workers=self.worker_count) @@ -72,11 +74,16 @@ def setUp(self): def tearDown(self): self.executor.shutdown(wait=True) + self.executor = None + dt = time.time() - self.t1 if test.support.verbose: print("%.2fs" % dt, end=' ') self.assertLess(dt, 60, "synchronization issue: test lasted too long") + test.support.threading_cleanup(*self._thread_cleanup) + test.support.reap_children() + def _prime_executor(self): # Make sure that the executor is ready to do work before running the # tests. This should reduce the probability of timeouts in the tests. From e42339d3a08a8fde3349722def85d7a8e49899be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 26 Jul 2017 05:38:22 +0200 Subject: [PATCH 0596/1345] [3.6] bpo-30595: Fix multiprocessing.Queue.get(timeout) (#2027) (#2881) * bpo-30595: Fix multiprocessing.Queue.get(timeout) (#2027) multiprocessing.Queue.get() with a timeout now polls its reader in non-blocking mode if it succeeded to aquire the lock but the acquire took longer than the timeout. Co-Authored-By: Grzegorz Grzywacz (cherry picked from commit 1b7863c3b6519c6e134c28cab8b8af0dea43e375) * bpo-30595: Increase test_queue_feeder_donot_stop_onexc() timeout (#2148) _test_multiprocessing.test_queue_feeder_donot_stop_onexc() now uses a timeout of 1 second on Queue.get(), instead of 0.1 second, for slow buildbots. (cherry picked from commit 8f6eeaf21cdf4aea25fdefeec814a1ce07453fe9) --- Lib/multiprocessing/queues.py | 2 +- .../next/Library/2017-07-26-04-46-12.bpo-30595.-zJ7d8.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-26-04-46-12.bpo-30595.-zJ7d8.rst diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 76684172528f93..513807cafecb6d 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -101,7 +101,7 @@ def get(self, block=True, timeout=None): try: if block: timeout = deadline - time.time() - if timeout < 0 or not self._poll(timeout): + if not self._poll(timeout): raise Empty elif not self._poll(): raise Empty diff --git a/Misc/NEWS.d/next/Library/2017-07-26-04-46-12.bpo-30595.-zJ7d8.rst b/Misc/NEWS.d/next/Library/2017-07-26-04-46-12.bpo-30595.-zJ7d8.rst new file mode 100644 index 00000000000000..4a2a3901cc6ee4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-26-04-46-12.bpo-30595.-zJ7d8.rst @@ -0,0 +1,3 @@ +multiprocessing.Queue.get() with a timeout now polls its reader in non- +blocking mode if it succeeded to aquire the lock but the acquire took longer +than the timeout. From 0dbfab2ac7398d5d8f7e93a024a3bc866f5dd472 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 25 Jul 2017 21:05:17 -0700 Subject: [PATCH 0597/1345] Add the link to asyncio source code in the docs (GH-2373) (GH-2884) (cherry picked from commit 627d2c8e8dc9cfe6c1cdfd9b34899eb30b600a88) --- Doc/library/asyncio-eventloop.rst | 2 ++ Doc/library/asyncio-eventloops.rst | 2 ++ Doc/library/asyncio-protocol.rst | 10 +++++++--- Doc/library/asyncio-queue.rst | 2 ++ Doc/library/asyncio-stream.rst | 2 ++ Doc/library/asyncio-subprocess.rst | 2 ++ Doc/library/asyncio-sync.rst | 2 ++ Doc/library/asyncio-task.rst | 4 ++++ 8 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index fa6a29604ca2ba..83bbb70b037928 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -5,6 +5,8 @@ Base Event Loop =============== +**Source code:** :source:`Lib/asyncio/events.py` + The event loop is the central execution device provided by :mod:`asyncio`. It provides multiple facilities, including: diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst index 1dc18fce79d021..d74fcb1e07f249 100644 --- a/Doc/library/asyncio-eventloops.rst +++ b/Doc/library/asyncio-eventloops.rst @@ -3,6 +3,8 @@ Event loops =========== +**Source code:** :source:`Lib/asyncio/events.py` + Event loop functions -------------------- diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index dfe37c239d6b39..cd84ae76b5d86e 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1,8 +1,12 @@ .. currentmodule:: asyncio -++++++++++++++++++++++++++++++++++++++++++++++ -Transports and protocols (callback based API) -++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++ +Transports and protocols (callback based API) ++++++++++++++++++++++++++++++++++++++++++++++ + +**Source code:** :source:`Lib/asyncio/transports.py` + +**Source code:** :source:`Lib/asyncio/protocols.py` .. _asyncio-transport: diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index f11c09ac29055e..ea78755008244d 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -3,6 +3,8 @@ Queues ====== +**Source code:** :source:`Lib/asyncio/queues.py` + Queues: * :class:`Queue` diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 6177b4bb0f8b14..491afdd610ca08 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -6,6 +6,8 @@ Streams (coroutine based API) +++++++++++++++++++++++++++++ +**Source code:** :source:`Lib/asyncio/streams.py` + Stream functions ================ diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 16ba9a3cd6c7f2..1c1d0be918da74 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -5,6 +5,8 @@ Subprocess ========== +**Source code:** :source:`Lib/asyncio/subprocess.py` + Windows event loop ------------------ diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 09093521524a89..14e3defbf4118a 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -4,6 +4,8 @@ Synchronization primitives ========================== +**Source code:** :source:`Lib/asyncio/locks.py` + Locks: * :class:`Lock` diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 804f1925b40929..5298c11058c4b1 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -3,6 +3,10 @@ Tasks and coroutines ==================== +**Source code:** :source:`Lib/asyncio/tasks.py` + +**Source code:** :source:`Lib/asyncio/coroutines.py` + .. _coroutine: Coroutines From c52cea49544621b612c7f17f45a0c2b8b61a6c67 Mon Sep 17 00:00:00 2001 From: Utkarsh Upadhyay Date: Wed, 26 Jul 2017 13:49:16 +0200 Subject: [PATCH 0598/1345] [3.6] bpo-30822: Fix testing of datetime module. (GH-2530) (GH-2783) (#2816) * [3.6] bpo-30822: Fix testing of datetime module. (GH-2530) (GH-2783) Only C implementation was tested. (cherry picked from commit 287c5594edc1ca08db64d1f4739cc36bfe75ae75) * [3.6] bpo-30822: Fix testing of datetime module. (GH-2530) (GH-2783) Only C implementation was tested.. (cherry picked from commit 287c5594edc1ca08db64d1f4739cc36bfe75ae75) --- Lib/datetime.py | 3 ++- Lib/test/datetimetester.py | 12 +++++++----- Lib/test/test_datetime.py | 8 +++++--- Misc/ACKS | 1 + 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 5d5579c1c6f4be..b95536fb7afc02 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -2271,7 +2271,8 @@ def _name_from_offset(delta): _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month, _days_before_year, _days_in_month, _format_time, _is_leap, _isoweek1monday, _math, _ord2ymd, - _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord) + _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, + _divide_and_round) # XXX Since import * above excludes names that start with _, # docstring does not get overwritten. In the future, it may be # appropriate to maintain a single module level docstring and diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index bccd97aa3c7f60..b25e6c17136790 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -61,8 +61,9 @@ def test_constants(self): self.assertEqual(datetime.MAXYEAR, 9999) def test_name_cleanup(self): - if '_Fast' not in str(self): - return + if '_Pure' in self.__class__.__name__: + self.skipTest('Only run for Fast C implementation') + datetime = datetime_module names = set(name for name in dir(datetime) if not name.startswith('__') and not name.endswith('__')) @@ -72,8 +73,9 @@ def test_name_cleanup(self): self.assertEqual(names - allowed, set([])) def test_divide_and_round(self): - if '_Fast' in str(self): - return + if '_Fast' in self.__class__.__name__: + self.skipTest('Only run for Pure Python implementation') + dar = datetime_module._divide_and_round self.assertEqual(dar(-10, -3), 3) @@ -2851,7 +2853,7 @@ def tzname(self, dt): return self.tz self.assertRaises(TypeError, t.strftime, "%Z") # Issue #6697: - if '_Fast' in str(self): + if '_Fast' in self.__class__.__name__: Badtzname.tz = '\ud800' self.assertRaises(ValueError, t.strftime, "%Z") diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 242e1bba035bf9..2d00b56c1cce1a 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -20,7 +20,7 @@ # XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might # not believe this, but in spite of all the sys.modules trickery running a _Pure # test last will leave a mix of pure and native datetime stuff lying around. -test_classes = [] +all_test_classes = [] for module, suffix in zip(test_modules, test_suffixes): test_classes = [] @@ -33,7 +33,8 @@ suit = cls() test_classes.extend(type(test) for test in suit) for cls in test_classes: - cls.__name__ = name + suffix + cls.__name__ += suffix + cls.__qualname__ += suffix @classmethod def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() @@ -46,9 +47,10 @@ def tearDownClass(cls_): sys.modules.update(cls_._save_sys_modules) cls.setUpClass = setUpClass cls.tearDownClass = tearDownClass + all_test_classes.extend(test_classes) def test_main(): - run_unittest(*test_classes) + run_unittest(*all_test_classes) if __name__ == "__main__": test_main() diff --git a/Misc/ACKS b/Misc/ACKS index f4ccdb88cdc962..b6b355b614b820 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1583,6 +1583,7 @@ Doobee R. Tzeck Eren Türkay Lionel Ulmer Adnan Umer +Utkarsh Upadhyay Roger Upole Daniel Urban Michael Urman From 8c2d4cf092c5f0335e7982392a33927579c4d512 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 26 Jul 2017 21:11:25 +0900 Subject: [PATCH 0599/1345] [3.6] bpo-30119: fix ftplib.FTP.putline() to throw an error for a illegal command (#1214) (#2886) --- Lib/ftplib.py | 2 ++ Lib/test/test_ftplib.py | 6 +++++- .../next/Library/2017-07-26-15-15-00.bpo-30119.DZ6C_S.rst | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-26-15-15-00.bpo-30119.DZ6C_S.rst diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 8f36f537e8a54f..a02e595cb0226f 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -186,6 +186,8 @@ def sanitize(self, s): # Internal: send one line to the server, appending CRLF def putline(self, line): + if '\r' in line or '\n' in line: + raise ValueError('an illegal newline character should not be contained') line = line + CRLF if self.debugging > 1: print('*put*', self.sanitize(line)) diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 12fabc5e8be3d6..a561e9efa0363a 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -484,6 +484,9 @@ def test_sanitize(self): self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****')) def test_exceptions(self): + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r0') self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400') self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499') self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500') @@ -492,7 +495,8 @@ def test_exceptions(self): def test_all_errors(self): exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, - ftplib.error_proto, ftplib.Error, OSError, EOFError) + ftplib.error_proto, ftplib.Error, OSError, + EOFError) for x in exceptions: try: raise x('exception not included in all_errors set') diff --git a/Misc/NEWS.d/next/Library/2017-07-26-15-15-00.bpo-30119.DZ6C_S.rst b/Misc/NEWS.d/next/Library/2017-07-26-15-15-00.bpo-30119.DZ6C_S.rst new file mode 100644 index 00000000000000..a37d37038424c6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-26-15-15-00.bpo-30119.DZ6C_S.rst @@ -0,0 +1,2 @@ +ftplib.FTP.putline() now throws ValueError on commands that contains CR or +LF. Patch by Dong-hee Na. From bb7fd3f4d08091b2b68333267a4d5fdef03bda44 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 26 Jul 2017 14:20:41 -0700 Subject: [PATCH 0600/1345] Fix build batch files (#2750) (#2897) * Enable building MSI, zip and nuget packages when Py_OutDir is set. * Restore the --build option, which got reverted at some point. * Ensure output directory is created. * Enables BuildForDaily and DailyBuildVersion options for nuget package. --- Tools/msi/buildrelease.bat | 7 ++++--- Tools/msi/make_zip.proj | 3 ++- Tools/msi/make_zip.py | 31 +++++++++++++++++-------------- Tools/nuget/build.bat | 5 +++-- Tools/nuget/make_pkg.proj | 6 +++++- Tools/nuget/pythondaily.nuspec | 18 ++++++++++++++++++ 6 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 Tools/nuget/pythondaily.nuspec diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 1f5278207b474b..d19950b659c037 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -29,6 +29,7 @@ set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename} set D=%~dp0 set PCBUILD=%D%..\..\PCBuild\ +if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD% set EXTERNALS=%D%..\..\externals\windows-installer\ set BUILDX86= @@ -110,12 +111,12 @@ exit /B 0 if "%1" EQU "x86" ( set PGO= - set BUILD=%PCBUILD%win32\ + set BUILD=%Py_OutDir%win32\ set BUILD_PLAT=Win32 set OUTDIR_PLAT=win32 set OBJDIR_PLAT=x86 ) else ( - set BUILD=%PCBUILD%amd64\ + set BUILD=%Py_OutDir%amd64\ set PGO=%~2 set BUILD_PLAT=x64 set OUTDIR_PLAT=amd64 @@ -167,7 +168,7 @@ if not "%SKIPBUILD%" EQU "1" ( if "%OUTDIR_PLAT%" EQU "win32" ( %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B -) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( +) else if not exist "%Py_OutDir%win32\en-us\launcher.msi" ( %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index b3588b7a0bae90..21411173421919 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -16,8 +16,9 @@ $(OutputPath)\$(TargetName)$(TargetExt) rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" - $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) + $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -b "$(BuildPath.TrimEnd(`\`))" set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform)     diff --git a/Tools/msi/make_zip.py b/Tools/msi/make_zip.py index 710e4a5c23ee28..0698c1407375f8 100644 --- a/Tools/msi/make_zip.py +++ b/Tools/msi/make_zip.py @@ -106,23 +106,23 @@ def include_in_tools(p): BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info) FULL_LAYOUT = [ - ('/', 'PCBuild/$arch', 'python.exe', is_not_debug), - ('/', 'PCBuild/$arch', 'pythonw.exe', is_not_debug), - ('/', 'PCBuild/$arch', 'python{}.dll'.format(sys.version_info.major), is_not_debug), - ('/', 'PCBuild/$arch', '{}.dll'.format(BASE_NAME), is_not_debug), - ('DLLs/', 'PCBuild/$arch', '*.pyd', is_not_debug), - ('DLLs/', 'PCBuild/$arch', '*.dll', is_not_debug_or_python), + ('/', '$build', 'python.exe', is_not_debug), + ('/', '$build', 'pythonw.exe', is_not_debug), + ('/', '$build', 'python{}.dll'.format(sys.version_info.major), is_not_debug), + ('/', '$build', '{}.dll'.format(BASE_NAME), is_not_debug), + ('DLLs/', '$build', '*.pyd', is_not_debug), + ('DLLs/', '$build', '*.dll', is_not_debug_or_python), ('include/', 'include', '*.h', None), ('include/', 'PC', 'pyconfig.h', None), ('Lib/', 'Lib', '**/*', include_in_lib), - ('libs/', 'PCBuild/$arch', '*.lib', include_in_libs), + ('libs/', '$build', '*.lib', include_in_libs), ('Tools/', 'Tools', '**/*', include_in_tools), ] EMBED_LAYOUT = [ - ('/', 'PCBuild/$arch', 'python*.exe', is_not_debug), - ('/', 'PCBuild/$arch', '*.pyd', is_not_debug), - ('/', 'PCBuild/$arch', '*.dll', is_not_debug), + ('/', '$build', 'python*.exe', is_not_debug), + ('/', '$build', '*.pyd', is_not_debug), + ('/', '$build', '*.dll', is_not_debug), ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_embeddable_lib), ] @@ -187,15 +187,15 @@ def main(): parser.add_argument('-o', '--out', metavar='file', help='The name of the output archive', type=Path, default=None) parser.add_argument('-t', '--temp', metavar='dir', help='A directory to temporarily extract files into', type=Path, default=None) parser.add_argument('-e', '--embed', help='Create an embedding layout', action='store_true', default=False) - parser.add_argument('-a', '--arch', help='Specify the architecture to use (win32/amd64)', type=str, default="win32") + parser.add_argument('-b', '--build', help='Specify the build directory', type=Path, default=None) ns = parser.parse_args() source = ns.source or (Path(__file__).resolve().parent.parent.parent) out = ns.out - arch = ns.arch + build = ns.build or Path(sys.exec_prefix) assert isinstance(source, Path) assert not out or isinstance(out, Path) - assert isinstance(arch, str) + assert isinstance(build, Path) if ns.temp: temp = ns.temp @@ -218,7 +218,10 @@ def main(): try: for t, s, p, c in layout: - fs = source / s.replace("$arch", arch) + if s == '$build': + fs = build + else: + fs = source / s files = rglob(fs, p, c) extra_files = [] if s == 'Lib' and p == '**/*': diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 54498ca28a2471..0b4f622e07841c 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -2,6 +2,7 @@ setlocal set D=%~dp0 set PCBUILD=%D%..\..\PCBuild\ +if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD% set BUILDX86= set BUILDX64= @@ -28,7 +29,7 @@ if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" if defined BUILDX86 ( if defined REBUILD ( call "%PCBUILD%build.bat" -e -r - ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e + ) else if not exist "%Py_OutDir%win32\python.exe" call "%PCBUILD%build.bat" -e if errorlevel 1 goto :eof %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% @@ -37,7 +38,7 @@ if defined BUILDX86 ( if defined BUILDX64 ( if defined REBUILD ( call "%PCBUILD%build.bat" -p x64 -e -r - ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e + ) else if not exist "%Py_OutDir%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e if errorlevel 1 goto :eof %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index 464ef0456af47a..3750d8dcc860e8 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -4,6 +4,7 @@ {10487945-15D1-4092-A214-338395C4116B} python $(OutputName)x86 + $(OutputName)daily false true @@ -15,6 +16,7 @@ $(ExternalsDir)\windows-installer\nuget\nuget.exe $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) $(NuspecVersion)-$(ReleaseLevelName) + $(MajorVersionNumber).$(MinorVersionNumber).$(DailyBuildVersion) false $(OutputName).$(NuspecVersion) .nupkg @@ -23,7 +25,7 @@ rmdir /q/s "$(IntermediateOutputPath)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\..\msi\make_zip.py" - $(PythonArguments) -t "$(IntermediateOutputPath)" -a $(ArchName) + $(PythonArguments) -t "$(IntermediateOutputPath)" -b "$(BuildPath.TrimEnd(`\`))" "$(IntermediateOutputPath)\python.exe" -B -c "import sys; sys.path.append(r'$(PySourcePath)\Lib'); import ensurepip; ensurepip._main()" "$(IntermediateOutputPath)\python.exe" -B -m pip install -U $(Packages) @@ -35,7 +37,9 @@ $(NugetArguments) -NoPackageAnalysis -NonInteractive set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) + $(Environment)%0D%0Amkdir "$(OutputPath.Trim(`\`))" >nul 2>nul diff --git a/Tools/nuget/pythondaily.nuspec b/Tools/nuget/pythondaily.nuspec new file mode 100644 index 00000000000000..2634ed14a88d6c --- /dev/null +++ b/Tools/nuget/pythondaily.nuspec @@ -0,0 +1,18 @@ + + + + pythondaily + Python (Daily build) + 0.0.0.0 + Python Software Foundation + https://docs.python.org/3/license.html + https://www.python.org/ + false + Installs an unsigned, untested build of Python for test purposes only. + https://www.python.org/static/favicon.ico + python + + + + + From 25de5baf3eaebddbf879aacf49c0f614f922dc42 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Thu, 27 Jul 2017 02:27:08 +0300 Subject: [PATCH 0601/1345] bpo-30980: Fix double close in asyncore.file_wrapper (#2789) (#2898) * bpo-30980: Fix close test to fail test_close_twice was not considering the fact that file_wrapper is duping the file descriptor. Closing the original descriptor left the duped one open, hiding the fact that close protection is not effective. * bpo-30980: Fix double close protection Invalidated self.fd before closing, handling correctly the case when os.close raises. * bpo-30980: Fix fd leak introduced in the fixed test --- Lib/asyncore.py | 3 ++- Lib/test/test_asyncore.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/asyncore.py b/Lib/asyncore.py index 705e4068130325..03d16838b739f9 100644 --- a/Lib/asyncore.py +++ b/Lib/asyncore.py @@ -619,8 +619,9 @@ def getsockopt(self, level, optname, buflen=None): def close(self): if self.fd < 0: return - os.close(self.fd) + fd = self.fd self.fd = -1 + os.close(fd) def fileno(self): return self.fd diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index dc2f716e0bb828..07edf2275bea0e 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -433,7 +433,10 @@ def test_close_twice(self): f = asyncore.file_wrapper(fd) os.close(fd) - f.close() + os.close(f.fd) # file_wrapper dupped fd + with self.assertRaises(OSError): + f.close() + self.assertEqual(f.fd, -1) # calling close twice should not fail f.close() From 0243bea55dc340067247e635442f2a227705315a Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 26 Jul 2017 20:53:13 -0400 Subject: [PATCH 0602/1345] [3.6] bpo-30853: IDLE: Factor a VarTrace class from configdialog.ConfigDialog. (GH-2872) (#2903) The new class manages pairs of tk Variables and trace callbacks. It is completely covered by new tests. (cherry picked from commit 45bf723) --- Lib/idlelib/configdialog.py | 55 +++++++++++++ Lib/idlelib/idle_test/test_configdialog.py | 94 +++++++++++++++++++++- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 1832e156dc6fac..f98af4600ee805 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1846,6 +1846,61 @@ def save_all_changed_extensions(self): self.ext_userCfg.Save() +class VarTrace: + """Maintain Tk variables trace state.""" + + def __init__(self): + """Store Tk variables and callbacks. + + untraced: List of tuples (var, callback) + that do not have the callback attached + to the Tk var. + traced: List of tuples (var, callback) where + that callback has been attached to the var. + """ + self.untraced = [] + self.traced = [] + + def add(self, var, callback): + """Add (var, callback) tuple to untraced list. + + Args: + var: Tk variable instance. + callback: Function to be used as a callback or + a tuple with IdleConf values for default + callback. + + Return: + Tk variable instance. + """ + if isinstance(callback, tuple): + callback = self.make_callback(var, callback) + self.untraced.append((var, callback)) + return var + + @staticmethod + def make_callback(var, config): + "Return default callback function to add values to changes instance." + def default_callback(*params): + "Add config values to changes instance." + changes.add_option(*config, var.get()) + return default_callback + + def attach(self): + "Attach callback to all vars that are not traced." + while self.untraced: + var, callback = self.untraced.pop() + var.trace_add('write', callback) + self.traced.append((var, callback)) + + def detach(self): + "Remove callback from traced vars." + while self.traced: + var, callback = self.traced.pop() + var.trace_remove('write', var.trace_info()[0][1]) + self.untraced.append((var, callback)) + + help_common = '''\ When you click either the Apply or Ok buttons, settings in this dialog that are different from IDLE's default are saved in diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 54b2d78d667fae..ce02ae4a8e619a 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -3,11 +3,12 @@ Half the class creates dialog, half works with user customizations. Coverage: 46% just by creating dialog, 60% with current tests. """ -from idlelib.configdialog import ConfigDialog, idleConf, changes +from idlelib.configdialog import ConfigDialog, idleConf, changes, VarTrace from test.support import requires requires('gui') -from tkinter import Tk +from tkinter import Tk, IntVar, BooleanVar import unittest +from unittest import mock import idlelib.config as config from idlelib.idle_test.mock_idle import Func @@ -248,5 +249,94 @@ def test_editor_size(self): #def test_help_sources(self): pass # TODO +class TestVarTrace(unittest.TestCase): + + def setUp(self): + changes.clear() + self.v1 = IntVar(root) + self.v2 = BooleanVar(root) + self.called = 0 + self.tracers = VarTrace() + + def tearDown(self): + del self.v1, self.v2 + + def var_changed_increment(self, *params): + self.called += 13 + + def var_changed_boolean(self, *params): + pass + + def test_init(self): + self.assertEqual(self.tracers.untraced, []) + self.assertEqual(self.tracers.traced, []) + + def test_add(self): + tr = self.tracers + func = Func() + cb = tr.make_callback = mock.Mock(return_value=func) + + v1 = tr.add(self.v1, self.var_changed_increment) + self.assertIsInstance(v1, IntVar) + v2 = tr.add(self.v2, self.var_changed_boolean) + self.assertIsInstance(v2, BooleanVar) + + v3 = IntVar(root) + v3 = tr.add(v3, ('main', 'section', 'option')) + cb.assert_called_once() + cb.assert_called_with(v3, ('main', 'section', 'option')) + + expected = [(v1, self.var_changed_increment), + (v2, self.var_changed_boolean), + (v3, func)] + self.assertEqual(tr.traced, []) + self.assertEqual(tr.untraced, expected) + + del tr.make_callback + + def test_make_callback(self): + tr = self.tracers + cb = tr.make_callback(self.v1, ('main', 'section', 'option')) + self.assertTrue(callable(cb)) + self.v1.set(42) + # Not attached, so set didn't invoke the callback. + self.assertNotIn('section', changes['main']) + # Invoke callback manually. + cb() + self.assertIn('section', changes['main']) + self.assertEqual(changes['main']['section']['option'], '42') + + def test_attach_detach(self): + tr = self.tracers + v1 = tr.add(self.v1, self.var_changed_increment) + v2 = tr.add(self.v2, self.var_changed_boolean) + expected = [(v1, self.var_changed_increment), + (v2, self.var_changed_boolean)] + + # Attach callbacks and test call increment. + tr.attach() + self.assertEqual(tr.untraced, []) + self.assertCountEqual(tr.traced, expected) + v1.set(1) + self.assertEqual(v1.get(), 1) + self.assertEqual(self.called, 13) + + # Check that only one callback is attached to a variable. + # If more than one callback were attached, then var_changed_increment + # would be called twice and the counter would be 2. + self.called = 0 + tr.attach() + v1.set(1) + self.assertEqual(self.called, 13) + + # Detach callbacks. + self.called = 0 + tr.detach() + self.assertEqual(tr.traced, []) + self.assertCountEqual(tr.untraced, expected) + v1.set(1) + self.assertEqual(self.called, 0) + + if __name__ == '__main__': unittest.main(verbosity=2) From 0c4c65104c951c70f3a2a2fd1a45bdf6540f3a8e Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 26 Jul 2017 21:41:26 -0400 Subject: [PATCH 0603/1345] [3.6] bpo-31003: IDLE - Add more tests for General tab (GH-2859) (#2906) * In configdialog: Document causal pathways in create_page_general. Move related functions to follow this. Simplify some attribute names. * In test_configdialog: Add tests for load and helplist functions. Coverage for the general tab is now complete, and 63% overall. (cherry picked from commit 2bc8f0e) --- Lib/idlelib/configdialog.py | 379 +++++++++--------- Lib/idlelib/idle_test/mock_idle.py | 6 +- Lib/idlelib/idle_test/test_configdialog.py | 192 ++++++++- .../2017-07-25-01-28-35.bpo-31003.bYINVH.rst | 1 + 4 files changed, 380 insertions(+), 198 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-25-01-28-35.bpo-31003.bYINVH.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index f98af4600ee805..daaa34459e2fa6 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -160,7 +160,7 @@ def create_page_font_tab(self): corresponding aspect of the font sample on this page and highlight sample on highlight page. - Load_font_cfg initializes font vars and widgets from + Funtion load_font_cfg initializes font vars and widgets from idleConf entries and tk. Fontlist: mouse button 1 click or up or down key invoke @@ -470,7 +470,7 @@ def tem(event, elem=element): event.widget.winfo_toplevel().highlight_target.set(elem) text.tag_bind( self.theme_elements[element][0], '', tem) - text.config(state=DISABLED) + text['state'] = DISABLED self.frame_color_set = Frame(frame_custom, relief=SOLID, borderwidth=1) frame_fg_bg_toggle = Frame(frame_custom) button_set_color = Button( @@ -650,59 +650,61 @@ def create_page_keys(self): frames[1].pack(side=TOP, fill=X, expand=True, pady=2) return frame + def create_page_general(self): """Return frame of widgets for General tab. - Tk Variables: - win_width: Initial window width in characters. - win_height: Initial window height in characters. - startup_edit: Selector for opening in editor or shell mode. - autosave: Selector for save prompt popup when using Run. - - Methods: - load_general_config: - help_source_selected: Bound to list_help button release. - set_helplist_button_states: Toggle based on list. - helplist_item_edit: Command for button_helplist_edit. - helplist_item_add: Command for button_helplist_add. - helplist_item_remove: Command for button_helplist_remove. - update_user_help_changed_items: Fill in changes. + Enable users to provisionally change general options. Function + load_general_cfg intializes tk variables and helplist using + idleConf. Radiobuttons startup_shell_on and startup_editor_on + set var startup_edit. Radiobuttons save_ask_on and save_auto_on + set var autosave. Entry boxes win_width_int and win_height_int + set var win_width and win_height. Setting var_name invokes the + var_changed_var_name callback that adds option to changes. + + Helplist: load_general_cfg loads list user_helplist with + name, position pairs and copies names to listbox helplist. + Clicking a name invokes help_source selected. Clicking + button_helplist_name invokes helplist_item_name, which also + changes user_helplist. These functions all call + set_add_delete_state. All but load call update_help_changes to + rewrite changes['main']['HelpFiles']. Widget Structure: (*) widgets bound to self frame frame_run: LabelFrame startup_title: Label - (*)radio_startup_edit: Radiobutton - startup_edit - (*)radio_startup_shell: Radiobutton - startup_edit + (*)startup_editor_on: Radiobutton - startup_edit + (*)startup_shell_on: Radiobutton - startup_edit frame_save: LabelFrame run_save_title: Label - (*)radio_save_ask: Radiobutton - autosave - (*)radio_save_auto: Radiobutton - autosave + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave frame_win_size: LabelFrame win_size_title: Label win_width_title: Label - (*)entry_win_width: Entry - win_width + (*)win_width_int: Entry - win_width win_height_title: Label - (*)entry_win_height: Entry - win_height + (*)win_height_int: Entry - win_height frame_help: LabelFrame frame_helplist: Frame frame_helplist_buttons: Frame (*)button_helplist_edit (*)button_helplist_add (*)button_helplist_remove + (*)helplist: ListBox scroll_helplist: Scrollbar - (*)list_help: ListBox """ parent = self.parent - self.win_width = StringVar(parent) - self.win_height = StringVar(parent) self.startup_edit = IntVar(parent) self.autosave = IntVar(parent) + self.win_width = StringVar(parent) + self.win_height = StringVar(parent) - #widget creation - #body + # Create widgets: + # body. frame = self.tab_pages.pages['General'].frame - #body section frames + # body section frames. frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Startup Preferences ') frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, @@ -710,41 +712,41 @@ def create_page_general(self): frame_win_size = Frame(frame, borderwidth=2, relief=GROOVE) frame_help = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Additional Help Sources ') - #frame_run + # frame_run. startup_title = Label(frame_run, text='At Startup') - self.radio_startup_edit = Radiobutton( + self.startup_editor_on = Radiobutton( frame_run, variable=self.startup_edit, value=1, text="Open Edit Window") - self.radio_startup_shell = Radiobutton( + self.startup_shell_on = Radiobutton( frame_run, variable=self.startup_edit, value=0, text='Open Shell Window') - #frame_save + # frame_save. run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.radio_save_ask = Radiobutton( + self.save_ask_on = Radiobutton( frame_save, variable=self.autosave, value=0, text="Prompt to Save") - self.radio_save_auto = Radiobutton( + self.save_auto_on = Radiobutton( frame_save, variable=self.autosave, value=1, text='No Prompt') - #frame_win_size + # frame_win_size. win_size_title = Label( frame_win_size, text='Initial Window Size (in characters)') win_width_title = Label(frame_win_size, text='Width') - self.entry_win_width = Entry( + self.win_width_int = Entry( frame_win_size, textvariable=self.win_width, width=3) win_height_title = Label(frame_win_size, text='Height') - self.entry_win_height = Entry( + self.win_height_int = Entry( frame_win_size, textvariable=self.win_height, width=3) - #frame_help + # frame_help. frame_helplist = Frame(frame_help) frame_helplist_buttons = Frame(frame_helplist) - scroll_helplist = Scrollbar(frame_helplist) - self.list_help = Listbox( + self.helplist = Listbox( frame_helplist, height=5, takefocus=FALSE, exportselection=FALSE) - scroll_helplist.config(command=self.list_help.yview) - self.list_help.config(yscrollcommand=scroll_helplist.set) - self.list_help.bind('', self.help_source_selected) + scroll_helplist = Scrollbar(frame_helplist) + scroll_helplist['command'] = self.helplist.yview + self.helplist['yscrollcommand'] = scroll_helplist.set + self.helplist.bind('', self.help_source_selected) self.button_helplist_edit = Button( frame_helplist_buttons, text='Edit', state=DISABLED, width=8, command=self.helplist_item_edit) @@ -755,36 +757,146 @@ def create_page_general(self): frame_helplist_buttons, text='Remove', state=DISABLED, width=8, command=self.helplist_item_remove) - #widget packing - #body + # Pack widgets: + # body. frame_run.pack(side=TOP, padx=5, pady=5, fill=X) frame_save.pack(side=TOP, padx=5, pady=5, fill=X) frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - #frame_run + # frame_run. startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.radio_startup_shell.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.radio_startup_edit.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frame_save + self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_save. run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.radio_save_auto.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.radio_save_ask.pack(side=RIGHT, anchor=W, padx=5, pady=5) - #frame_win_size + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_win_size. win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.entry_win_height.pack(side=RIGHT, anchor=E, padx=10, pady=5) + self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_height_title.pack(side=RIGHT, anchor=E, pady=5) - self.entry_win_width.pack(side=RIGHT, anchor=E, padx=10, pady=5) + self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_width_title.pack(side=RIGHT, anchor=E, pady=5) - #frame_help + # frame_help. frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) - self.list_help.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) self.button_helplist_add.pack(side=TOP, anchor=W) self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) + return frame + def load_general_cfg(self): + "Load current configuration settings for the general options." + # Set startup state. + self.startup_edit.set(idleConf.GetOption( + 'main', 'General', 'editor-on-startup', default=0, type='bool')) + # Set autosave state. + self.autosave.set(idleConf.GetOption( + 'main', 'General', 'autosave', default=0, type='bool')) + # Set initial window size. + self.win_width.set(idleConf.GetOption( + 'main', 'EditorWindow', 'width', type='int')) + self.win_height.set(idleConf.GetOption( + 'main', 'EditorWindow', 'height', type='int')) + # Set additional help sources. + self.user_helplist = idleConf.GetAllExtraHelpSourcesList() + self.helplist.delete(0, 'end') + for help_item in self.user_helplist: + self.helplist.insert(END, help_item[0]) + self.set_add_delete_state() + + def var_changed_startup_edit(self, *params): + "Store change to toggle for starting IDLE in the editor or shell." + value = self.startup_edit.get() + changes.add_option('main', 'General', 'editor-on-startup', value) + + def var_changed_autosave(self, *params): + "Store change to autosave." + value = self.autosave.get() + changes.add_option('main', 'General', 'autosave', value) + + def var_changed_win_width(self, *params): + "Store change to window width." + value = self.win_width.get() + changes.add_option('main', 'EditorWindow', 'width', value) + + def var_changed_win_height(self, *params): + "Store change to window height." + value = self.win_height.get() + changes.add_option('main', 'EditorWindow', 'height', value) + + def help_source_selected(self, event): + "Handle event for selecting additional help." + self.set_add_delete_state() + + def set_add_delete_state(self): + "Toggle the state for the help list buttons based on list entries." + if self.helplist.size() < 1: # No entries in list. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED + else: # Some entries. + if self.helplist.curselection(): # There currently is a selection. + self.button_helplist_edit['state'] = NORMAL + self.button_helplist_remove['state'] = NORMAL + else: # There currently is not a selection. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED + + def helplist_item_add(self): + """Handle add button for the help list. + + Query for name and location of new help sources and add + them to the list. + """ + help_source = HelpSource(self, 'New Help Source').result + if help_source: + self.user_helplist.append(help_source) + self.helplist.insert(END, help_source[0]) + self.update_help_changes() + + def helplist_item_edit(self): + """Handle edit button for the help list. + + Query with existing help source information and update + config if the values are changed. + """ + item_index = self.helplist.index(ANCHOR) + help_source = self.user_helplist[item_index] + new_help_source = HelpSource( + self, 'Edit Help Source', + menuitem=help_source[0], + filepath=help_source[1], + ).result + if new_help_source and new_help_source != help_source: + self.user_helplist[item_index] = new_help_source + self.helplist.delete(item_index) + self.helplist.insert(item_index, new_help_source[0]) + self.update_help_changes() + self.set_add_delete_state() # Selected will be un-selected + + def helplist_item_remove(self): + """Handle remove button for the help list. + + Delete the help list item from config. + """ + item_index = self.helplist.index(ANCHOR) + del(self.user_helplist[item_index]) + self.helplist.delete(item_index) + self.update_help_changes() + self.set_add_delete_state() + + def update_help_changes(self): + "Clear and rebuild the HelpFiles section in changes" + changes['main']['HelpFiles'] = {} + for num in range(1, len(self.user_helplist) + 1): + changes.add_option( + 'main', 'HelpFiles', str(num), + ';'.join(self.user_helplist[num-1][:2])) + + def attach_var_callbacks(self): "Attach callbacks to variables that can be changed." self.font_size.trace_add('write', self.var_changed_font) @@ -917,26 +1029,6 @@ def var_changed_are_keys_builtin(self, *params): else: self.var_changed_custom_keys() - def var_changed_win_width(self, *params): - "Store change to window width." - value = self.win_width.get() - changes.add_option('main', 'EditorWindow', 'width', value) - - def var_changed_win_height(self, *params): - "Store change to window height." - value = self.win_height.get() - changes.add_option('main', 'EditorWindow', 'height', value) - - def var_changed_startup_edit(self, *params): - "Store change to toggle for starting IDLE in the editor or shell." - value = self.startup_edit.get() - changes.add_option('main', 'General', 'editor-on-startup', value) - - def var_changed_autosave(self, *params): - "Store change to autosave." - value = self.autosave.get() - changes.add_option('main', 'General', 'autosave', value) - def set_theme_type(self): """Set available screen options based on builtin or custom theme. @@ -956,26 +1048,26 @@ def set_theme_type(self): load_theme_cfg """ if self.is_builtin_theme.get(): - self.opt_menu_theme_builtin.config(state=NORMAL) - self.opt_menu_theme_custom.config(state=DISABLED) - self.button_delete_custom_theme.config(state=DISABLED) + self.opt_menu_theme_builtin['state'] = NORMAL + self.opt_menu_theme_custom['state'] = DISABLED + self.button_delete_custom_theme['state'] = DISABLED else: - self.opt_menu_theme_builtin.config(state=DISABLED) - self.radio_theme_custom.config(state=NORMAL) - self.opt_menu_theme_custom.config(state=NORMAL) - self.button_delete_custom_theme.config(state=NORMAL) + self.opt_menu_theme_builtin['state'] = DISABLED + self.radio_theme_custom['state'] = NORMAL + self.opt_menu_theme_custom['state'] = NORMAL + self.button_delete_custom_theme['state'] = NORMAL def set_keys_type(self): "Set available screen options based on builtin or custom key set." if self.are_keys_builtin.get(): - self.opt_menu_keys_builtin.config(state=NORMAL) - self.opt_menu_keys_custom.config(state=DISABLED) - self.button_delete_custom_keys.config(state=DISABLED) + self.opt_menu_keys_builtin['state'] = NORMAL + self.opt_menu_keys_custom['state'] = DISABLED + self.button_delete_custom_keys['state'] = DISABLED else: - self.opt_menu_keys_builtin.config(state=DISABLED) - self.radio_keys_custom.config(state=NORMAL) - self.opt_menu_keys_custom.config(state=NORMAL) - self.button_delete_custom_keys.config(state=NORMAL) + self.opt_menu_keys_builtin['state'] = DISABLED + self.radio_keys_custom['state'] = NORMAL + self.opt_menu_keys_custom['state'] = NORMAL + self.button_delete_custom_keys['state'] = NORMAL def get_new_keys(self): """Handle event to change key binding for selected line. @@ -1037,7 +1129,7 @@ def save_as_new_key_set(self): def keybinding_selected(self, event): "Activate button to assign new keys to selected action." - self.button_new_keys.config(state=NORMAL) + self.button_new_keys['state'] = NORMAL def create_new_key_set(self, new_key_set_name): """Create a new custom key set with the given name. @@ -1115,7 +1207,7 @@ def delete_custom_keys(self): item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() if not item_list: - self.radio_keys_custom.config(state=DISABLED) + self.radio_keys_custom['state'] = DISABLED self.opt_menu_keys_custom.SetMenu(item_list, '- no custom keys -') else: self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) @@ -1164,7 +1256,7 @@ def delete_custom_theme(self): item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() if not item_list: - self.radio_theme_custom.config(state=DISABLED) + self.radio_theme_custom['state'] = DISABLED self.opt_menu_theme_custom.SetMenu(item_list, '- no custom themes -') else: self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) @@ -1303,12 +1395,12 @@ def set_highlight_target(self): load_theme_cfg """ if self.highlight_target.get() == 'Cursor': # bg not possible - self.radio_fg.config(state=DISABLED) - self.radio_bg.config(state=DISABLED) + self.radio_fg['state'] = DISABLED + self.radio_bg['state'] = DISABLED self.fg_bg_toggle.set(1) else: # Both fg and bg can be set. - self.radio_fg.config(state=NORMAL) - self.radio_bg.config(state=NORMAL) + self.radio_fg['state'] = NORMAL + self.radio_bg['state'] = NORMAL self.fg_bg_toggle.set(1) self.set_color_sample() @@ -1378,76 +1470,6 @@ def paint_theme_sample(self): self.highlight_sample.tag_config(element, **colors) self.set_color_sample() - def help_source_selected(self, event): - "Handle event for selecting additional help." - self.set_helplist_button_states() - - def set_helplist_button_states(self): - "Toggle the state for the help list buttons based on list entries." - if self.list_help.size() < 1: # No entries in list. - self.button_helplist_edit.config(state=DISABLED) - self.button_helplist_remove.config(state=DISABLED) - else: # Some entries. - if self.list_help.curselection(): # There currently is a selection. - self.button_helplist_edit.config(state=NORMAL) - self.button_helplist_remove.config(state=NORMAL) - else: # There currently is not a selection. - self.button_helplist_edit.config(state=DISABLED) - self.button_helplist_remove.config(state=DISABLED) - - def helplist_item_add(self): - """Handle add button for the help list. - - Query for name and location of new help sources and add - them to the list. - """ - help_source = HelpSource(self, 'New Help Source', - ).result - if help_source: - self.user_helplist.append((help_source[0], help_source[1])) - self.list_help.insert(END, help_source[0]) - self.update_user_help_changed_items() - self.set_helplist_button_states() - - def helplist_item_edit(self): - """Handle edit button for the help list. - - Query with existing help source information and update - config if the values are changed. - """ - item_index = self.list_help.index(ANCHOR) - help_source = self.user_helplist[item_index] - new_help_source = HelpSource( - self, 'Edit Help Source', - menuitem=help_source[0], - filepath=help_source[1], - ).result - if new_help_source and new_help_source != help_source: - self.user_helplist[item_index] = new_help_source - self.list_help.delete(item_index) - self.list_help.insert(item_index, new_help_source[0]) - self.update_user_help_changed_items() - self.set_helplist_button_states() - - def helplist_item_remove(self): - """Handle remove button for the help list. - - Delete the help list item from config. - """ - item_index = self.list_help.index(ANCHOR) - del(self.user_helplist[item_index]) - self.list_help.delete(item_index) - self.update_user_help_changed_items() - self.set_helplist_button_states() - - def update_user_help_changed_items(self): - "Clear and rebuild the HelpFiles section in changes" - changes['main']['HelpFiles'] = {} - for num in range(1, len(self.user_helplist) + 1): - changes.add_option( - 'main', 'HelpFiles', str(num), - ';'.join(self.user_helplist[num-1][:2])) - def load_theme_cfg(self): """Load current configuration settings for the theme options. @@ -1481,7 +1503,7 @@ def load_theme_cfg(self): item_list = idleConf.GetSectionList('user', 'highlight') item_list.sort() if not item_list: - self.radio_theme_custom.config(state=DISABLED) + self.radio_theme_custom['state'] = DISABLED self.custom_theme.set('- no custom themes -') else: self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) @@ -1515,7 +1537,7 @@ def load_key_cfg(self): item_list = idleConf.GetSectionList('user', 'keys') item_list.sort() if not item_list: - self.radio_keys_custom.config(state=DISABLED) + self.radio_keys_custom['state'] = DISABLED self.custom_keys.set('- no custom keys -') else: self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) @@ -1531,25 +1553,6 @@ def load_key_cfg(self): keyset_name = idleConf.CurrentKeys() self.load_keys_list(keyset_name) - def load_general_cfg(self): - "Load current configuration settings for the general options." - # Set startup state. - self.startup_edit.set(idleConf.GetOption( - 'main', 'General', 'editor-on-startup', default=1, type='bool')) - # Set autosave state. - self.autosave.set(idleConf.GetOption( - 'main', 'General', 'autosave', default=0, type='bool')) - # Set initial window size. - self.win_width.set(idleConf.GetOption( - 'main', 'EditorWindow', 'width', type='int')) - self.win_height.set(idleConf.GetOption( - 'main', 'EditorWindow', 'height', type='int')) - # Set additional help sources. - self.user_helplist = idleConf.GetAllExtraHelpSourcesList() - for help_item in self.user_helplist: - self.list_help.insert(END, help_item[0]) - self.set_helplist_button_states() - def load_configs(self): """Load configuration for each page. diff --git a/Lib/idlelib/idle_test/mock_idle.py b/Lib/idlelib/idle_test/mock_idle.py index 8f3147b1c5ce3d..f279a52fd511f0 100644 --- a/Lib/idlelib/idle_test/mock_idle.py +++ b/Lib/idlelib/idle_test/mock_idle.py @@ -13,14 +13,16 @@ class Func: self.args - capture positional arguments. self.kwds - capture keyword arguments. self.result - return or raise value set in __init__. + self.return_self - return self instead, to mock query class return. Most common use will probably be to mock instance methods. Given class instance, can set and delete as instance attribute. Mock_tk.Var and Mbox_func are special variants of this. ''' - def __init__(self, result=None): + def __init__(self, result=None, return_self=False): self.called = 0 self.result = result + self.return_self = return_self self.args = None self.kwds = None def __call__(self, *args, **kwds): @@ -29,6 +31,8 @@ def __call__(self, *args, **kwds): self.kwds = kwds if isinstance(self.result, BaseException): raise self.result + elif self.return_self: + return self else: return self.result diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index ce02ae4a8e619a..7296075fbecc21 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -1,16 +1,17 @@ """Test idlelib.configdialog. Half the class creates dialog, half works with user customizations. -Coverage: 46% just by creating dialog, 60% with current tests. +Coverage: 63%. """ -from idlelib.configdialog import ConfigDialog, idleConf, changes, VarTrace +from idlelib import configdialog from test.support import requires requires('gui') -from tkinter import Tk, IntVar, BooleanVar import unittest from unittest import mock -import idlelib.config as config from idlelib.idle_test.mock_idle import Func +from tkinter import Tk, IntVar, BooleanVar, DISABLED, NORMAL +from idlelib import config +from idlelib.configdialog import ConfigDialog, idleConf, changes, VarTrace # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. @@ -226,27 +227,200 @@ def setUp(self): class GeneralTest(unittest.TestCase): + """Test that general tab widgets enable users to make changes. + + Test that widget actions set vars, that var changes add + options to changes and that helplist works correctly. + """ + @classmethod + def setUpClass(cls): + # Mask instance methods used by help functions. + d = dialog + d.set = d.set_add_delete_state = Func() + d.upc = d.update_help_changes = Func() + + @classmethod + def tearDownClass(cls): + d = dialog + del d.set, d.set_add_delete_state + del d.upc, d.update_help_changes + d.helplist.delete(0, 'end') + d.user_helplist.clear() def setUp(self): changes.clear() + def test_load_general_cfg(self): + # Set to wrong values, load, check right values. + eq = self.assertEqual + d = dialog + d.startup_edit.set(1) + d.autosave.set(1) + d.win_width.set(1) + d.win_height.set(1) + d.helplist.insert('end', 'bad') + d.user_helplist = ['bad', 'worse'] + idleConf.SetOption('main', 'HelpFiles', '1', 'name;file') + d.load_general_cfg() + eq(d.startup_edit.get(), 0) + eq(d.autosave.get(), 0) + eq(d.win_width.get(), '80') + eq(d.win_height.get(), '40') + eq(d.helplist.get(0, 'end'), ('name',)) + eq(d.user_helplist, [('name', 'file', '1')]) + def test_startup(self): - dialog.radio_startup_edit.invoke() + dialog.startup_editor_on.invoke() self.assertEqual(mainpage, {'General': {'editor-on-startup': '1'}}) + changes.clear() + dialog.startup_shell_on.invoke() + self.assertEqual(mainpage, + {'General': {'editor-on-startup': '0'}}) def test_autosave(self): - dialog.radio_save_auto.invoke() + dialog.save_auto_on.invoke() self.assertEqual(mainpage, {'General': {'autosave': '1'}}) + dialog.save_ask_on.invoke() + self.assertEqual(mainpage, {'General': {'autosave': '0'}}) def test_editor_size(self): - dialog.entry_win_height.insert(0, '1') + dialog.win_height_int.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'height': '140'}}) changes.clear() - dialog.entry_win_width.insert(0, '1') + dialog.win_width_int.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'width': '180'}}) - #def test_help_sources(self): pass # TODO + def test_source_selected(self): + d = dialog + d.set = d.set_add_delete_state + d.upc = d.update_help_changes + helplist = d.helplist + helplist.insert(0, 'source') + helplist.activate(0) + + helplist.focus_force() + helplist.see(0) + helplist.update() + x, y, dx, dy = helplist.bbox(0) + x += dx // 2 + y += dy // 2 + d.set.called = d.upc.called = 0 + helplist.event_generate('', x=x, y=y) + helplist.event_generate('', x=x, y=y) + self.assertEqual(helplist.get('anchor'), 'source') + self.assertTrue(d.set.called) + self.assertFalse(d.upc.called) + + def test_set_add_delete_state(self): + # Call with 0 items, 1 unselected item, 1 selected item. + eq = self.assertEqual + d = dialog + del d.set_add_delete_state # Unmask method. + sad = d.set_add_delete_state + h = d.helplist + + h.delete(0, 'end') + sad() + eq(d.button_helplist_edit['state'], DISABLED) + eq(d.button_helplist_remove['state'], DISABLED) + + h.insert(0, 'source') + sad() + eq(d.button_helplist_edit['state'], DISABLED) + eq(d.button_helplist_remove['state'], DISABLED) + + h.selection_set(0) + sad() + eq(d.button_helplist_edit['state'], NORMAL) + eq(d.button_helplist_remove['state'], NORMAL) + d.set_add_delete_state = Func() # Mask method. + + def test_helplist_item_add(self): + # Call without and twice with HelpSource result. + # Double call enables check on order. + eq = self.assertEqual + orig_helpsource = configdialog.HelpSource + hs = configdialog.HelpSource = Func(return_self=True) + d = dialog + d.helplist.delete(0, 'end') + d.user_helplist.clear() + d.set.called = d.upc.called = 0 + + hs.result = '' + d.helplist_item_add() + self.assertTrue(list(d.helplist.get(0, 'end')) == + d.user_helplist == []) + self.assertFalse(d.upc.called) + + hs.result = ('name1', 'file1') + d.helplist_item_add() + hs.result = ('name2', 'file2') + d.helplist_item_add() + eq(d.helplist.get(0, 'end'), ('name1', 'name2')) + eq(d.user_helplist, [('name1', 'file1'), ('name2', 'file2')]) + eq(d.upc.called, 2) + self.assertFalse(d.set.called) + + configdialog.HelpSource = orig_helpsource + + def test_helplist_item_edit(self): + # Call without and with HelpSource change. + eq = self.assertEqual + orig_helpsource = configdialog.HelpSource + hs = configdialog.HelpSource = Func(return_self=True) + d = dialog + d.helplist.delete(0, 'end') + d.helplist.insert(0, 'name1') + d.helplist.selection_set(0) + d.helplist.selection_anchor(0) + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.set.called = d.upc.called = 0 + + hs.result = '' + d.helplist_item_edit() + hs.result = ('name1', 'file1') + d.helplist_item_edit() + eq(d.helplist.get(0, 'end'), ('name1',)) + eq(d.user_helplist, [('name1', 'file1')]) + self.assertFalse(d.upc.called) + + hs.result = ('name2', 'file2') + d.helplist_item_edit() + eq(d.helplist.get(0, 'end'), ('name2',)) + eq(d.user_helplist, [('name2', 'file2')]) + self.assertTrue(d.upc.called == d.set.called == 1) + + configdialog.HelpSource = orig_helpsource + + def test_helplist_item_remove(self): + eq = self.assertEqual + d = dialog + d.helplist.delete(0, 'end') + d.helplist.insert(0, 'name1') + d.helplist.selection_set(0) + d.helplist.selection_anchor(0) + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.set.called = d.upc.called = 0 + + d.helplist_item_remove() + eq(d.helplist.get(0, 'end'), ()) + eq(d.user_helplist, []) + self.assertTrue(d.upc.called == d.set.called == 1) + + def test_update_help_changes(self): + d = dialog + del d.update_help_changes + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.user_helplist.append(('name2', 'file2')) + + d.update_help_changes() + self.assertEqual(mainpage['HelpFiles'], + {'1': 'name1;file1', '2': 'name2;file2'}) + d.update_help_changes = Func() class TestVarTrace(unittest.TestCase): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-25-01-28-35.bpo-31003.bYINVH.rst b/Misc/NEWS.d/next/IDLE/2017-07-25-01-28-35.bpo-31003.bYINVH.rst new file mode 100644 index 00000000000000..f3dab0fd9e6627 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-25-01-28-35.bpo-31003.bYINVH.rst @@ -0,0 +1 @@ +IDLE: Add more tests for General tab. From 95b16a9705d6b4d31c016c014e59744fc33d53ea Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 27 Jul 2017 13:15:59 +0300 Subject: [PATCH 0604/1345] [3.6] bpo-30814, bpo-30876: Add new import test files to projects. (GH-2851). (#2912) (cherry picked from commit d5ed47dea25e04a3a144eddf99a4ac4a29242dbc) --- Makefile.pre.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 82e830727ef849..93a2bab6d983fe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1206,6 +1206,8 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/test_import/data \ test/test_import/data/circular_imports \ test/test_import/data/circular_imports/subpkg \ + test/test_import/data/package \ + test/test_import/data/package2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ From d019c7965d411b2132f68ed6bb33e538b4d9eba3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 27 Jul 2017 18:05:46 +0200 Subject: [PATCH 0605/1345] bpo-31028: Fix test_pydoc when run directly (#2864) (#2910) * bpo-31028: Fix test_pydoc when run directly Fix get_pydoc_link() of test_pydoc to fix "./python Lib/test/test_pydoc.py": get the absolute path to __file__ to prevent relative directories. * Use realpath() instead of abspath() (cherry picked from commit fd46561167af6cd697191dd7ebb8c2fef5ad6493) --- Lib/test/test_pydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ed438d5664c7c0..0383a6732401be 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -361,7 +361,7 @@ def get_pydoc_html(module): def get_pydoc_link(module): "Returns a documentation web link of a module" dirname = os.path.dirname - basedir = dirname(dirname(__file__)) + basedir = dirname(dirname(os.path.realpath(__file__))) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc From 54cb3400e500f99bd57a0273aa7ef84ffd37558e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 27 Jul 2017 18:42:11 +0200 Subject: [PATCH 0606/1345] bpo-31044: Skip test_posix.test_makedev() on FreeBSD (#2915) (#2917) There is a bug in FreeBSD CURRENT with 64-bit dev_t. Skip the test if dev_t is larger than 32-bit, until the bug is fixed in FreeBSD CURRENT. (cherry picked from commit 12953ffe12ac781332b384c36b25d12216b1db62) --- Lib/test/test_posix.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a6e2fed302f878..9df3206324fc67 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -478,6 +478,10 @@ def test_makedev(self): self.assertRaises(TypeError, posix.minor) self.assertRaises((ValueError, OverflowError), posix.minor, -1) + if sys.platform.startswith('freebsd') and dev >= 0x1_0000_0000: + self.skipTest("bpo-31044: on FreeBSD CURRENT, minor() truncates " + "64-bit dev to 32-bit") + self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) From 2cbb6733bfe54e87622a6372f83b73bddf3c4078 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 27 Jul 2017 20:50:39 -0400 Subject: [PATCH 0607/1345] [3.6] bpo-31060: IDLE: Finish regrouping ConfigDialog methods (GH-2908) (#2925) Finish resorting the 72 ConfigDialog methods into 7 groups that represent the dialog, action buttons, and font, highlight, keys, general, and extension pages. This will help with continuing to add tests and improve the pages. It will enable splitting ConfigDialog into 6 or 7 more comprehensible classes. (cherry picked from commit b166080) --- Lib/idlelib/configdialog.py | 1824 +++++++++-------- .../2017-07-27-14-48-42.bpo-31060.GdY_VY.rst | 3 + 2 files changed, 917 insertions(+), 910 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-27-14-48-42.bpo-31060.GdY_VY.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index daaa34459e2fa6..e359ec24cd3a5b 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -30,7 +30,6 @@ changes = ConfigChanges() - class ConfigDialog(Toplevel): """Config dialog for IDLE. """ @@ -112,6 +111,58 @@ def create_widgets(self): self.create_page_extensions() self.create_action_buttons().pack(side=BOTTOM) + def load_configs(self): + """Load configuration for each page. + + Load configuration from default and user config files and populate + the widgets on the config dialog pages. + + Methods: + load_font_cfg + load_tab_cfg + load_theme_cfg + load_key_cfg + load_general_cfg + """ + self.load_font_cfg() + self.load_tab_cfg() + self.load_theme_cfg() + self.load_key_cfg() + self.load_general_cfg() + # note: extension page handled separately + + def attach_var_callbacks(self): + "Attach callbacks to variables that can be changed." + self.font_size.trace_add('write', self.var_changed_font) + self.font_name.trace_add('write', self.var_changed_font) + self.font_bold.trace_add('write', self.var_changed_font) + self.space_num.trace_add('write', self.var_changed_space_num) + self.color.trace_add('write', self.var_changed_color) + self.builtin_theme.trace_add('write', self.var_changed_builtin_theme) + self.custom_theme.trace_add('write', self.var_changed_custom_theme) + self.is_builtin_theme.trace_add('write', self.var_changed_is_builtin_theme) + self.highlight_target.trace_add('write', self.var_changed_highlight_target) + self.keybinding.trace_add('write', self.var_changed_keybinding) + self.builtin_keys.trace_add('write', self.var_changed_builtin_keys) + self.custom_keys.trace_add('write', self.var_changed_custom_keys) + self.are_keys_builtin.trace_add('write', self.var_changed_are_keys_builtin) + self.win_width.trace_add('write', self.var_changed_win_width) + self.win_height.trace_add('write', self.var_changed_win_height) + self.startup_edit.trace_add('write', self.var_changed_startup_edit) + self.autosave.trace_add('write', self.var_changed_autosave) + + def remove_var_callbacks(self): + "Remove callbacks to prevent memory leaks." + for var in ( + self.font_size, self.font_name, self.font_bold, + self.space_num, self.color, self.builtin_theme, + self.custom_theme, self.is_builtin_theme, self.highlight_target, + self.keybinding, self.builtin_keys, self.custom_keys, + self.are_keys_builtin, self.win_width, self.win_height, + self.startup_edit, self.autosave,): + var.trace_remove('write', var.trace_info()[0][1]) + + def create_action_buttons(self): """Return frame of action buttons for dialog. @@ -150,6 +201,50 @@ def create_action_buttons(self): buttons.pack(side=BOTTOM) return outer + def ok(self): + """Apply config changes, then dismiss dialog. + + Methods: + apply + destroy: inherited + """ + self.apply() + self.destroy() + + def apply(self): + """Apply config changes and leave dialog open. + + Methods: + deactivate_current_config + save_all_changed_extensions + activate_config_changes + """ + self.deactivate_current_config() + changes.save_all() + self.save_all_changed_extensions() + self.activate_config_changes() + + def cancel(self): + """Dismiss config dialog. + + Methods: + destroy: inherited + """ + self.destroy() + + def help(self): + """Create textview for config dialog help. + + Attrbutes accessed: + tab_pages + + Methods: + view_text: Method from textview module. + """ + page = self.tab_pages._current_page + view_text(self, title='Help for IDLE preferences', + text=help_common+help_pages.get(page, '')) + def create_page_font_tab(self): """Return frame of widgets for Font/Tabs tab. @@ -299,16 +394,6 @@ def load_font_cfg(self): # Set font weight. self.font_bold.set(font_bold) - def on_fontlist_select(self, event): - """Handle selecting a font from the list. - - Event can result from either mouse click or Up or Down key. - Set font_name and example displays to selection. - """ - font = self.fontlist.get( - ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) - self.font_name.set(font.lower()) - def var_changed_font(self, *params): """Store changes to font attributes. @@ -324,6 +409,16 @@ def var_changed_font(self, *params): changes.add_option('main', 'EditorWindow', 'font-bold', value) self.set_samples() + def on_fontlist_select(self, event): + """Handle selecting a font from the list. + + Event can result from either mouse click or Up or Down key. + Set font_name and example displays to selection. + """ + font = self.fontlist.get( + ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) + self.font_name.set(font.lower()) + def set_samples(self, event=None): """Update update both screen samples with the font settings. @@ -531,466 +626,551 @@ def tem(event, elem=element): self.new_custom_theme.pack(side=TOP, fill=X, pady=5) return frame - def create_page_keys(self): - """Return frame of widgets for Keys tab. + def load_theme_cfg(self): + """Load current configuration settings for the theme options. - Tk Variables: - builtin_keys: Menu variable for built-in keybindings. - custom_keys: Menu variable for custom keybindings. - are_keys_builtin: Selector for built-in or custom keybindings. - keybinding: Action/key bindings. + Based on the is_builtin_theme toggle, the theme is set as + either builtin or custom and the initial widget values + reflect the current settings from idleConf. - Methods: - load_key_config: Set table. - load_keys_list: Reload active set. - keybinding_selected: Bound to list_bindings button release. - get_new_keys: Command for button_new_keys. - get_new_keys_name: Call popup. - create_new_key_set: Combine active keyset and changes. - set_keys_type: Command for are_keys_builtin. - delete_custom_keys: Command for button_delete_custom_keys. - save_as_new_key_set: Command for button_save_custom_keys. - save_new_key_set: Save to idleConf.userCfg['keys'] (is function). - deactivate_current_config: Remove keys bindings in editors. + Attributes updated: + is_builtin_theme: Set from idleConf. + opt_menu_theme_builtin: List of default themes from idleConf. + opt_menu_theme_custom: List of custom themes from idleConf. + radio_theme_custom: Disabled if there are no custom themes. + custom_theme: Message with additional information. + opt_menu_highlight_target: Create menu from self.theme_elements. - Widget Structure: (*) widgets bound to self - frame - frame_custom: LabelFrame - frame_target: Frame - target_title: Label - scroll_target_y: Scrollbar - scroll_target_x: Scrollbar - (*)list_bindings: ListBox - (*)button_new_keys: Button - frame_key_sets: LabelFrame - frames[0]: Frame - (*)radio_keys_builtin: Radiobutton - are_keys_builtin - (*)radio_keys_custom: Radiobutton - are_keys_builtin - (*)opt_menu_keys_builtin: DynOptionMenu - builtin_keys - (*)opt_menu_keys_custom: DynOptionMenu - custom_keys - (*)new_custom_keys: Label - frames[1]: Frame - (*)button_delete_custom_keys: Button - button_save_custom_keys: Button + Methods: + set_theme_type + paint_theme_sample + set_highlight_target """ - parent = self.parent - self.builtin_keys = StringVar(parent) - self.custom_keys = StringVar(parent) - self.are_keys_builtin = BooleanVar(parent) - self.keybinding = StringVar(parent) + # Set current theme type radiobutton. + self.is_builtin_theme.set(idleConf.GetOption( + 'main', 'Theme', 'default', type='bool', default=1)) + # Set current theme. + current_option = idleConf.CurrentTheme() + # Load available theme option menus. + if self.is_builtin_theme.get(): # Default theme selected. + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.opt_menu_theme_builtin.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.radio_theme_custom['state'] = DISABLED + self.custom_theme.set('- no custom themes -') + else: + self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) + else: # User theme selected. + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + self.opt_menu_theme_custom.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.opt_menu_theme_builtin.SetMenu(item_list, item_list[0]) + self.set_theme_type() + # Load theme element option menu. + theme_names = list(self.theme_elements.keys()) + theme_names.sort(key=lambda x: self.theme_elements[x][1]) + self.opt_menu_highlight_target.SetMenu(theme_names, theme_names[0]) + self.paint_theme_sample() + self.set_highlight_target() - ##widget creation - #body frame - frame = self.tab_pages.pages['Keys'].frame - #body section frames - frame_custom = LabelFrame( - frame, borderwidth=2, relief=GROOVE, - text=' Custom Key Bindings ') - frame_key_sets = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Key Set ') - #frame_custom - frame_target = Frame(frame_custom) - target_title = Label(frame_target, text='Action - Key(s)') - scroll_target_y = Scrollbar(frame_target) - scroll_target_x = Scrollbar(frame_target, orient=HORIZONTAL) - self.list_bindings = Listbox( - frame_target, takefocus=FALSE, exportselection=FALSE) - self.list_bindings.bind('', self.keybinding_selected) - scroll_target_y.config(command=self.list_bindings.yview) - scroll_target_x.config(command=self.list_bindings.xview) - self.list_bindings.config(yscrollcommand=scroll_target_y.set) - self.list_bindings.config(xscrollcommand=scroll_target_x.set) - self.button_new_keys = Button( - frame_custom, text='Get New Keys for Selection', - command=self.get_new_keys, state=DISABLED) - #frame_key_sets - frames = [Frame(frame_key_sets, padx=2, pady=2, borderwidth=0) - for i in range(2)] - self.radio_keys_builtin = Radiobutton( - frames[0], variable=self.are_keys_builtin, value=1, - command=self.set_keys_type, text='Use a Built-in Key Set') - self.radio_keys_custom = Radiobutton( - frames[0], variable=self.are_keys_builtin, value=0, - command=self.set_keys_type, text='Use a Custom Key Set') - self.opt_menu_keys_builtin = DynOptionMenu( - frames[0], self.builtin_keys, None, command=None) - self.opt_menu_keys_custom = DynOptionMenu( - frames[0], self.custom_keys, None, command=None) - self.button_delete_custom_keys = Button( - frames[1], text='Delete Custom Key Set', - command=self.delete_custom_keys) - button_save_custom_keys = Button( - frames[1], text='Save as New Custom Key Set', - command=self.save_as_new_key_set) - self.new_custom_keys = Label(frames[0], bd=2) + def var_changed_builtin_theme(self, *params): + """Process new builtin theme selection. - ##widget packing - #body - frame_custom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_key_sets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) - #frame_custom - self.button_new_keys.pack(side=BOTTOM, fill=X, padx=5, pady=5) - frame_target.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - #frame target - frame_target.columnconfigure(0, weight=1) - frame_target.rowconfigure(1, weight=1) - target_title.grid(row=0, column=0, columnspan=2, sticky=W) - self.list_bindings.grid(row=1, column=0, sticky=NSEW) - scroll_target_y.grid(row=1, column=1, sticky=NS) - scroll_target_x.grid(row=2, column=0, sticky=EW) - #frame_key_sets - self.radio_keys_builtin.grid(row=0, column=0, sticky=W+NS) - self.radio_keys_custom.grid(row=1, column=0, sticky=W+NS) - self.opt_menu_keys_builtin.grid(row=0, column=1, sticky=NSEW) - self.opt_menu_keys_custom.grid(row=1, column=1, sticky=NSEW) - self.new_custom_keys.grid(row=0, column=2, sticky=NSEW, padx=5, pady=5) - self.button_delete_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) - button_save_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) - frames[0].pack(side=TOP, fill=BOTH, expand=True) - frames[1].pack(side=TOP, fill=X, expand=True, pady=2) - return frame + Add the changed theme's name to the changed_items and recreate + the sample with the values from the selected theme. + """ + old_themes = ('IDLE Classic', 'IDLE New') + value = self.builtin_theme.get() + if value not in old_themes: + if idleConf.GetOption('main', 'Theme', 'name') not in old_themes: + changes.add_option('main', 'Theme', 'name', old_themes[0]) + changes.add_option('main', 'Theme', 'name2', value) + self.new_custom_theme.config(text='New theme, see Help', + fg='#500000') + else: + changes.add_option('main', 'Theme', 'name', value) + changes.add_option('main', 'Theme', 'name2', '') + self.new_custom_theme.config(text='', fg='black') + self.paint_theme_sample() + def var_changed_custom_theme(self, *params): + """Process new custom theme selection. - def create_page_general(self): - """Return frame of widgets for General tab. + If a new custom theme is selected, add the name to the + changed_items and apply the theme to the sample. + """ + value = self.custom_theme.get() + if value != '- no custom themes -': + changes.add_option('main', 'Theme', 'name', value) + self.paint_theme_sample() - Enable users to provisionally change general options. Function - load_general_cfg intializes tk variables and helplist using - idleConf. Radiobuttons startup_shell_on and startup_editor_on - set var startup_edit. Radiobuttons save_ask_on and save_auto_on - set var autosave. Entry boxes win_width_int and win_height_int - set var win_width and win_height. Setting var_name invokes the - var_changed_var_name callback that adds option to changes. + def var_changed_is_builtin_theme(self, *params): + """Process toggle between builtin and custom theme. - Helplist: load_general_cfg loads list user_helplist with - name, position pairs and copies names to listbox helplist. - Clicking a name invokes help_source selected. Clicking - button_helplist_name invokes helplist_item_name, which also - changes user_helplist. These functions all call - set_add_delete_state. All but load call update_help_changes to - rewrite changes['main']['HelpFiles']. + Update the default toggle value and apply the newly + selected theme type. + """ + value = self.is_builtin_theme.get() + changes.add_option('main', 'Theme', 'default', value) + if value: + self.var_changed_builtin_theme() + else: + self.var_changed_custom_theme() - Widget Structure: (*) widgets bound to self - frame - frame_run: LabelFrame - startup_title: Label - (*)startup_editor_on: Radiobutton - startup_edit - (*)startup_shell_on: Radiobutton - startup_edit - frame_save: LabelFrame - run_save_title: Label - (*)save_ask_on: Radiobutton - autosave - (*)save_auto_on: Radiobutton - autosave - frame_win_size: LabelFrame - win_size_title: Label - win_width_title: Label - (*)win_width_int: Entry - win_width - win_height_title: Label - (*)win_height_int: Entry - win_height - frame_help: LabelFrame - frame_helplist: Frame - frame_helplist_buttons: Frame - (*)button_helplist_edit - (*)button_helplist_add - (*)button_helplist_remove - (*)helplist: ListBox - scroll_helplist: Scrollbar + def var_changed_color(self, *params): + "Process change to color choice." + self.on_new_color_set() + + def var_changed_highlight_target(self, *params): + "Process selection of new target tag for highlighting." + self.set_highlight_target() + + def set_theme_type(self): + """Set available screen options based on builtin or custom theme. + + Attributes accessed: + is_builtin_theme + + Attributes updated: + opt_menu_theme_builtin + opt_menu_theme_custom + button_delete_custom_theme + radio_theme_custom + + Called from: + handler for radio_theme_builtin and radio_theme_custom + delete_custom_theme + create_new_theme + load_theme_cfg """ - parent = self.parent - self.startup_edit = IntVar(parent) - self.autosave = IntVar(parent) - self.win_width = StringVar(parent) - self.win_height = StringVar(parent) + if self.is_builtin_theme.get(): + self.opt_menu_theme_builtin['state'] = NORMAL + self.opt_menu_theme_custom['state'] = DISABLED + self.button_delete_custom_theme['state'] = DISABLED + else: + self.opt_menu_theme_builtin['state'] = DISABLED + self.radio_theme_custom['state'] = NORMAL + self.opt_menu_theme_custom['state'] = NORMAL + self.button_delete_custom_theme['state'] = NORMAL - # Create widgets: - # body. - frame = self.tab_pages.pages['General'].frame - # body section frames. - frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' autosave Preferences ') - frame_win_size = Frame(frame, borderwidth=2, relief=GROOVE) - frame_help = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') - # frame_run. - startup_title = Label(frame_run, text='At Startup') - self.startup_editor_on = Radiobutton( - frame_run, variable=self.startup_edit, value=1, - text="Open Edit Window") - self.startup_shell_on = Radiobutton( - frame_run, variable=self.startup_edit, value=0, - text='Open Shell Window') - # frame_save. - run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.save_ask_on = Radiobutton( - frame_save, variable=self.autosave, value=0, - text="Prompt to Save") - self.save_auto_on = Radiobutton( - frame_save, variable=self.autosave, value=1, - text='No Prompt') - # frame_win_size. - win_size_title = Label( - frame_win_size, text='Initial Window Size (in characters)') - win_width_title = Label(frame_win_size, text='Width') - self.win_width_int = Entry( - frame_win_size, textvariable=self.win_width, width=3) - win_height_title = Label(frame_win_size, text='Height') - self.win_height_int = Entry( - frame_win_size, textvariable=self.win_height, width=3) - # frame_help. - frame_helplist = Frame(frame_help) - frame_helplist_buttons = Frame(frame_helplist) - self.helplist = Listbox( - frame_helplist, height=5, takefocus=FALSE, - exportselection=FALSE) - scroll_helplist = Scrollbar(frame_helplist) - scroll_helplist['command'] = self.helplist.yview - self.helplist['yscrollcommand'] = scroll_helplist.set - self.helplist.bind('', self.help_source_selected) - self.button_helplist_edit = Button( - frame_helplist_buttons, text='Edit', state=DISABLED, - width=8, command=self.helplist_item_edit) - self.button_helplist_add = Button( - frame_helplist_buttons, text='Add', - width=8, command=self.helplist_item_add) - self.button_helplist_remove = Button( - frame_helplist_buttons, text='Remove', state=DISABLED, - width=8, command=self.helplist_item_remove) + def get_color(self): + """Handle button to select a new color for the target tag. - # Pack widgets: - # body. - frame_run.pack(side=TOP, padx=5, pady=5, fill=X) - frame_save.pack(side=TOP, padx=5, pady=5, fill=X) - frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) - frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - # frame_run. - startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_save. - run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_win_size. - win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) - win_height_title.pack(side=RIGHT, anchor=E, pady=5) - self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) - win_width_title.pack(side=RIGHT, anchor=E, pady=5) - # frame_help. - frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) - self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) - self.button_helplist_add.pack(side=TOP, anchor=W) - self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) + If a new color is selected while using a builtin theme, a + name must be supplied to create a custom theme. + + Attributes accessed: + highlight_target + frame_color_set + is_builtin_theme + + Attributes updated: + color + + Methods: + get_new_theme_name + create_new_theme + """ + target = self.highlight_target.get() + prev_color = self.frame_color_set.cget('bg') + rgbTuplet, color_string = tkColorChooser.askcolor( + parent=self, title='Pick new color for : '+target, + initialcolor=prev_color) + if color_string and (color_string != prev_color): + # User didn't cancel and they chose a new color. + if self.is_builtin_theme.get(): # Current theme is a built-in. + message = ('Your changes will be saved as a new Custom Theme. ' + 'Enter a name for your new Custom Theme below.') + new_theme = self.get_new_theme_name(message) + if not new_theme: # User cancelled custom theme creation. + return + else: # Create new custom theme based on previously active theme. + self.create_new_theme(new_theme) + self.color.set(color_string) + else: # Current theme is user defined. + self.color.set(color_string) + + def on_new_color_set(self): + "Display sample of new color selection on the dialog." + new_color=self.color.get() + self.frame_color_set.config(bg=new_color) # Set sample. + plane ='foreground' if self.fg_bg_toggle.get() else 'background' + sample_element = self.theme_elements[self.highlight_target.get()][0] + self.highlight_sample.tag_config(sample_element, **{plane:new_color}) + theme = self.custom_theme.get() + theme_element = sample_element + '-' + plane + changes.add_option('highlight', theme, theme_element, new_color) + + def get_new_theme_name(self, message): + "Return name of new theme from query popup." + used_names = (idleConf.GetSectionList('user', 'highlight') + + idleConf.GetSectionList('default', 'highlight')) + new_theme = SectionName( + self, 'New Custom Theme', message, used_names).result + return new_theme + + def save_as_new_theme(self): + """Prompt for new theme name and create the theme. + + Methods: + get_new_theme_name + create_new_theme + """ + new_theme_name = self.get_new_theme_name('New Theme Name:') + if new_theme_name: + self.create_new_theme(new_theme_name) + + def create_new_theme(self, new_theme_name): + """Create a new custom theme with the given name. - return frame + Create the new theme based on the previously active theme + with the current changes applied. Once it is saved, then + activate the new theme. - def load_general_cfg(self): - "Load current configuration settings for the general options." - # Set startup state. - self.startup_edit.set(idleConf.GetOption( - 'main', 'General', 'editor-on-startup', default=0, type='bool')) - # Set autosave state. - self.autosave.set(idleConf.GetOption( - 'main', 'General', 'autosave', default=0, type='bool')) - # Set initial window size. - self.win_width.set(idleConf.GetOption( - 'main', 'EditorWindow', 'width', type='int')) - self.win_height.set(idleConf.GetOption( - 'main', 'EditorWindow', 'height', type='int')) - # Set additional help sources. - self.user_helplist = idleConf.GetAllExtraHelpSourcesList() - self.helplist.delete(0, 'end') - for help_item in self.user_helplist: - self.helplist.insert(END, help_item[0]) - self.set_add_delete_state() + Attributes accessed: + builtin_theme + custom_theme - def var_changed_startup_edit(self, *params): - "Store change to toggle for starting IDLE in the editor or shell." - value = self.startup_edit.get() - changes.add_option('main', 'General', 'editor-on-startup', value) + Attributes updated: + opt_menu_theme_custom + is_builtin_theme - def var_changed_autosave(self, *params): - "Store change to autosave." - value = self.autosave.get() - changes.add_option('main', 'General', 'autosave', value) + Method: + save_new_theme + set_theme_type + """ + if self.is_builtin_theme.get(): + theme_type = 'default' + theme_name = self.builtin_theme.get() + else: + theme_type = 'user' + theme_name = self.custom_theme.get() + new_theme = idleConf.GetThemeDict(theme_type, theme_name) + # Apply any of the old theme's unsaved changes to the new theme. + if theme_name in changes['highlight']: + theme_changes = changes['highlight'][theme_name] + for element in theme_changes: + new_theme[element] = theme_changes[element] + # Save the new theme. + self.save_new_theme(new_theme_name, new_theme) + # Change GUI over to the new theme. + custom_theme_list = idleConf.GetSectionList('user', 'highlight') + custom_theme_list.sort() + self.opt_menu_theme_custom.SetMenu(custom_theme_list, new_theme_name) + self.is_builtin_theme.set(0) + self.set_theme_type() - def var_changed_win_width(self, *params): - "Store change to window width." - value = self.win_width.get() - changes.add_option('main', 'EditorWindow', 'width', value) + def set_highlight_target(self): + """Set fg/bg toggle and color based on highlight tag target. - def var_changed_win_height(self, *params): - "Store change to window height." - value = self.win_height.get() - changes.add_option('main', 'EditorWindow', 'height', value) + Instance variables accessed: + highlight_target - def help_source_selected(self, event): - "Handle event for selecting additional help." - self.set_add_delete_state() + Attributes updated: + radio_fg + radio_bg + fg_bg_toggle - def set_add_delete_state(self): - "Toggle the state for the help list buttons based on list entries." - if self.helplist.size() < 1: # No entries in list. - self.button_helplist_edit['state'] = DISABLED - self.button_helplist_remove['state'] = DISABLED - else: # Some entries. - if self.helplist.curselection(): # There currently is a selection. - self.button_helplist_edit['state'] = NORMAL - self.button_helplist_remove['state'] = NORMAL - else: # There currently is not a selection. - self.button_helplist_edit['state'] = DISABLED - self.button_helplist_remove['state'] = DISABLED + Methods: + set_color_sample - def helplist_item_add(self): - """Handle add button for the help list. + Called from: + var_changed_highlight_target + load_theme_cfg + """ + if self.highlight_target.get() == 'Cursor': # bg not possible + self.radio_fg['state'] = DISABLED + self.radio_bg['state'] = DISABLED + self.fg_bg_toggle.set(1) + else: # Both fg and bg can be set. + self.radio_fg['state'] = NORMAL + self.radio_bg['state'] = NORMAL + self.fg_bg_toggle.set(1) + self.set_color_sample() - Query for name and location of new help sources and add - them to the list. + def set_color_sample_binding(self, *args): + """Change color sample based on foreground/background toggle. + + Methods: + set_color_sample """ - help_source = HelpSource(self, 'New Help Source').result - if help_source: - self.user_helplist.append(help_source) - self.helplist.insert(END, help_source[0]) - self.update_help_changes() + self.set_color_sample() - def helplist_item_edit(self): - """Handle edit button for the help list. + def set_color_sample(self): + """Set the color of the frame background to reflect the selected target. - Query with existing help source information and update - config if the values are changed. + Instance variables accessed: + theme_elements + highlight_target + fg_bg_toggle + highlight_sample + + Attributes updated: + frame_color_set """ - item_index = self.helplist.index(ANCHOR) - help_source = self.user_helplist[item_index] - new_help_source = HelpSource( - self, 'Edit Help Source', - menuitem=help_source[0], - filepath=help_source[1], - ).result - if new_help_source and new_help_source != help_source: - self.user_helplist[item_index] = new_help_source - self.helplist.delete(item_index) - self.helplist.insert(item_index, new_help_source[0]) - self.update_help_changes() - self.set_add_delete_state() # Selected will be un-selected + # Set the color sample area. + tag = self.theme_elements[self.highlight_target.get()][0] + plane = 'foreground' if self.fg_bg_toggle.get() else 'background' + color = self.highlight_sample.tag_cget(tag, plane) + self.frame_color_set.config(bg=color) - def helplist_item_remove(self): - """Handle remove button for the help list. + def paint_theme_sample(self): + """Apply the theme colors to each element tag in the sample text. - Delete the help list item from config. + Instance attributes accessed: + theme_elements + is_builtin_theme + builtin_theme + custom_theme + + Attributes updated: + highlight_sample: Set the tag elements to the theme. + + Methods: + set_color_sample + + Called from: + var_changed_builtin_theme + var_changed_custom_theme + load_theme_cfg """ - item_index = self.helplist.index(ANCHOR) - del(self.user_helplist[item_index]) - self.helplist.delete(item_index) - self.update_help_changes() - self.set_add_delete_state() + if self.is_builtin_theme.get(): # Default theme + theme = self.builtin_theme.get() + else: # User theme + theme = self.custom_theme.get() + for element_title in self.theme_elements: + element = self.theme_elements[element_title][0] + colors = idleConf.GetHighlight(theme, element) + if element == 'cursor': # Cursor sample needs special painting. + colors['background'] = idleConf.GetHighlight( + theme, 'normal', fgBg='bg') + # Handle any unsaved changes to this theme. + if theme in changes['highlight']: + theme_dict = changes['highlight'][theme] + if element + '-foreground' in theme_dict: + colors['foreground'] = theme_dict[element + '-foreground'] + if element + '-background' in theme_dict: + colors['background'] = theme_dict[element + '-background'] + self.highlight_sample.tag_config(element, **colors) + self.set_color_sample() - def update_help_changes(self): - "Clear and rebuild the HelpFiles section in changes" - changes['main']['HelpFiles'] = {} - for num in range(1, len(self.user_helplist) + 1): - changes.add_option( - 'main', 'HelpFiles', str(num), - ';'.join(self.user_helplist[num-1][:2])) + def save_new_theme(self, theme_name, theme): + """Save a newly created theme to idleConf. + + theme_name - string, the name of the new theme + theme - dictionary containing the new theme + """ + if not idleConf.userCfg['highlight'].has_section(theme_name): + idleConf.userCfg['highlight'].add_section(theme_name) + for element in theme: + value = theme[element] + idleConf.userCfg['highlight'].SetOption(theme_name, element, value) + + def delete_custom_theme(self): + """Handle event to delete custom theme. + + The current theme is deactivated and the default theme is + activated. The custom theme is permanently removed from + the config file. + Attributes accessed: + custom_theme + + Attributes updated: + radio_theme_custom + opt_menu_theme_custom + is_builtin_theme + builtin_theme + + Methods: + deactivate_current_config + save_all_changed_extensions + activate_config_changes + set_theme_type + """ + theme_name = self.custom_theme.get() + delmsg = 'Are you sure you wish to delete the theme %r ?' + if not tkMessageBox.askyesno( + 'Delete Theme', delmsg % theme_name, parent=self): + return + self.deactivate_current_config() + # Remove theme from changes, config, and file. + changes.delete_section('highlight', theme_name) + # Reload user theme list. + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.radio_theme_custom['state'] = DISABLED + self.opt_menu_theme_custom.SetMenu(item_list, '- no custom themes -') + else: + self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) + # Revert to default theme. + self.is_builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) + self.builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) + # User can't back out of these changes, they must be applied now. + changes.save_all() + self.save_all_changed_extensions() + self.activate_config_changes() + self.set_theme_type() - def attach_var_callbacks(self): - "Attach callbacks to variables that can be changed." - self.font_size.trace_add('write', self.var_changed_font) - self.font_name.trace_add('write', self.var_changed_font) - self.font_bold.trace_add('write', self.var_changed_font) - self.space_num.trace_add('write', self.var_changed_space_num) - self.color.trace_add('write', self.var_changed_color) - self.builtin_theme.trace_add('write', self.var_changed_builtin_theme) - self.custom_theme.trace_add('write', self.var_changed_custom_theme) - self.is_builtin_theme.trace_add('write', self.var_changed_is_builtin_theme) - self.highlight_target.trace_add('write', self.var_changed_highlight_target) - self.keybinding.trace_add('write', self.var_changed_keybinding) - self.builtin_keys.trace_add('write', self.var_changed_builtin_keys) - self.custom_keys.trace_add('write', self.var_changed_custom_keys) - self.are_keys_builtin.trace_add('write', self.var_changed_are_keys_builtin) - self.win_width.trace_add('write', self.var_changed_win_width) - self.win_height.trace_add('write', self.var_changed_win_height) - self.startup_edit.trace_add('write', self.var_changed_startup_edit) - self.autosave.trace_add('write', self.var_changed_autosave) - def remove_var_callbacks(self): - "Remove callbacks to prevent memory leaks." - for var in ( - self.font_size, self.font_name, self.font_bold, - self.space_num, self.color, self.builtin_theme, - self.custom_theme, self.is_builtin_theme, self.highlight_target, - self.keybinding, self.builtin_keys, self.custom_keys, - self.are_keys_builtin, self.win_width, self.win_height, - self.startup_edit, self.autosave,): - var.trace_remove('write', var.trace_info()[0][1]) + def create_page_keys(self): + """Return frame of widgets for Keys tab. - def var_changed_color(self, *params): - "Process change to color choice." - self.on_new_color_set() + Tk Variables: + builtin_keys: Menu variable for built-in keybindings. + custom_keys: Menu variable for custom keybindings. + are_keys_builtin: Selector for built-in or custom keybindings. + keybinding: Action/key bindings. - def var_changed_builtin_theme(self, *params): - """Process new builtin theme selection. + Methods: + load_key_config: Set table. + load_keys_list: Reload active set. + keybinding_selected: Bound to list_bindings button release. + get_new_keys: Command for button_new_keys. + get_new_keys_name: Call popup. + create_new_key_set: Combine active keyset and changes. + set_keys_type: Command for are_keys_builtin. + delete_custom_keys: Command for button_delete_custom_keys. + save_as_new_key_set: Command for button_save_custom_keys. + save_new_key_set: Save to idleConf.userCfg['keys'] (is function). + deactivate_current_config: Remove keys bindings in editors. - Add the changed theme's name to the changed_items and recreate - the sample with the values from the selected theme. + Widget Structure: (*) widgets bound to self + frame + frame_custom: LabelFrame + frame_target: Frame + target_title: Label + scroll_target_y: Scrollbar + scroll_target_x: Scrollbar + (*)list_bindings: ListBox + (*)button_new_keys: Button + frame_key_sets: LabelFrame + frames[0]: Frame + (*)radio_keys_builtin: Radiobutton - are_keys_builtin + (*)radio_keys_custom: Radiobutton - are_keys_builtin + (*)opt_menu_keys_builtin: DynOptionMenu - builtin_keys + (*)opt_menu_keys_custom: DynOptionMenu - custom_keys + (*)new_custom_keys: Label + frames[1]: Frame + (*)button_delete_custom_keys: Button + button_save_custom_keys: Button """ - old_themes = ('IDLE Classic', 'IDLE New') - value = self.builtin_theme.get() - if value not in old_themes: - if idleConf.GetOption('main', 'Theme', 'name') not in old_themes: - changes.add_option('main', 'Theme', 'name', old_themes[0]) - changes.add_option('main', 'Theme', 'name2', value) - self.new_custom_theme.config(text='New theme, see Help', - fg='#500000') - else: - changes.add_option('main', 'Theme', 'name', value) - changes.add_option('main', 'Theme', 'name2', '') - self.new_custom_theme.config(text='', fg='black') - self.paint_theme_sample() + parent = self.parent + self.builtin_keys = StringVar(parent) + self.custom_keys = StringVar(parent) + self.are_keys_builtin = BooleanVar(parent) + self.keybinding = StringVar(parent) - def var_changed_custom_theme(self, *params): - """Process new custom theme selection. + ##widget creation + #body frame + frame = self.tab_pages.pages['Keys'].frame + #body section frames + frame_custom = LabelFrame( + frame, borderwidth=2, relief=GROOVE, + text=' Custom Key Bindings ') + frame_key_sets = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Key Set ') + #frame_custom + frame_target = Frame(frame_custom) + target_title = Label(frame_target, text='Action - Key(s)') + scroll_target_y = Scrollbar(frame_target) + scroll_target_x = Scrollbar(frame_target, orient=HORIZONTAL) + self.list_bindings = Listbox( + frame_target, takefocus=FALSE, exportselection=FALSE) + self.list_bindings.bind('', self.keybinding_selected) + scroll_target_y.config(command=self.list_bindings.yview) + scroll_target_x.config(command=self.list_bindings.xview) + self.list_bindings.config(yscrollcommand=scroll_target_y.set) + self.list_bindings.config(xscrollcommand=scroll_target_x.set) + self.button_new_keys = Button( + frame_custom, text='Get New Keys for Selection', + command=self.get_new_keys, state=DISABLED) + #frame_key_sets + frames = [Frame(frame_key_sets, padx=2, pady=2, borderwidth=0) + for i in range(2)] + self.radio_keys_builtin = Radiobutton( + frames[0], variable=self.are_keys_builtin, value=1, + command=self.set_keys_type, text='Use a Built-in Key Set') + self.radio_keys_custom = Radiobutton( + frames[0], variable=self.are_keys_builtin, value=0, + command=self.set_keys_type, text='Use a Custom Key Set') + self.opt_menu_keys_builtin = DynOptionMenu( + frames[0], self.builtin_keys, None, command=None) + self.opt_menu_keys_custom = DynOptionMenu( + frames[0], self.custom_keys, None, command=None) + self.button_delete_custom_keys = Button( + frames[1], text='Delete Custom Key Set', + command=self.delete_custom_keys) + button_save_custom_keys = Button( + frames[1], text='Save as New Custom Key Set', + command=self.save_as_new_key_set) + self.new_custom_keys = Label(frames[0], bd=2) - If a new custom theme is selected, add the name to the - changed_items and apply the theme to the sample. - """ - value = self.custom_theme.get() - if value != '- no custom themes -': - changes.add_option('main', 'Theme', 'name', value) - self.paint_theme_sample() + ##widget packing + #body + frame_custom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_key_sets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) + #frame_custom + self.button_new_keys.pack(side=BOTTOM, fill=X, padx=5, pady=5) + frame_target.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + #frame target + frame_target.columnconfigure(0, weight=1) + frame_target.rowconfigure(1, weight=1) + target_title.grid(row=0, column=0, columnspan=2, sticky=W) + self.list_bindings.grid(row=1, column=0, sticky=NSEW) + scroll_target_y.grid(row=1, column=1, sticky=NS) + scroll_target_x.grid(row=2, column=0, sticky=EW) + #frame_key_sets + self.radio_keys_builtin.grid(row=0, column=0, sticky=W+NS) + self.radio_keys_custom.grid(row=1, column=0, sticky=W+NS) + self.opt_menu_keys_builtin.grid(row=0, column=1, sticky=NSEW) + self.opt_menu_keys_custom.grid(row=1, column=1, sticky=NSEW) + self.new_custom_keys.grid(row=0, column=2, sticky=NSEW, padx=5, pady=5) + self.button_delete_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) + button_save_custom_keys.pack(side=LEFT, fill=X, expand=True, padx=2) + frames[0].pack(side=TOP, fill=BOTH, expand=True) + frames[1].pack(side=TOP, fill=X, expand=True, pady=2) + return frame - def var_changed_is_builtin_theme(self, *params): - """Process toggle between builtin and custom theme. + def load_key_cfg(self): + "Load current configuration settings for the keybinding options." + # Set current keys type radiobutton. + self.are_keys_builtin.set(idleConf.GetOption( + 'main', 'Keys', 'default', type='bool', default=1)) + # Set current keys. + current_option = idleConf.CurrentKeys() + # Load available keyset option menus. + if self.are_keys_builtin.get(): # Default theme selected. + item_list = idleConf.GetSectionList('default', 'keys') + item_list.sort() + self.opt_menu_keys_builtin.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + if not item_list: + self.radio_keys_custom['state'] = DISABLED + self.custom_keys.set('- no custom keys -') + else: + self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) + else: # User key set selected. + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + self.opt_menu_keys_custom.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('default', 'keys') + item_list.sort() + self.opt_menu_keys_builtin.SetMenu(item_list, idleConf.default_keys()) + self.set_keys_type() + # Load keyset element list. + keyset_name = idleConf.CurrentKeys() + self.load_keys_list(keyset_name) - Update the default toggle value and apply the newly - selected theme type. - """ - value = self.is_builtin_theme.get() - changes.add_option('main', 'Theme', 'default', value) - if value: - self.var_changed_builtin_theme() - else: - self.var_changed_custom_theme() - def var_changed_highlight_target(self, *params): - "Process selection of new target tag for highlighting." - self.set_highlight_target() - def var_changed_keybinding(self, *params): - "Store change to a keybinding." - value = self.keybinding.get() - key_set = self.custom_keys.get() - event = self.list_bindings.get(ANCHOR).split()[0] - if idleConf.IsCoreBinding(event): - changes.add_option('keys', key_set, event, value) - else: # Event is an extension binding. - ext_name = idleConf.GetExtnNameForEvent(event) - ext_keybind_section = ext_name + '_cfgBindings' - changes.add_option('extensions', ext_keybind_section, event, value) def var_changed_builtin_keys(self, *params): "Process selection of builtin key set." @@ -1016,46 +1196,30 @@ def var_changed_builtin_keys(self, *params): def var_changed_custom_keys(self, *params): "Process selection of custom key set." value = self.custom_keys.get() - if value != '- no custom keys -': - changes.add_option('main', 'Keys', 'name', value) - self.load_keys_list(value) - - def var_changed_are_keys_builtin(self, *params): - "Process toggle between builtin key set and custom key set." - value = self.are_keys_builtin.get() - changes.add_option('main', 'Keys', 'default', value) - if value: - self.var_changed_builtin_keys() - else: - self.var_changed_custom_keys() - - def set_theme_type(self): - """Set available screen options based on builtin or custom theme. - - Attributes accessed: - is_builtin_theme - - Attributes updated: - opt_menu_theme_builtin - opt_menu_theme_custom - button_delete_custom_theme - radio_theme_custom + if value != '- no custom keys -': + changes.add_option('main', 'Keys', 'name', value) + self.load_keys_list(value) - Called from: - handler for radio_theme_builtin and radio_theme_custom - delete_custom_theme - create_new_theme - load_theme_cfg - """ - if self.is_builtin_theme.get(): - self.opt_menu_theme_builtin['state'] = NORMAL - self.opt_menu_theme_custom['state'] = DISABLED - self.button_delete_custom_theme['state'] = DISABLED + def var_changed_are_keys_builtin(self, *params): + "Process toggle between builtin key set and custom key set." + value = self.are_keys_builtin.get() + changes.add_option('main', 'Keys', 'default', value) + if value: + self.var_changed_builtin_keys() else: - self.opt_menu_theme_builtin['state'] = DISABLED - self.radio_theme_custom['state'] = NORMAL - self.opt_menu_theme_custom['state'] = NORMAL - self.button_delete_custom_theme['state'] = NORMAL + self.var_changed_custom_keys() + + def var_changed_keybinding(self, *params): + "Store change to a keybinding." + value = self.keybinding.get() + key_set = self.custom_keys.get() + event = self.list_bindings.get(ANCHOR).split()[0] + if idleConf.IsCoreBinding(event): + changes.add_option('keys', key_set, event, value) + else: # Event is an extension binding. + ext_name = idleConf.GetExtnNameForEvent(event) + ext_keybind_section = ext_name + '_cfgBindings' + changes.add_option('extensions', ext_keybind_section, event, value) def set_keys_type(self): "Set available screen options based on builtin or custom key set." @@ -1183,489 +1347,329 @@ def load_keys_list(self, keyset_name): if bind_name in changes['keys'][keyset_name]: key = changes['keys'][keyset_name][bind_name] self.list_bindings.insert(END, bind_name+' - '+key) - if reselect: - self.list_bindings.see(list_index) - self.list_bindings.select_set(list_index) - self.list_bindings.select_anchor(list_index) - - def delete_custom_keys(self): - """Handle event to delete a custom key set. - - Applying the delete deactivates the current configuration and - reverts to the default. The custom key set is permanently - deleted from the config file. - """ - keyset_name=self.custom_keys.get() - delmsg = 'Are you sure you wish to delete the key set %r ?' - if not tkMessageBox.askyesno( - 'Delete Key Set', delmsg % keyset_name, parent=self): - return - self.deactivate_current_config() - # Remove key set from changes, config, and file. - changes.delete_section('keys', keyset_name) - # Reload user key set list. - item_list = idleConf.GetSectionList('user', 'keys') - item_list.sort() - if not item_list: - self.radio_keys_custom['state'] = DISABLED - self.opt_menu_keys_custom.SetMenu(item_list, '- no custom keys -') - else: - self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) - # Revert to default key set. - self.are_keys_builtin.set(idleConf.defaultCfg['main'] - .Get('Keys', 'default')) - self.builtin_keys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') - or idleConf.default_keys()) - # User can't back out of these changes, they must be applied now. - changes.save_all() - self.save_all_changed_extensions() - self.activate_config_changes() - self.set_keys_type() - - def delete_custom_theme(self): - """Handle event to delete custom theme. - - The current theme is deactivated and the default theme is - activated. The custom theme is permanently removed from - the config file. - - Attributes accessed: - custom_theme - - Attributes updated: - radio_theme_custom - opt_menu_theme_custom - is_builtin_theme - builtin_theme - - Methods: - deactivate_current_config - save_all_changed_extensions - activate_config_changes - set_theme_type - """ - theme_name = self.custom_theme.get() - delmsg = 'Are you sure you wish to delete the theme %r ?' - if not tkMessageBox.askyesno( - 'Delete Theme', delmsg % theme_name, parent=self): - return - self.deactivate_current_config() - # Remove theme from changes, config, and file. - changes.delete_section('highlight', theme_name) - # Reload user theme list. - item_list = idleConf.GetSectionList('user', 'highlight') - item_list.sort() - if not item_list: - self.radio_theme_custom['state'] = DISABLED - self.opt_menu_theme_custom.SetMenu(item_list, '- no custom themes -') - else: - self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) - # Revert to default theme. - self.is_builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) - self.builtin_theme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) - # User can't back out of these changes, they must be applied now. - changes.save_all() - self.save_all_changed_extensions() - self.activate_config_changes() - self.set_theme_type() - - def get_color(self): - """Handle button to select a new color for the target tag. - - If a new color is selected while using a builtin theme, a - name must be supplied to create a custom theme. - - Attributes accessed: - highlight_target - frame_color_set - is_builtin_theme - - Attributes updated: - color - - Methods: - get_new_theme_name - create_new_theme - """ - target = self.highlight_target.get() - prev_color = self.frame_color_set.cget('bg') - rgbTuplet, color_string = tkColorChooser.askcolor( - parent=self, title='Pick new color for : '+target, - initialcolor=prev_color) - if color_string and (color_string != prev_color): - # User didn't cancel and they chose a new color. - if self.is_builtin_theme.get(): # Current theme is a built-in. - message = ('Your changes will be saved as a new Custom Theme. ' - 'Enter a name for your new Custom Theme below.') - new_theme = self.get_new_theme_name(message) - if not new_theme: # User cancelled custom theme creation. - return - else: # Create new custom theme based on previously active theme. - self.create_new_theme(new_theme) - self.color.set(color_string) - else: # Current theme is user defined. - self.color.set(color_string) - - def on_new_color_set(self): - "Display sample of new color selection on the dialog." - new_color=self.color.get() - self.frame_color_set.config(bg=new_color) # Set sample. - plane ='foreground' if self.fg_bg_toggle.get() else 'background' - sample_element = self.theme_elements[self.highlight_target.get()][0] - self.highlight_sample.tag_config(sample_element, **{plane:new_color}) - theme = self.custom_theme.get() - theme_element = sample_element + '-' + plane - changes.add_option('highlight', theme, theme_element, new_color) - - def get_new_theme_name(self, message): - "Return name of new theme from query popup." - used_names = (idleConf.GetSectionList('user', 'highlight') + - idleConf.GetSectionList('default', 'highlight')) - new_theme = SectionName( - self, 'New Custom Theme', message, used_names).result - return new_theme - - def save_as_new_theme(self): - """Prompt for new theme name and create the theme. - - Methods: - get_new_theme_name - create_new_theme - """ - new_theme_name = self.get_new_theme_name('New Theme Name:') - if new_theme_name: - self.create_new_theme(new_theme_name) - - def create_new_theme(self, new_theme_name): - """Create a new custom theme with the given name. - - Create the new theme based on the previously active theme - with the current changes applied. Once it is saved, then - activate the new theme. - - Attributes accessed: - builtin_theme - custom_theme - - Attributes updated: - opt_menu_theme_custom - is_builtin_theme - - Method: - save_new_theme - set_theme_type - """ - if self.is_builtin_theme.get(): - theme_type = 'default' - theme_name = self.builtin_theme.get() - else: - theme_type = 'user' - theme_name = self.custom_theme.get() - new_theme = idleConf.GetThemeDict(theme_type, theme_name) - # Apply any of the old theme's unsaved changes to the new theme. - if theme_name in changes['highlight']: - theme_changes = changes['highlight'][theme_name] - for element in theme_changes: - new_theme[element] = theme_changes[element] - # Save the new theme. - self.save_new_theme(new_theme_name, new_theme) - # Change GUI over to the new theme. - custom_theme_list = idleConf.GetSectionList('user', 'highlight') - custom_theme_list.sort() - self.opt_menu_theme_custom.SetMenu(custom_theme_list, new_theme_name) - self.is_builtin_theme.set(0) - self.set_theme_type() - - def set_highlight_target(self): - """Set fg/bg toggle and color based on highlight tag target. - - Instance variables accessed: - highlight_target - - Attributes updated: - radio_fg - radio_bg - fg_bg_toggle - - Methods: - set_color_sample - - Called from: - var_changed_highlight_target - load_theme_cfg - """ - if self.highlight_target.get() == 'Cursor': # bg not possible - self.radio_fg['state'] = DISABLED - self.radio_bg['state'] = DISABLED - self.fg_bg_toggle.set(1) - else: # Both fg and bg can be set. - self.radio_fg['state'] = NORMAL - self.radio_bg['state'] = NORMAL - self.fg_bg_toggle.set(1) - self.set_color_sample() + if reselect: + self.list_bindings.see(list_index) + self.list_bindings.select_set(list_index) + self.list_bindings.select_anchor(list_index) - def set_color_sample_binding(self, *args): - """Change color sample based on foreground/background toggle. + def save_new_key_set(self, keyset_name, keyset): + """Save a newly created core key set. - Methods: - set_color_sample + keyset_name - string, the name of the new key set + keyset - dictionary containing the new key set """ - self.set_color_sample() - - def set_color_sample(self): - """Set the color of the frame background to reflect the selected target. + if not idleConf.userCfg['keys'].has_section(keyset_name): + idleConf.userCfg['keys'].add_section(keyset_name) + for event in keyset: + value = keyset[event] + idleConf.userCfg['keys'].SetOption(keyset_name, event, value) - Instance variables accessed: - theme_elements - highlight_target - fg_bg_toggle - highlight_sample + def delete_custom_keys(self): + """Handle event to delete a custom key set. - Attributes updated: - frame_color_set + Applying the delete deactivates the current configuration and + reverts to the default. The custom key set is permanently + deleted from the config file. """ - # Set the color sample area. - tag = self.theme_elements[self.highlight_target.get()][0] - plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - color = self.highlight_sample.tag_cget(tag, plane) - self.frame_color_set.config(bg=color) - - def paint_theme_sample(self): - """Apply the theme colors to each element tag in the sample text. + keyset_name=self.custom_keys.get() + delmsg = 'Are you sure you wish to delete the key set %r ?' + if not tkMessageBox.askyesno( + 'Delete Key Set', delmsg % keyset_name, parent=self): + return + self.deactivate_current_config() + # Remove key set from changes, config, and file. + changes.delete_section('keys', keyset_name) + # Reload user key set list. + item_list = idleConf.GetSectionList('user', 'keys') + item_list.sort() + if not item_list: + self.radio_keys_custom['state'] = DISABLED + self.opt_menu_keys_custom.SetMenu(item_list, '- no custom keys -') + else: + self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) + # Revert to default key set. + self.are_keys_builtin.set(idleConf.defaultCfg['main'] + .Get('Keys', 'default')) + self.builtin_keys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') + or idleConf.default_keys()) + # User can't back out of these changes, they must be applied now. + changes.save_all() + self.save_all_changed_extensions() + self.activate_config_changes() + self.set_keys_type() - Instance attributes accessed: - theme_elements - is_builtin_theme - builtin_theme - custom_theme + def deactivate_current_config(self): + """Remove current key bindings. - Attributes updated: - highlight_sample: Set the tag elements to the theme. + Iterate over window instances defined in parent and remove + the keybindings. + """ + # Before a config is saved, some cleanup of current + # config must be done - remove the previous keybindings. + win_instances = self.parent.instance_dict.keys() + for instance in win_instances: + instance.RemoveKeybindings() - Methods: - set_color_sample + def activate_config_changes(self): + """Apply configuration changes to current windows. - Called from: - var_changed_builtin_theme - var_changed_custom_theme - load_theme_cfg + Dynamically update the current parent window instances + with some of the configuration changes. """ - if self.is_builtin_theme.get(): # Default theme - theme = self.builtin_theme.get() - else: # User theme - theme = self.custom_theme.get() - for element_title in self.theme_elements: - element = self.theme_elements[element_title][0] - colors = idleConf.GetHighlight(theme, element) - if element == 'cursor': # Cursor sample needs special painting. - colors['background'] = idleConf.GetHighlight( - theme, 'normal', fgBg='bg') - # Handle any unsaved changes to this theme. - if theme in changes['highlight']: - theme_dict = changes['highlight'][theme] - if element + '-foreground' in theme_dict: - colors['foreground'] = theme_dict[element + '-foreground'] - if element + '-background' in theme_dict: - colors['background'] = theme_dict[element + '-background'] - self.highlight_sample.tag_config(element, **colors) - self.set_color_sample() - - def load_theme_cfg(self): - """Load current configuration settings for the theme options. + win_instances = self.parent.instance_dict.keys() + for instance in win_instances: + instance.ResetColorizer() + instance.ResetFont() + instance.set_notabs_indentwidth() + instance.ApplyKeybindings() + instance.reset_help_menu_entries() - Based on the is_builtin_theme toggle, the theme is set as - either builtin or custom and the initial widget values - reflect the current settings from idleConf. - Attributes updated: - is_builtin_theme: Set from idleConf. - opt_menu_theme_builtin: List of default themes from idleConf. - opt_menu_theme_custom: List of custom themes from idleConf. - radio_theme_custom: Disabled if there are no custom themes. - custom_theme: Message with additional information. - opt_menu_highlight_target: Create menu from self.theme_elements. + def create_page_general(self): + """Return frame of widgets for General tab. - Methods: - set_theme_type - paint_theme_sample - set_highlight_target - """ - # Set current theme type radiobutton. - self.is_builtin_theme.set(idleConf.GetOption( - 'main', 'Theme', 'default', type='bool', default=1)) - # Set current theme. - current_option = idleConf.CurrentTheme() - # Load available theme option menus. - if self.is_builtin_theme.get(): # Default theme selected. - item_list = idleConf.GetSectionList('default', 'highlight') - item_list.sort() - self.opt_menu_theme_builtin.SetMenu(item_list, current_option) - item_list = idleConf.GetSectionList('user', 'highlight') - item_list.sort() - if not item_list: - self.radio_theme_custom['state'] = DISABLED - self.custom_theme.set('- no custom themes -') - else: - self.opt_menu_theme_custom.SetMenu(item_list, item_list[0]) - else: # User theme selected. - item_list = idleConf.GetSectionList('user', 'highlight') - item_list.sort() - self.opt_menu_theme_custom.SetMenu(item_list, current_option) - item_list = idleConf.GetSectionList('default', 'highlight') - item_list.sort() - self.opt_menu_theme_builtin.SetMenu(item_list, item_list[0]) - self.set_theme_type() - # Load theme element option menu. - theme_names = list(self.theme_elements.keys()) - theme_names.sort(key=lambda x: self.theme_elements[x][1]) - self.opt_menu_highlight_target.SetMenu(theme_names, theme_names[0]) - self.paint_theme_sample() - self.set_highlight_target() + Enable users to provisionally change general options. Function + load_general_cfg intializes tk variables and helplist using + idleConf. Radiobuttons startup_shell_on and startup_editor_on + set var startup_edit. Radiobuttons save_ask_on and save_auto_on + set var autosave. Entry boxes win_width_int and win_height_int + set var win_width and win_height. Setting var_name invokes the + var_changed_var_name callback that adds option to changes. - def load_key_cfg(self): - "Load current configuration settings for the keybinding options." - # Set current keys type radiobutton. - self.are_keys_builtin.set(idleConf.GetOption( - 'main', 'Keys', 'default', type='bool', default=1)) - # Set current keys. - current_option = idleConf.CurrentKeys() - # Load available keyset option menus. - if self.are_keys_builtin.get(): # Default theme selected. - item_list = idleConf.GetSectionList('default', 'keys') - item_list.sort() - self.opt_menu_keys_builtin.SetMenu(item_list, current_option) - item_list = idleConf.GetSectionList('user', 'keys') - item_list.sort() - if not item_list: - self.radio_keys_custom['state'] = DISABLED - self.custom_keys.set('- no custom keys -') - else: - self.opt_menu_keys_custom.SetMenu(item_list, item_list[0]) - else: # User key set selected. - item_list = idleConf.GetSectionList('user', 'keys') - item_list.sort() - self.opt_menu_keys_custom.SetMenu(item_list, current_option) - item_list = idleConf.GetSectionList('default', 'keys') - item_list.sort() - self.opt_menu_keys_builtin.SetMenu(item_list, idleConf.default_keys()) - self.set_keys_type() - # Load keyset element list. - keyset_name = idleConf.CurrentKeys() - self.load_keys_list(keyset_name) + Helplist: load_general_cfg loads list user_helplist with + name, position pairs and copies names to listbox helplist. + Clicking a name invokes help_source selected. Clicking + button_helplist_name invokes helplist_item_name, which also + changes user_helplist. These functions all call + set_add_delete_state. All but load call update_help_changes to + rewrite changes['main']['HelpFiles']. - def load_configs(self): - """Load configuration for each page. + Widget Structure: (*) widgets bound to self + frame + frame_run: LabelFrame + startup_title: Label + (*)startup_editor_on: Radiobutton - startup_edit + (*)startup_shell_on: Radiobutton - startup_edit + frame_save: LabelFrame + run_save_title: Label + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave + frame_win_size: LabelFrame + win_size_title: Label + win_width_title: Label + (*)win_width_int: Entry - win_width + win_height_title: Label + (*)win_height_int: Entry - win_height + frame_help: LabelFrame + frame_helplist: Frame + frame_helplist_buttons: Frame + (*)button_helplist_edit + (*)button_helplist_add + (*)button_helplist_remove + (*)helplist: ListBox + scroll_helplist: Scrollbar + """ + parent = self.parent + self.startup_edit = IntVar(parent) + self.autosave = IntVar(parent) + self.win_width = StringVar(parent) + self.win_height = StringVar(parent) - Load configuration from default and user config files and populate - the widgets on the config dialog pages. + # Create widgets: + # body. + frame = self.tab_pages.pages['General'].frame + # body section frames. + frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Startup Preferences ') + frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' autosave Preferences ') + frame_win_size = Frame(frame, borderwidth=2, relief=GROOVE) + frame_help = LabelFrame(frame, borderwidth=2, relief=GROOVE, + text=' Additional Help Sources ') + # frame_run. + startup_title = Label(frame_run, text='At Startup') + self.startup_editor_on = Radiobutton( + frame_run, variable=self.startup_edit, value=1, + text="Open Edit Window") + self.startup_shell_on = Radiobutton( + frame_run, variable=self.startup_edit, value=0, + text='Open Shell Window') + # frame_save. + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + self.save_ask_on = Radiobutton( + frame_save, variable=self.autosave, value=0, + text="Prompt to Save") + self.save_auto_on = Radiobutton( + frame_save, variable=self.autosave, value=1, + text='No Prompt') + # frame_win_size. + win_size_title = Label( + frame_win_size, text='Initial Window Size (in characters)') + win_width_title = Label(frame_win_size, text='Width') + self.win_width_int = Entry( + frame_win_size, textvariable=self.win_width, width=3) + win_height_title = Label(frame_win_size, text='Height') + self.win_height_int = Entry( + frame_win_size, textvariable=self.win_height, width=3) + # frame_help. + frame_helplist = Frame(frame_help) + frame_helplist_buttons = Frame(frame_helplist) + self.helplist = Listbox( + frame_helplist, height=5, takefocus=FALSE, + exportselection=FALSE) + scroll_helplist = Scrollbar(frame_helplist) + scroll_helplist['command'] = self.helplist.yview + self.helplist['yscrollcommand'] = scroll_helplist.set + self.helplist.bind('', self.help_source_selected) + self.button_helplist_edit = Button( + frame_helplist_buttons, text='Edit', state=DISABLED, + width=8, command=self.helplist_item_edit) + self.button_helplist_add = Button( + frame_helplist_buttons, text='Add', + width=8, command=self.helplist_item_add) + self.button_helplist_remove = Button( + frame_helplist_buttons, text='Remove', state=DISABLED, + width=8, command=self.helplist_item_remove) - Methods: - load_font_cfg - load_tab_cfg - load_theme_cfg - load_key_cfg - load_general_cfg - """ - self.load_font_cfg() - self.load_tab_cfg() - self.load_theme_cfg() - self.load_key_cfg() - self.load_general_cfg() - # note: extension page handled separately + # Pack widgets: + # body. + frame_run.pack(side=TOP, padx=5, pady=5, fill=X) + frame_save.pack(side=TOP, padx=5, pady=5, fill=X) + frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) + frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + # frame_run. + startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_save. + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_win_size. + win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_height_title.pack(side=RIGHT, anchor=E, pady=5) + self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_width_title.pack(side=RIGHT, anchor=E, pady=5) + # frame_help. + frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) + frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) + self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) + self.button_helplist_add.pack(side=TOP, anchor=W) + self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) - def save_new_key_set(self, keyset_name, keyset): - """Save a newly created core key set. + return frame - keyset_name - string, the name of the new key set - keyset - dictionary containing the new key set - """ - if not idleConf.userCfg['keys'].has_section(keyset_name): - idleConf.userCfg['keys'].add_section(keyset_name) - for event in keyset: - value = keyset[event] - idleConf.userCfg['keys'].SetOption(keyset_name, event, value) + def load_general_cfg(self): + "Load current configuration settings for the general options." + # Set startup state. + self.startup_edit.set(idleConf.GetOption( + 'main', 'General', 'editor-on-startup', default=0, type='bool')) + # Set autosave state. + self.autosave.set(idleConf.GetOption( + 'main', 'General', 'autosave', default=0, type='bool')) + # Set initial window size. + self.win_width.set(idleConf.GetOption( + 'main', 'EditorWindow', 'width', type='int')) + self.win_height.set(idleConf.GetOption( + 'main', 'EditorWindow', 'height', type='int')) + # Set additional help sources. + self.user_helplist = idleConf.GetAllExtraHelpSourcesList() + self.helplist.delete(0, 'end') + for help_item in self.user_helplist: + self.helplist.insert(END, help_item[0]) + self.set_add_delete_state() - def save_new_theme(self, theme_name, theme): - """Save a newly created theme to idleConf. + def var_changed_startup_edit(self, *params): + "Store change to toggle for starting IDLE in the editor or shell." + value = self.startup_edit.get() + changes.add_option('main', 'General', 'editor-on-startup', value) - theme_name - string, the name of the new theme - theme - dictionary containing the new theme - """ - if not idleConf.userCfg['highlight'].has_section(theme_name): - idleConf.userCfg['highlight'].add_section(theme_name) - for element in theme: - value = theme[element] - idleConf.userCfg['highlight'].SetOption(theme_name, element, value) + def var_changed_autosave(self, *params): + "Store change to autosave." + value = self.autosave.get() + changes.add_option('main', 'General', 'autosave', value) - def deactivate_current_config(self): - """Remove current key bindings. + def var_changed_win_width(self, *params): + "Store change to window width." + value = self.win_width.get() + changes.add_option('main', 'EditorWindow', 'width', value) - Iterate over window instances defined in parent and remove - the keybindings. - """ - # Before a config is saved, some cleanup of current - # config must be done - remove the previous keybindings. - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: - instance.RemoveKeybindings() + def var_changed_win_height(self, *params): + "Store change to window height." + value = self.win_height.get() + changes.add_option('main', 'EditorWindow', 'height', value) - def activate_config_changes(self): - """Apply configuration changes to current windows. + def help_source_selected(self, event): + "Handle event for selecting additional help." + self.set_add_delete_state() - Dynamically update the current parent window instances - with some of the configuration changes. - """ - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: - instance.ResetColorizer() - instance.ResetFont() - instance.set_notabs_indentwidth() - instance.ApplyKeybindings() - instance.reset_help_menu_entries() + def set_add_delete_state(self): + "Toggle the state for the help list buttons based on list entries." + if self.helplist.size() < 1: # No entries in list. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED + else: # Some entries. + if self.helplist.curselection(): # There currently is a selection. + self.button_helplist_edit['state'] = NORMAL + self.button_helplist_remove['state'] = NORMAL + else: # There currently is not a selection. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED - def cancel(self): - """Dismiss config dialog. + def helplist_item_add(self): + """Handle add button for the help list. - Methods: - destroy: inherited + Query for name and location of new help sources and add + them to the list. """ - self.destroy() + help_source = HelpSource(self, 'New Help Source').result + if help_source: + self.user_helplist.append(help_source) + self.helplist.insert(END, help_source[0]) + self.update_help_changes() - def ok(self): - """Apply config changes, then dismiss dialog. + def helplist_item_edit(self): + """Handle edit button for the help list. - Methods: - apply - destroy: inherited + Query with existing help source information and update + config if the values are changed. """ - self.apply() - self.destroy() + item_index = self.helplist.index(ANCHOR) + help_source = self.user_helplist[item_index] + new_help_source = HelpSource( + self, 'Edit Help Source', + menuitem=help_source[0], + filepath=help_source[1], + ).result + if new_help_source and new_help_source != help_source: + self.user_helplist[item_index] = new_help_source + self.helplist.delete(item_index) + self.helplist.insert(item_index, new_help_source[0]) + self.update_help_changes() + self.set_add_delete_state() # Selected will be un-selected - def apply(self): - """Apply config changes and leave dialog open. + def helplist_item_remove(self): + """Handle remove button for the help list. - Methods: - deactivate_current_config - save_all_changed_extensions - activate_config_changes + Delete the help list item from config. """ - self.deactivate_current_config() - changes.save_all() - self.save_all_changed_extensions() - self.activate_config_changes() - - def help(self): - """Create textview for config dialog help. + item_index = self.helplist.index(ANCHOR) + del(self.user_helplist[item_index]) + self.helplist.delete(item_index) + self.update_help_changes() + self.set_add_delete_state() - Attrbutes accessed: - tab_pages + def update_help_changes(self): + "Clear and rebuild the HelpFiles section in changes" + changes['main']['HelpFiles'] = {} + for num in range(1, len(self.user_helplist) + 1): + changes.add_option( + 'main', 'HelpFiles', str(num), + ';'.join(self.user_helplist[num-1][:2])) - Methods: - view_text: Method from textview module. - """ - page = self.tab_pages._current_page - view_text(self, title='Help for IDLE preferences', - text=help_common+help_pages.get(page, '')) def create_page_extensions(self): """Part of the config dialog used for configuring IDLE extensions. diff --git a/Misc/NEWS.d/next/IDLE/2017-07-27-14-48-42.bpo-31060.GdY_VY.rst b/Misc/NEWS.d/next/IDLE/2017-07-27-14-48-42.bpo-31060.GdY_VY.rst new file mode 100644 index 00000000000000..1d202c7fa2c5fe --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-27-14-48-42.bpo-31060.GdY_VY.rst @@ -0,0 +1,3 @@ +IDLE - Finish rearranging methods of ConfigDialog Grouping methods +pertaining to each tab and the buttons will aid writing tests and improving +the tabs and will enable splitting the groups into classes. From 02f88d2a411a6a789b33be281adfc3570c49efd5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 28 Jul 2017 15:42:43 -0400 Subject: [PATCH 0608/1345] [3.6] bpo-30853: IDLE: Convert font and general vars to use VarTrace (GH-2914) (#2935) Instance tracers manages pairs consisting of a tk variable and a callback function. When tracing is turned on, setting the variable calls the function. Test coverage for the new class is 100%. (cherry picked from commit 5b59154) --- Lib/idlelib/configdialog.py | 104 +++++++----------- .../2017-07-27-10-01-14.bpo-30853.enPvvc.rst | 5 + 2 files changed, 47 insertions(+), 62 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-27-10-01-14.bpo-30853.enPvvc.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index e359ec24cd3a5b..92155e3726c16a 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -30,6 +30,7 @@ changes = ConfigChanges() + class ConfigDialog(Toplevel): """Config dialog for IDLE. """ @@ -75,13 +76,15 @@ def __init__(self, parent, title='', _htest=False, _utest=False): # self.bind('', self.Apply) #apply changes, save # self.bind('', self.Help) #context help self.load_configs() - self.attach_var_callbacks() # Avoid callbacks during load_configs. + # Avoid callbacks during load_configs. + tracers.attach() if not _utest: self.grab_set() self.wm_deiconify() self.wait_window() + def create_widgets(self): """Create and place widgets for tabbed dialog. @@ -96,7 +99,6 @@ def create_widgets(self): create_page_extensions create_action_buttons load_configs: Load pages except for extensions. - attach_var_callbacks remove_var_callbacks activate_config_changes: Tell editors to reload. """ @@ -131,37 +133,9 @@ def load_configs(self): self.load_general_cfg() # note: extension page handled separately - def attach_var_callbacks(self): - "Attach callbacks to variables that can be changed." - self.font_size.trace_add('write', self.var_changed_font) - self.font_name.trace_add('write', self.var_changed_font) - self.font_bold.trace_add('write', self.var_changed_font) - self.space_num.trace_add('write', self.var_changed_space_num) - self.color.trace_add('write', self.var_changed_color) - self.builtin_theme.trace_add('write', self.var_changed_builtin_theme) - self.custom_theme.trace_add('write', self.var_changed_custom_theme) - self.is_builtin_theme.trace_add('write', self.var_changed_is_builtin_theme) - self.highlight_target.trace_add('write', self.var_changed_highlight_target) - self.keybinding.trace_add('write', self.var_changed_keybinding) - self.builtin_keys.trace_add('write', self.var_changed_builtin_keys) - self.custom_keys.trace_add('write', self.var_changed_custom_keys) - self.are_keys_builtin.trace_add('write', self.var_changed_are_keys_builtin) - self.win_width.trace_add('write', self.var_changed_win_width) - self.win_height.trace_add('write', self.var_changed_win_height) - self.startup_edit.trace_add('write', self.var_changed_startup_edit) - self.autosave.trace_add('write', self.var_changed_autosave) - def remove_var_callbacks(self): "Remove callbacks to prevent memory leaks." - for var in ( - self.font_size, self.font_name, self.font_bold, - self.space_num, self.color, self.builtin_theme, - self.custom_theme, self.is_builtin_theme, self.highlight_target, - self.keybinding, self.builtin_keys, self.custom_keys, - self.are_keys_builtin, self.win_width, self.win_height, - self.startup_edit, self.autosave,): - var.trace_remove('write', var.trace_info()[0][1]) - + tracers.detach() def create_action_buttons(self): """Return frame of action buttons for dialog. @@ -273,7 +247,7 @@ def create_page_font_tab(self): Tabs: Enable users to change spaces entered for indent tabs. Changing indent_scale value with the mouse sets Var space_num, - which invokes var_changed_space_num, which adds an entry to + which invokes the default callback to add an entry to changes. Load_tab_cfg initializes space_num to default. Widget Structure: (*) widgets bound to self @@ -294,10 +268,10 @@ def create_page_font_tab(self): (*)indent_scale: Scale - space_num """ parent = self.parent - self.font_name = StringVar(parent) - self.font_size = StringVar(parent) - self.font_bold = BooleanVar(parent) - self.space_num = IntVar(parent) + self.font_name = tracers.add(StringVar(parent), self.var_changed_font) + self.font_size = tracers.add(StringVar(parent), self.var_changed_font) + self.font_bold = tracers.add(BooleanVar(parent), self.var_changed_font) + self.space_num = tracers.add(IntVar(parent), ('main', 'Indent', 'num-spaces')) # Create widgets: # body and body section frames. @@ -443,12 +417,6 @@ def load_tab_cfg(self): 'main', 'Indent', 'num-spaces', default=4, type='int') self.space_num.set(space_num) - def var_changed_space_num(self, *params): - "Store change to indentation size." - value = self.space_num.get() - changes.add_option('main', 'Indent', 'num-spaces', value) - - def create_page_highlight(self): """Return frame of widgets for Highlighting tab. @@ -518,12 +486,17 @@ def create_page_highlight(self): 'Shell Stderr Text': ('stderr', '13'), } parent = self.parent - self.builtin_theme = StringVar(parent) - self.custom_theme = StringVar(parent) + self.builtin_theme = tracers.add( + StringVar(parent), self.var_changed_builtin_theme) + self.custom_theme = tracers.add( + StringVar(parent), self.var_changed_custom_theme) self.fg_bg_toggle = BooleanVar(parent) - self.color = StringVar(parent) - self.is_builtin_theme = BooleanVar(parent) - self.highlight_target = StringVar(parent) + self.color = tracers.add( + StringVar(parent), self.var_changed_color) + self.is_builtin_theme = tracers.add( + BooleanVar(parent), self.var_changed_is_builtin_theme) + self.highlight_target = tracers.add( + StringVar(parent), self.var_changed_highlight_target) ##widget creation #body frame @@ -1062,10 +1035,14 @@ def create_page_keys(self): button_save_custom_keys: Button """ parent = self.parent - self.builtin_keys = StringVar(parent) - self.custom_keys = StringVar(parent) - self.are_keys_builtin = BooleanVar(parent) - self.keybinding = StringVar(parent) + self.builtin_keys = tracers.add( + StringVar(parent), self.var_changed_builtin_keys) + self.custom_keys = tracers.add( + StringVar(parent), self.var_changed_custom_keys) + self.are_keys_builtin = tracers.add( + BooleanVar(parent), self.var_changed_are_keys_builtin) + self.keybinding = tracers.add( + StringVar(parent), self.var_changed_keybinding) ##widget creation #body frame @@ -1169,9 +1146,6 @@ def load_key_cfg(self): keyset_name = idleConf.CurrentKeys() self.load_keys_list(keyset_name) - - - def var_changed_builtin_keys(self, *params): "Process selection of builtin key set." old_keys = ( @@ -1434,7 +1408,7 @@ def create_page_general(self): set var startup_edit. Radiobuttons save_ask_on and save_auto_on set var autosave. Entry boxes win_width_int and win_height_int set var win_width and win_height. Setting var_name invokes the - var_changed_var_name callback that adds option to changes. + default callback that adds option to changes. Helplist: load_general_cfg loads list user_helplist with name, position pairs and copies names to listbox helplist. @@ -1470,10 +1444,14 @@ def create_page_general(self): scroll_helplist: Scrollbar """ parent = self.parent - self.startup_edit = IntVar(parent) - self.autosave = IntVar(parent) - self.win_width = StringVar(parent) - self.win_height = StringVar(parent) + self.startup_edit = tracers.add( + IntVar(parent), ('main', 'General', 'editor-on-startup')) + self.autosave = tracers.add( + IntVar(parent), ('main', 'General', 'autosave')) + self.win_width = tracers.add( + StringVar(parent), ('main', 'EditorWindow', 'width')) + self.win_height = tracers.add( + StringVar(parent), ('main', 'EditorWindow', 'height')) # Create widgets: # body. @@ -1873,9 +1851,9 @@ def add(self, var, callback): Args: var: Tk variable instance. - callback: Function to be used as a callback or - a tuple with IdleConf values for default - callback. + callback: Either function name to be used as a callback + or a tuple with IdleConf config-type, section, and + option names used in the default callback. Return: Tk variable instance. @@ -1908,6 +1886,8 @@ def detach(self): self.untraced.append((var, callback)) +tracers = VarTrace() + help_common = '''\ When you click either the Apply or Ok buttons, settings in this dialog that are different from IDLE's default are saved in diff --git a/Misc/NEWS.d/next/IDLE/2017-07-27-10-01-14.bpo-30853.enPvvc.rst b/Misc/NEWS.d/next/IDLE/2017-07-27-10-01-14.bpo-30853.enPvvc.rst new file mode 100644 index 00000000000000..9cabca193b91cd --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-27-10-01-14.bpo-30853.enPvvc.rst @@ -0,0 +1,5 @@ +IDLE -- Factor a VarTrace class out of ConfigDialog. + +Instance tracers manages pairs consisting of a tk variable and a +callback function. When tracing is turned on, setting the variable +calls the function. Test coverage for the new class is 100%. From ecc80b3f1b56f1e4df9e592f8527e622a6b45e01 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 28 Jul 2017 18:36:30 -0400 Subject: [PATCH 0609/1345] [3.6] bpo-30853: IDLE - touch-up configdialog.VarTrace and tests. (GH-2936) (#2937) Add clear method for tests. Adjust tests to use global instance. Remove unneeded ConfigDialog method. (cherry picked from commit 5d0f30a) --- Lib/idlelib/configdialog.py | 10 ++++---- Lib/idlelib/idle_test/test_configdialog.py | 29 ++++++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 92155e3726c16a..87e0d685b778c5 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -99,7 +99,6 @@ def create_widgets(self): create_page_extensions create_action_buttons load_configs: Load pages except for extensions. - remove_var_callbacks activate_config_changes: Tell editors to reload. """ self.tab_pages = TabbedPageSet(self, @@ -133,10 +132,6 @@ def load_configs(self): self.load_general_cfg() # note: extension page handled separately - def remove_var_callbacks(self): - "Remove callbacks to prevent memory leaks." - tracers.detach() - def create_action_buttons(self): """Return frame of action buttons for dialog. @@ -1846,6 +1841,11 @@ def __init__(self): self.untraced = [] self.traced = [] + def clear(self): + "Clear lists (for tests)." + self.untraced.clear() + self.traced.clear() + def add(self, var, callback): """Add (var, callback) tuple to untraced list. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 7296075fbecc21..cf6c3b31ed5b7d 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -11,7 +11,7 @@ from idlelib.idle_test.mock_idle import Func from tkinter import Tk, IntVar, BooleanVar, DISABLED, NORMAL from idlelib import config -from idlelib.configdialog import ConfigDialog, idleConf, changes, VarTrace +from idlelib.configdialog import idleConf, changes, tracers # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. @@ -35,12 +35,12 @@ def setUpModule(): idleConf.userCfg = testcfg root = Tk() # root.withdraw() # Comment out, see issue 30870 - dialog = ConfigDialog(root, 'Test', _utest=True) + dialog = configdialog.ConfigDialog(root, 'Test', _utest=True) def tearDownModule(): global root, dialog idleConf.userCfg = usercfg - dialog.remove_var_callbacks() + tracers.detach() del dialog root.update_idletasks() root.destroy() @@ -423,14 +423,14 @@ def test_update_help_changes(self): d.update_help_changes = Func() -class TestVarTrace(unittest.TestCase): +class VarTraceTest(unittest.TestCase): def setUp(self): changes.clear() + tracers.clear() self.v1 = IntVar(root) self.v2 = BooleanVar(root) self.called = 0 - self.tracers = VarTrace() def tearDown(self): del self.v1, self.v2 @@ -442,11 +442,19 @@ def var_changed_boolean(self, *params): pass def test_init(self): - self.assertEqual(self.tracers.untraced, []) - self.assertEqual(self.tracers.traced, []) + tracers.__init__() + self.assertEqual(tracers.untraced, []) + self.assertEqual(tracers.traced, []) + + def test_clear(self): + tracers.untraced.append(0) + tracers.traced.append(1) + tracers.clear() + self.assertEqual(tracers.untraced, []) + self.assertEqual(tracers.traced, []) def test_add(self): - tr = self.tracers + tr = tracers func = Func() cb = tr.make_callback = mock.Mock(return_value=func) @@ -469,8 +477,7 @@ def test_add(self): del tr.make_callback def test_make_callback(self): - tr = self.tracers - cb = tr.make_callback(self.v1, ('main', 'section', 'option')) + cb = tracers.make_callback(self.v1, ('main', 'section', 'option')) self.assertTrue(callable(cb)) self.v1.set(42) # Not attached, so set didn't invoke the callback. @@ -481,7 +488,7 @@ def test_make_callback(self): self.assertEqual(changes['main']['section']['option'], '42') def test_attach_detach(self): - tr = self.tracers + tr = tracers v1 = tr.add(self.v1, self.var_changed_increment) v2 = tr.add(self.v2, self.var_changed_boolean) expected = [(v1, self.var_changed_increment), From e05119ad4e8285dfc48050a15ebe0cbaa066e310 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 28 Jul 2017 19:55:23 -0700 Subject: [PATCH 0610/1345] Fix trivial typo in multiprocessing documentation (GH-2930) (GH-2940) (cherry picked from commit 6fcb69dad579cc9a7dc15eabead43b6c37464f8c) --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 96d1424126caed..2f770b6322309e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1024,7 +1024,7 @@ Connection objects are usually created using :func:`Pipe` -- see also .. method:: recv() Return an object sent from the other end of the connection using - :meth:`send`. Blocks until there its something to receive. Raises + :meth:`send`. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. From 8364feff6730f54063116b607605167d9027ce85 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jul 2017 01:28:05 -0400 Subject: [PATCH 0611/1345] [3.6] bpo-30781: IDLE - Use ttk Notebook in ConfigDialog (GH-2938) (#2944) The notebook looks a bit better. It will work better with separate page classes. Traversal of widgets by Tab works better. Switching tabs with keys becomes possible. The font sample box works better at large font sizes. One of the two simulated click tests no longer works. This will be investigated while fixing a bug with the widget itself. (cherry picked from commit b331f80) --- Lib/idlelib/configdialog.py | 61 ++++++++++--------- Lib/idlelib/idle_test/test_configdialog.py | 16 +++-- .../2017-07-28-18-59-06.bpo-30781.ud5m18.rst | 1 + 3 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-28-18-59-06.bpo-30781.ud5m18.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 87e0d685b778c5..67165298a9963e 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -15,7 +15,7 @@ TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) -from tkinter.ttk import Scrollbar +from tkinter.ttk import Notebook, Scrollbar import tkinter.colorchooser as tkColorChooser import tkinter.font as tkFont import tkinter.messagebox as tkMessageBox @@ -101,15 +101,19 @@ def create_widgets(self): load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.tab_pages = TabbedPageSet(self, - page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', - 'Extensions']) - self.tab_pages.pack(side=TOP, expand=TRUE, fill=BOTH) - self.create_page_font_tab() - self.create_page_highlight() - self.create_page_keys() - self.create_page_general() - self.create_page_extensions() + self.note = note = Notebook(self, width=450, height=450) + fontpage = self.create_page_font_tab() + highpage = self.create_page_highlight() + keyspage = self.create_page_keys() + genpage = self.create_page_general() + extpage = self.create_page_extensions() + note.add(fontpage, text='Fonts/Tabs') + note.add(highpage, text='Highlights') + note.add(keyspage, text=' Keys ') + note.add(genpage, text=' General ') + note.add(extpage, text='Extensions') + note.enable_traversal() + note.pack(side=TOP, expand=TRUE, fill=BOTH) self.create_action_buttons().pack(side=BOTTOM) def load_configs(self): @@ -270,7 +274,7 @@ def create_page_font_tab(self): # Create widgets: # body and body section frames. - frame = self.tab_pages.pages['Fonts/Tabs'].frame + frame = Frame(self.note) frame_font = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') frame_indent = LabelFrame( @@ -281,7 +285,7 @@ def create_page_font_tab(self): font_name_title = Label( frame_font_name, justify=LEFT, text='Font Face :') self.fontlist = Listbox(frame_font_name, height=5, - takefocus=FALSE, exportselection=FALSE) + takefocus=True, exportselection=FALSE) self.fontlist.bind('', self.on_fontlist_select) self.fontlist.bind('', self.on_fontlist_select) self.fontlist.bind('', self.on_fontlist_select) @@ -297,7 +301,7 @@ def create_page_font_tab(self): temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) self.font_sample = Label( frame_font_sample, justify=LEFT, font=temp_font, - text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') + text='AaBbCcDdEe\nFfGgHhIiJj\n1234567890\n#:+=(){}[]') # frame_indent. indent_title = Label( frame_indent, justify=LEFT, @@ -493,10 +497,9 @@ def create_page_highlight(self): self.highlight_target = tracers.add( StringVar(parent), self.var_changed_highlight_target) - ##widget creation - #body frame - frame = self.tab_pages.pages['Highlighting'].frame - #body section frames + # Widget creation: + # body frame and section frames + frame = Frame(self.note) frame_custom = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Custom Highlighting ') frame_theme = LabelFrame(frame, borderwidth=2, relief=GROOVE, @@ -504,12 +507,12 @@ def create_page_highlight(self): #frame_custom self.highlight_sample=Text( frame_custom, relief=SOLID, borderwidth=1, - font=('courier', 12, ''), cursor='hand2', width=21, height=11, + font=('courier', 12, ''), cursor='hand2', width=21, height=13, takefocus=FALSE, highlightthickness=0, wrap=NONE) text=self.highlight_sample text.bind('', lambda e: 'break') text.bind('', lambda e: 'break') - text_and_tags=( + text_and_tags=(('\n', 'normal'), ('#you can click here', 'comment'), ('\n', 'normal'), ('#to choose items', 'comment'), ('\n', 'normal'), ('def', 'keyword'), (' ', 'normal'), @@ -525,7 +528,7 @@ def create_page_highlight(self): ('cursor |', 'cursor'), ('\n ', 'normal'), ('shell', 'console'), (' ', 'normal'), ('stdout', 'stdout'), (' ', 'normal'), - ('stderr', 'stderr'), ('\n', 'normal')) + ('stderr', 'stderr'), ('\n\n', 'normal')) for texttag in text_and_tags: text.insert(END, texttag[0], texttag[1]) for element in self.theme_elements: @@ -1039,10 +1042,9 @@ def create_page_keys(self): self.keybinding = tracers.add( StringVar(parent), self.var_changed_keybinding) - ##widget creation - #body frame - frame = self.tab_pages.pages['Keys'].frame - #body section frames + # Widget creation: + # body and section frames. + frame = Frame(self.note) frame_custom = LabelFrame( frame, borderwidth=2, relief=GROOVE, text=' Custom Key Bindings ') @@ -1449,9 +1451,8 @@ def create_page_general(self): StringVar(parent), ('main', 'EditorWindow', 'height')) # Create widgets: - # body. - frame = self.tab_pages.pages['General'].frame - # body section frames. + # body and section frames. + frame = Frame(self.note) frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Startup Preferences ') frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, @@ -1488,7 +1489,7 @@ def create_page_general(self): frame_helplist = Frame(frame_help) frame_helplist_buttons = Frame(frame_helplist) self.helplist = Listbox( - frame_helplist, height=5, takefocus=FALSE, + frame_helplist, height=5, takefocus=True, exportselection=FALSE) scroll_helplist = Scrollbar(frame_helplist) scroll_helplist['command'] = self.helplist.yview @@ -1669,7 +1670,7 @@ def create_page_extensions(self): save_all_changed_extensions: Call extension page Save(). """ parent = self.parent - frame = self.tab_pages.pages['Extensions'].frame + frame = Frame(self.note) self.ext_defaultCfg = idleConf.defaultCfg['extensions'] self.ext_userCfg = idleConf.userCfg['extensions'] self.is_int = self.register(is_int) @@ -1704,6 +1705,8 @@ def create_page_extensions(self): self.extension_list.selection_set(0) self.extension_selected(None) + return frame + def load_extensions(self): "Fill self.extensions with data from the default and user configs." self.extensions = {} diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index cf6c3b31ed5b7d..aff3c2fdc3b2b0 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -296,20 +296,24 @@ def test_source_selected(self): d.set = d.set_add_delete_state d.upc = d.update_help_changes helplist = d.helplist - helplist.insert(0, 'source') - helplist.activate(0) + dex = 'end' + helplist.insert(dex, 'source') + helplist.activate(dex) helplist.focus_force() - helplist.see(0) + helplist.see(dex) helplist.update() - x, y, dx, dy = helplist.bbox(0) + x, y, dx, dy = helplist.bbox(dex) x += dx // 2 y += dy // 2 d.set.called = d.upc.called = 0 + helplist.event_generate('', x=0, y=0) + helplist.event_generate('', x=x, y=y) helplist.event_generate('', x=x, y=y) helplist.event_generate('', x=x, y=y) - self.assertEqual(helplist.get('anchor'), 'source') - self.assertTrue(d.set.called) + # The following fail after the switch to + # self.assertEqual(helplist.get('anchor'), 'source') + # self.assertTrue(d.set.called) self.assertFalse(d.upc.called) def test_set_add_delete_state(self): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-28-18-59-06.bpo-30781.ud5m18.rst b/Misc/NEWS.d/next/IDLE/2017-07-28-18-59-06.bpo-30781.ud5m18.rst new file mode 100644 index 00000000000000..18f40a41f291b2 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-28-18-59-06.bpo-30781.ud5m18.rst @@ -0,0 +1 @@ +IDLE - Use ttk Notebook in ConfigDialog From b26cc82b2252dd54e73661a026b32e7808fef945 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jul 2017 14:38:24 -0400 Subject: [PATCH 0612/1345] [3.6] bpo-31027: Fix test_listcomps failure when run directly (GH-2939) (#2945) Bug appears to be incomplete copy-paste-edit. (cherry picked from commit ceb93f4) --- Lib/test/test_listcomps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1f8265f077e662..ddb169fe58957c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -139,7 +139,7 @@ def test_main(verbose=None): import gc counts = [None] * 5 for i in range(len(counts)): - support.run_doctest(test_genexps, verbose) + support.run_doctest(test_listcomps, verbose) gc.collect() counts[i] = sys.gettotalrefcount() print(counts) From 4c7b368de7bcabdd821059c023c46c9d85668d3f Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 29 Jul 2017 18:56:06 -0400 Subject: [PATCH 0613/1345] [3.6] bpo-30803: clarify truth value testing documentation (GH-2508) (#2946) Initial patch by Peter Thomassen. (cherry picked from commit caa1280) --- Doc/library/stdtypes.rst | 33 ++++++++----------- .../2017-07-29-14-55-50.bpo-30803.6hutqQ.rst | 1 + 2 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2017-07-29-14-55-50.bpo-30803.6hutqQ.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index e929f0b68767d4..b8c4d59363d2a6 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -39,31 +39,26 @@ Truth Value Testing single: false Any object can be tested for truth value, for use in an :keyword:`if` or -:keyword:`while` condition or as operand of the Boolean operations below. The -following values are considered false: +:keyword:`while` condition or as operand of the Boolean operations below. - .. index:: single: None (Built-in object) - -* ``None`` - - .. index:: single: False (Built-in object) - -* ``False`` - -* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``. +.. index:: single: true -* any empty sequence, for example, ``''``, ``()``, ``[]``. +By default, an object is considered true unless its class defines either a +:meth:`__bool__` method that returns ``False`` or a :meth:`__len__` method that +returns zero, when called with the object. [1]_ Here are most of the built-in +objects considered false: -* any empty mapping, for example, ``{}``. + .. index:: + single: None (Built-in object) + single: False (Built-in object) -* instances of user-defined classes, if the class defines a :meth:`__bool__` or - :meth:`__len__` method, when that method returns the integer zero or - :class:`bool` value ``False``. [1]_ +* constants defined to be false: ``None`` and ``False``. -.. index:: single: true +* zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, + ``Fraction(0, 1)`` -All other values are considered true --- so objects of many types are always -true. +* empty sequences and collections: ``''``, ``()``, ``[]``, ``{}``, ``set()``, + ``range(0)`` .. index:: operator: or diff --git a/Misc/NEWS.d/next/Documentation/2017-07-29-14-55-50.bpo-30803.6hutqQ.rst b/Misc/NEWS.d/next/Documentation/2017-07-29-14-55-50.bpo-30803.6hutqQ.rst new file mode 100644 index 00000000000000..4699713ee6e286 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2017-07-29-14-55-50.bpo-30803.6hutqQ.rst @@ -0,0 +1 @@ +Clarify doc on truth value testing. Original patch by Peter Thomassen. From c9c85321df7803137a413d26db41aaa377b13d79 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jul 2017 13:35:46 -0400 Subject: [PATCH 0614/1345] [3.6] bpo-30928: Update IDLE News.txt. (GH-2948) (#2949) (cherry picked from commit c3aa47f655abe564a2c2fb6d091ab19bdc0113b5) --- Lib/idlelib/NEWS.txt | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 3c5cba6af477fe..35a0115c73b37c 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,44 @@ Released on 2017-09-25? ======================== +bpo-30781: IDLE - Use ttk Notebook in ConfigDialog. +Patch by Terry Jan Reedy. + +bpo-31060: IDLE - Finish rearranging methods of ConfigDialog. +Grouping methods pertaining to each tab and the buttons will aid +writing tests and improving the tabs and will enable splitting the +groups into classes. +Patch by Terry Jan Reedy. + +bpo-30853: IDLE -- Factor a VarTrace class out of ConfigDialog. +Instance tracers manages pairs consisting of a tk variable and a +callback function. When tracing is turned on, setting the variable +calls the function. Test coverage for the new class is 100%. +Patch by Terry Jan Reedy. + +bpo-31003: IDLE: Add more tests for General tab. +Patch by Terry Jan Reedy. + +bpo-30993: IDLE - Improve configdialog font page and tests. +*In configdialog: Document causal pathways in create_font_tab +docstring. Simplify some attribute names. Move set_samples calls to +var_changed_font (idea from Cheryl Sabella). Move related functions to +positions after the create widgets function. +* In test_configdialog: Fix test_font_set so not order dependent. Fix +renamed test_indent_scale so it tests the widget. Adjust tests for +movement of set_samples call. Add tests for load functions. Put all +font tests in one class and tab indent tests in another. Except for +two lines, these tests completely cover the related functions. +Patch by Terry Jan Reedy. + +bpo-30981: IDLE -- Add more configdialog font page tests. + +bpo-28523: IDLE: replace 'colour' with 'color' in configdialog. + +bpo-30917: Add tests for idlelib.config.IdleConf. +Increase coverage from 46% to 96%. +Patch by Louie Lu. + bpo-30913: Document ConfigDialog tk Vars, methods, and widgets in docstrings This will facilitate improving the dialog and splitting up the class. Original patch by Cheryl Sabella. @@ -11,7 +49,7 @@ bpo-30899: Add tests for ConfigParser subclasses in config. Coverage is 100% for those classes and ConfigChanges. Patch by Louie Lu. -bpo-30881: Add docstrings to browser.py. +bpo-30881: Add docstrings to browser.py. Patch by Cheryl Sabella. bpo-30851: Remove unused tk variables in configdialog. @@ -48,7 +86,7 @@ bp0-30723: Make several improvements to parenmatch. * Make 'default' style, which is not default, a synonym for 'opener'. * Make time-delay work the same with all styles. * Add help for config dialog extensions tab, including parenmatch. -* Add new tests. +* Add new tests. Original patch by Charles Wohlganger. Revisions by Terry Jan Reedy bpo-30674: Grep -- Add docstrings. Patch by Cheryl Sabella. From 7582226a92481ca63dedbfe14ef465d1349d66a9 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jul 2017 15:00:50 -0400 Subject: [PATCH 0615/1345] [3.6] bpo-31004: IDLE: Factor out FontPage class from configdialog (step 1) (GH-2905) (#2950) The slightly modified tests continue to pass. The General test broken by the switch to Notebook is fixed. Patch mostly by Cheryl Sabella. (cherry picked from commit 9397e2a) --- Lib/idlelib/configdialog.py | 438 +++++++++--------- Lib/idlelib/idle_test/test_configdialog.py | 68 +-- .../2017-07-30-01-00-58.bpo-31004.m8cc1t.rst | 4 + 3 files changed, 271 insertions(+), 239 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-30-01-00-58.bpo-31004.m8cc1t.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 67165298a9963e..92bbc106344d94 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -69,7 +69,7 @@ def __init__(self, parent, title='', _htest=False, _utest=False): self.resizable(height=FALSE, width=FALSE) self.transient(parent) self.protocol("WM_DELETE_WINDOW", self.cancel) - self.fontlist.focus_set() + self.fontpage.fontlist.focus_set() # XXX Decide whether to keep or delete these key bindings. # Key bindings for this dialog. # self.bind('', self.Cancel) #dismiss dialog, no save @@ -102,16 +102,16 @@ def create_widgets(self): activate_config_changes: Tell editors to reload. """ self.note = note = Notebook(self, width=450, height=450) - fontpage = self.create_page_font_tab() - highpage = self.create_page_highlight() - keyspage = self.create_page_keys() - genpage = self.create_page_general() - extpage = self.create_page_extensions() - note.add(fontpage, text='Fonts/Tabs') - note.add(highpage, text='Highlights') - note.add(keyspage, text=' Keys ') - note.add(genpage, text=' General ') - note.add(extpage, text='Extensions') + self.highpage = self.create_page_highlight() + self.fontpage = FontPage(note, self.highpage) + self.keyspage = self.create_page_keys() + self.genpage = self.create_page_general() + self.extpage = self.create_page_extensions() + note.add(self.fontpage, text='Fonts/Tabs') + note.add(self.highpage, text='Highlights') + note.add(self.keyspage, text=' Keys ') + note.add(self.genpage, text=' General ') + note.add(self.extpage, text='Extensions') note.enable_traversal() note.pack(side=TOP, expand=TRUE, fill=BOTH) self.create_action_buttons().pack(side=BOTTOM) @@ -129,8 +129,8 @@ def load_configs(self): load_key_cfg load_general_cfg """ - self.load_font_cfg() - self.load_tab_cfg() + #self.load_font_cfg() + #self.load_tab_cfg() self.load_theme_cfg() self.load_key_cfg() self.load_general_cfg() @@ -219,203 +219,6 @@ def help(self): text=help_common+help_pages.get(page, '')) - def create_page_font_tab(self): - """Return frame of widgets for Font/Tabs tab. - - Fonts: Enable users to provisionally change font face, size, or - boldness and to see the consequence of proposed choices. Each - action set 3 options in changes structuree and changes the - corresponding aspect of the font sample on this page and - highlight sample on highlight page. - - Funtion load_font_cfg initializes font vars and widgets from - idleConf entries and tk. - - Fontlist: mouse button 1 click or up or down key invoke - on_fontlist_select(), which sets var font_name. - - Sizelist: clicking the menubutton opens the dropdown menu. A - mouse button 1 click or return key sets var font_size. - - Bold_toggle: clicking the box toggles var font_bold. - - Changing any of the font vars invokes var_changed_font, which - adds all 3 font options to changes and calls set_samples. - Set_samples applies a new font constructed from the font vars to - font_sample and to highlight_sample on the hightlight page. - - Tabs: Enable users to change spaces entered for indent tabs. - Changing indent_scale value with the mouse sets Var space_num, - which invokes the default callback to add an entry to - changes. Load_tab_cfg initializes space_num to default. - - Widget Structure: (*) widgets bound to self - frame (of tab_pages) - frame_font: LabelFrame - frame_font_name: Frame - font_name_title: Label - (*)fontlist: ListBox - font_name - scroll_font: Scrollbar - frame_font_param: Frame - font_size_title: Label - (*)sizelist: DynOptionMenu - font_size - (*)bold_toggle: Checkbutton - font_bold - frame_font_sample: Frame - (*)font_sample: Label - frame_indent: LabelFrame - indent_title: Label - (*)indent_scale: Scale - space_num - """ - parent = self.parent - self.font_name = tracers.add(StringVar(parent), self.var_changed_font) - self.font_size = tracers.add(StringVar(parent), self.var_changed_font) - self.font_bold = tracers.add(BooleanVar(parent), self.var_changed_font) - self.space_num = tracers.add(IntVar(parent), ('main', 'Indent', 'num-spaces')) - - # Create widgets: - # body and body section frames. - frame = Frame(self.note) - frame_font = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') - frame_indent = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') - # frame_font. - frame_font_name = Frame(frame_font) - frame_font_param = Frame(frame_font) - font_name_title = Label( - frame_font_name, justify=LEFT, text='Font Face :') - self.fontlist = Listbox(frame_font_name, height=5, - takefocus=True, exportselection=FALSE) - self.fontlist.bind('', self.on_fontlist_select) - self.fontlist.bind('', self.on_fontlist_select) - self.fontlist.bind('', self.on_fontlist_select) - scroll_font = Scrollbar(frame_font_name) - scroll_font.config(command=self.fontlist.yview) - self.fontlist.config(yscrollcommand=scroll_font.set) - font_size_title = Label(frame_font_param, text='Size :') - self.sizelist = DynOptionMenu(frame_font_param, self.font_size, None) - self.bold_toggle = Checkbutton( - frame_font_param, variable=self.font_bold, - onvalue=1, offvalue=0, text='Bold') - frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) - temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) - self.font_sample = Label( - frame_font_sample, justify=LEFT, font=temp_font, - text='AaBbCcDdEe\nFfGgHhIiJj\n1234567890\n#:+=(){}[]') - # frame_indent. - indent_title = Label( - frame_indent, justify=LEFT, - text='Python Standard: 4 Spaces!') - self.indent_scale = Scale( - frame_indent, variable=self.space_num, - orient='horizontal', tickinterval=2, from_=2, to=16) - - # Pack widgets: - # body. - frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) - # frame_font. - frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) - frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) - font_name_title.pack(side=TOP, anchor=W) - self.fontlist.pack(side=LEFT, expand=TRUE, fill=X) - scroll_font.pack(side=LEFT, fill=Y) - font_size_title.pack(side=LEFT, anchor=W) - self.sizelist.pack(side=LEFT, anchor=W) - self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) - frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - self.font_sample.pack(expand=TRUE, fill=BOTH) - # frame_indent. - frame_indent.pack(side=TOP, fill=X) - indent_title.pack(side=TOP, anchor=W, padx=5) - self.indent_scale.pack(side=TOP, padx=5, fill=X) - - return frame - - def load_font_cfg(self): - """Load current configuration settings for the font options. - - Retrieve current font with idleConf.GetFont and font families - from tk. Setup fontlist and set font_name. Setup sizelist, - which sets font_size. Set font_bold. Setting font variables - calls set_samples (thrice). - """ - configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') - font_name = configured_font[0].lower() - font_size = configured_font[1] - font_bold = configured_font[2]=='bold' - - # Set editor font selection list and font_name. - fonts = list(tkFont.families(self)) - fonts.sort() - for font in fonts: - self.fontlist.insert(END, font) - self.font_name.set(font_name) - lc_fonts = [s.lower() for s in fonts] - try: - current_font_index = lc_fonts.index(font_name) - self.fontlist.see(current_font_index) - self.fontlist.select_set(current_font_index) - self.fontlist.select_anchor(current_font_index) - self.fontlist.activate(current_font_index) - except ValueError: - pass - # Set font size dropdown. - self.sizelist.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', - '16', '18', '20', '22', '25', '29', '34', '40'), - font_size) - # Set font weight. - self.font_bold.set(font_bold) - - def var_changed_font(self, *params): - """Store changes to font attributes. - - When one font attribute changes, save them all, as they are - not independent from each other. In particular, when we are - overriding the default font, we need to write out everything. - """ - value = self.font_name.get() - changes.add_option('main', 'EditorWindow', 'font', value) - value = self.font_size.get() - changes.add_option('main', 'EditorWindow', 'font-size', value) - value = self.font_bold.get() - changes.add_option('main', 'EditorWindow', 'font-bold', value) - self.set_samples() - - def on_fontlist_select(self, event): - """Handle selecting a font from the list. - - Event can result from either mouse click or Up or Down key. - Set font_name and example displays to selection. - """ - font = self.fontlist.get( - ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) - self.font_name.set(font.lower()) - - def set_samples(self, event=None): - """Update update both screen samples with the font settings. - - Called on font initialization and change events. - Accesses font_name, font_size, and font_bold Variables. - Updates font_sample and hightlight page highlight_sample. - """ - font_name = self.font_name.get() - font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL - new_font = (font_name, self.font_size.get(), font_weight) - self.font_sample['font'] = new_font - self.highlight_sample['font'] = new_font - - def load_tab_cfg(self): - """Load current configuration settings for the tab options. - - Attributes updated: - space_num: Set to value from idleConf. - """ - # Set indent sizes. - space_num = idleConf.GetOption( - 'main', 'Indent', 'num-spaces', default=4, type='int') - self.space_num.set(space_num) - def create_page_highlight(self): """Return frame of widgets for Highlighting tab. @@ -505,7 +308,7 @@ def create_page_highlight(self): frame_theme = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Highlighting Theme ') #frame_custom - self.highlight_sample=Text( + text = self.highlight_sample = frame.highlight_sample = Text( frame_custom, relief=SOLID, borderwidth=1, font=('courier', 12, ''), cursor='hand2', width=21, height=13, takefocus=FALSE, highlightthickness=0, wrap=NONE) @@ -1829,6 +1632,219 @@ def save_all_changed_extensions(self): self.ext_userCfg.Save() +class FontPage(Frame): + + def __init__(self, parent, highpage): + super().__init__(parent) + self.parent = parent + self.highlight_sample = highpage.highlight_sample + self.create_page_font_tab() + self.load_font_cfg() + self.load_tab_cfg() + + def create_page_font_tab(self): + """Return frame of widgets for Font/Tabs tab. + + Fonts: Enable users to provisionally change font face, size, or + boldness and to see the consequence of proposed choices. Each + action set 3 options in changes structuree and changes the + corresponding aspect of the font sample on this page and + highlight sample on highlight page. + + Function load_font_cfg initializes font vars and widgets from + idleConf entries and tk. + + Fontlist: mouse button 1 click or up or down key invoke + on_fontlist_select(), which sets var font_name. + + Sizelist: clicking the menubutton opens the dropdown menu. A + mouse button 1 click or return key sets var font_size. + + Bold_toggle: clicking the box toggles var font_bold. + + Changing any of the font vars invokes var_changed_font, which + adds all 3 font options to changes and calls set_samples. + Set_samples applies a new font constructed from the font vars to + font_sample and to highlight_sample on the hightlight page. + + Tabs: Enable users to change spaces entered for indent tabs. + Changing indent_scale value with the mouse sets Var space_num, + which invokes the default callback to add an entry to + changes. Load_tab_cfg initializes space_num to default. + + Widget Structure: (*) widgets bound to self + frame (of tab_pages) + frame_font: LabelFrame + frame_font_name: Frame + font_name_title: Label + (*)fontlist: ListBox - font_name + scroll_font: Scrollbar + frame_font_param: Frame + font_size_title: Label + (*)sizelist: DynOptionMenu - font_size + (*)bold_toggle: Checkbutton - font_bold + frame_font_sample: Frame + (*)font_sample: Label + frame_indent: LabelFrame + indent_title: Label + (*)indent_scale: Scale - space_num + """ + parent = self.parent + self.font_name = tracers.add(StringVar(parent), self.var_changed_font) + self.font_size = tracers.add(StringVar(parent), self.var_changed_font) + self.font_bold = tracers.add(BooleanVar(parent), self.var_changed_font) + self.space_num = tracers.add(IntVar(self), ('main', 'Indent', 'num-spaces')) + + # Create widgets: + # body and body section frames. + frame = self + frame_font = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') + frame_indent = LabelFrame( + frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') + # frame_font. + frame_font_name = Frame(frame_font) + frame_font_param = Frame(frame_font) + font_name_title = Label( + frame_font_name, justify=LEFT, text='Font Face :') + self.fontlist = Listbox(frame_font_name, height=5, + takefocus=True, exportselection=FALSE) + self.fontlist.bind('', self.on_fontlist_select) + self.fontlist.bind('', self.on_fontlist_select) + self.fontlist.bind('', self.on_fontlist_select) + scroll_font = Scrollbar(frame_font_name) + scroll_font.config(command=self.fontlist.yview) + self.fontlist.config(yscrollcommand=scroll_font.set) + font_size_title = Label(frame_font_param, text='Size :') + self.sizelist = DynOptionMenu(frame_font_param, self.font_size, None) + self.bold_toggle = Checkbutton( + frame_font_param, variable=self.font_bold, + onvalue=1, offvalue=0, text='Bold') + frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) + temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) + self.font_sample = Label( + frame_font_sample, justify=LEFT, font=temp_font, + text='AaBbCcDdEe\nFfGgHhIiJj\n1234567890\n#:+=(){}[]') + # frame_indent. + indent_title = Label( + frame_indent, justify=LEFT, + text='Python Standard: 4 Spaces!') + self.indent_scale = Scale( + frame_indent, variable=self.space_num, + orient='horizontal', tickinterval=2, from_=2, to=16) + + # Pack widgets: + # body. + frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) + # frame_font. + frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) + frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) + font_name_title.pack(side=TOP, anchor=W) + self.fontlist.pack(side=LEFT, expand=TRUE, fill=X) + scroll_font.pack(side=LEFT, fill=Y) + font_size_title.pack(side=LEFT, anchor=W) + self.sizelist.pack(side=LEFT, anchor=W) + self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) + frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.font_sample.pack(expand=TRUE, fill=BOTH) + # frame_indent. + frame_indent.pack(side=TOP, fill=X) + indent_title.pack(side=TOP, anchor=W, padx=5) + self.indent_scale.pack(side=TOP, padx=5, fill=X) + + return frame + + def load_font_cfg(self): + """Load current configuration settings for the font options. + + Retrieve current font with idleConf.GetFont and font families + from tk. Setup fontlist and set font_name. Setup sizelist, + which sets font_size. Set font_bold. Call set_samples. + """ + configured_font = idleConf.GetFont(self, 'main', 'EditorWindow') + font_name = configured_font[0].lower() + font_size = configured_font[1] + font_bold = configured_font[2]=='bold' + + # Set editor font selection list and font_name. + fonts = list(tkFont.families(self)) + fonts.sort() + for font in fonts: + self.fontlist.insert(END, font) + self.font_name.set(font_name) + lc_fonts = [s.lower() for s in fonts] + try: + current_font_index = lc_fonts.index(font_name) + self.fontlist.see(current_font_index) + self.fontlist.select_set(current_font_index) + self.fontlist.select_anchor(current_font_index) + self.fontlist.activate(current_font_index) + except ValueError: + pass + # Set font size dropdown. + self.sizelist.SetMenu(('7', '8', '9', '10', '11', '12', '13', '14', + '16', '18', '20', '22', '25', '29', '34', '40'), + font_size) + # Set font weight. + self.font_bold.set(font_bold) + self.set_samples() + + def var_changed_font(self, *params): + """Store changes to font attributes. + + When one font attribute changes, save them all, as they are + not independent from each other. In particular, when we are + overriding the default font, we need to write out everything. + """ + value = self.font_name.get() + changes.add_option('main', 'EditorWindow', 'font', value) + value = self.font_size.get() + changes.add_option('main', 'EditorWindow', 'font-size', value) + value = self.font_bold.get() + changes.add_option('main', 'EditorWindow', 'font-bold', value) + self.set_samples() + + def on_fontlist_select(self, event): + """Handle selecting a font from the list. + + Event can result from either mouse click or Up or Down key. + Set font_name and example displays to selection. + """ + font = self.fontlist.get( + ACTIVE if event.type.name == 'KeyRelease' else ANCHOR) + self.font_name.set(font.lower()) + + def set_samples(self, event=None): + """Update update both screen samples with the font settings. + + Called on font initialization and change events. + Accesses font_name, font_size, and font_bold Variables. + Updates font_sample and hightlight page highlight_sample. + """ + font_name = self.font_name.get() + font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + new_font = (font_name, self.font_size.get(), font_weight) + self.font_sample['font'] = new_font + self.highlight_sample['font'] = new_font + + def load_tab_cfg(self): + """Load current configuration settings for the tab options. + + Attributes updated: + space_num: Set to value from idleConf. + """ + # Set indent sizes. + space_num = idleConf.GetOption( + 'main', 'Indent', 'num-spaces', default=4, type='int') + self.space_num.set(space_num) + + def var_changed_space_num(self, *params): + "Store change to indentation size." + value = self.space_num.get() + changes.add_option('main', 'Indent', 'num-spaces', value) + + class VarTrace: """Maintain Tk variables trace state.""" diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index aff3c2fdc3b2b0..8da726b80c0773 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -9,7 +9,7 @@ import unittest from unittest import mock from idlelib.idle_test.mock_idle import Func -from tkinter import Tk, IntVar, BooleanVar, DISABLED, NORMAL +from tkinter import Tk, Frame, IntVar, BooleanVar, DISABLED, NORMAL from idlelib import config from idlelib.configdialog import idleConf, changes, tracers @@ -47,7 +47,7 @@ def tearDownModule(): del root -class FontTest(unittest.TestCase): +class FontPageTest(unittest.TestCase): """Test that font widgets enable users to make font changes. Test that widget actions set vars, that var changes add three @@ -56,11 +56,15 @@ class FontTest(unittest.TestCase): """ @classmethod def setUpClass(cls): - dialog.set_samples = Func() # Mask instance method. + page = cls.page = dialog.fontpage + #dialog.note.insert(0, page, text='copy') + #dialog.note.add(page, text='copyfgfg') + dialog.note.select(page) + page.set_samples = Func() # Mask instance method. @classmethod def tearDownClass(cls): - del dialog.set_samples # Unmask instance method. + del cls.page.set_samples # Unmask instance method. def setUp(self): changes.clear() @@ -68,7 +72,8 @@ def setUp(self): def test_load_font_cfg(self): # Leave widget load test to human visual check. # TODO Improve checks when add IdleConf.get_font_values. - d = dialog + tracers.detach() + d = self.page d.font_name.set('Fake') d.font_size.set('1') d.font_bold.set(True) @@ -77,16 +82,17 @@ def test_load_font_cfg(self): self.assertNotEqual(d.font_name.get(), 'Fake') self.assertNotEqual(d.font_size.get(), '1') self.assertFalse(d.font_bold.get()) - self.assertEqual(d.set_samples.called, 3) + self.assertEqual(d.set_samples.called, 1) + tracers.attach() def test_fontlist_key(self): # Up and Down keys should select a new font. - - if dialog.fontlist.size() < 2: - cls.skipTest('need at least 2 fonts') - fontlist = dialog.fontlist + d = self.page + if d.fontlist.size() < 2: + self.skipTest('need at least 2 fonts') + fontlist = d.fontlist fontlist.activate(0) - font = dialog.fontlist.get('active') + font = d.fontlist.get('active') # Test Down key. fontlist.focus_force() @@ -96,7 +102,7 @@ def test_fontlist_key(self): down_font = fontlist.get('active') self.assertNotEqual(down_font, font) - self.assertIn(dialog.font_name.get(), down_font.lower()) + self.assertIn(d.font_name.get(), down_font.lower()) # Test Up key. fontlist.focus_force() @@ -106,14 +112,14 @@ def test_fontlist_key(self): up_font = fontlist.get('active') self.assertEqual(up_font, font) - self.assertIn(dialog.font_name.get(), up_font.lower()) + self.assertIn(d.font_name.get(), up_font.lower()) def test_fontlist_mouse(self): # Click on item should select that item. - - if dialog.fontlist.size() < 2: + d = self.page + if d.fontlist.size() < 2: cls.skipTest('need at least 2 fonts') - fontlist = dialog.fontlist + fontlist = d.fontlist fontlist.activate(0) # Select next item in listbox @@ -129,17 +135,17 @@ def test_fontlist_mouse(self): font1 = fontlist.get(1) select_font = fontlist.get('anchor') self.assertEqual(select_font, font1) - self.assertIn(dialog.font_name.get(), font1.lower()) + self.assertIn(d.font_name.get(), font1.lower()) def test_sizelist(self): # Click on number shouod select that number - d = dialog + d = self.page d.sizelist.variable.set(40) self.assertEqual(d.font_size.get(), '40') def test_bold_toggle(self): # Click on checkbutton should invert it. - d = dialog + d = self.page d.font_bold.set(False) d.bold_toggle.invoke() self.assertTrue(d.font_bold.get()) @@ -154,7 +160,7 @@ def test_font_set(self): default_font = idleConf.GetFont(root, 'main', 'EditorWindow') default_size = str(default_font[1]) default_bold = default_font[2] == 'bold' - d = dialog + d = self.page d.font_size.set(default_size) d.font_bold.set(default_bold) d.set_samples.called = 0 @@ -183,7 +189,7 @@ def test_font_set(self): self.assertEqual(d.set_samples.called, 3) def test_set_samples(self): - d = dialog + d = self.page del d.set_samples # Unmask method for test d.font_sample, d.highlight_sample = {}, {} d.font_name.set('test') @@ -201,16 +207,21 @@ def test_set_samples(self): class IndentTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.page = dialog.fontpage + def test_load_tab_cfg(self): - d = dialog + d = self.page d.space_num.set(16) d.load_tab_cfg() self.assertEqual(d.space_num.get(), 4) def test_indent_scale(self): + d = self.page changes.clear() - dialog.indent_scale.set(26) - self.assertEqual(dialog.space_num.get(), 16) + d.indent_scale.set(20) + self.assertEqual(d.space_num.get(), 16) self.assertEqual(mainpage, {'Indent': {'num-spaces': '16'}}) @@ -234,8 +245,10 @@ class GeneralTest(unittest.TestCase): """ @classmethod def setUpClass(cls): - # Mask instance methods used by help functions. d = dialog + # Select General tab so can force focus on helplist. + d.note.select(d.genpage) + # Mask instance methods used by help functions. d.set = d.set_add_delete_state = Func() d.upc = d.update_help_changes = Func() @@ -311,9 +324,8 @@ def test_source_selected(self): helplist.event_generate('', x=x, y=y) helplist.event_generate('', x=x, y=y) helplist.event_generate('', x=x, y=y) - # The following fail after the switch to - # self.assertEqual(helplist.get('anchor'), 'source') - # self.assertTrue(d.set.called) + self.assertEqual(helplist.get('anchor'), 'source') + self.assertTrue(d.set.called) self.assertFalse(d.upc.called) def test_set_add_delete_state(self): diff --git a/Misc/NEWS.d/next/IDLE/2017-07-30-01-00-58.bpo-31004.m8cc1t.rst b/Misc/NEWS.d/next/IDLE/2017-07-30-01-00-58.bpo-31004.m8cc1t.rst new file mode 100644 index 00000000000000..47fed949d106fb --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-30-01-00-58.bpo-31004.m8cc1t.rst @@ -0,0 +1,4 @@ +IDLE - Factor FontPage(Frame) class from ConfigDialog. + +Slightly modified tests continue to pass. Fix General tests. Patch mostly by +Cheryl Sabella. From 8c4e5be1dfb4d1c74e8cc7ac925e3196818e07ac Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 30 Jul 2017 19:02:51 -0400 Subject: [PATCH 0616/1345] [3.6] bpo-31050: IDLE: Factor GenPage class from ConfigDialog (GH-2952) (#2955) The slightly modified tests for the General tab continue to pass. Patch by Cheryl Sabella. (cherry picked from commit e8eb17b) --- Lib/idlelib/configdialog.py | 498 +++++++++--------- Lib/idlelib/idle_test/test_configdialog.py | 53 +- .../2017-07-30-17-39-59.bpo-31050.AXR3kP.rst | 2 + 3 files changed, 269 insertions(+), 284 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-30-17-39-59.bpo-31050.AXR3kP.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 92bbc106344d94..28070a51709dd2 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -89,14 +89,14 @@ def create_widgets(self): """Create and place widgets for tabbed dialog. Widgets Bound to self: - tab_pages: TabbedPageSet + note: Notebook + highpage: self.create_page_highlight + fontpage: FontPage + keyspage: self.create_page_keys + genpage: GenPage + extpageL self.create_page_extensions Methods: - create_page_font_tab - create_page_highlight - create_page_keys - create_page_general - create_page_extensions create_action_buttons load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. @@ -105,7 +105,7 @@ def create_widgets(self): self.highpage = self.create_page_highlight() self.fontpage = FontPage(note, self.highpage) self.keyspage = self.create_page_keys() - self.genpage = self.create_page_general() + self.genpage = GenPage(note) self.extpage = self.create_page_extensions() note.add(self.fontpage, text='Fonts/Tabs') note.add(self.highpage, text='Highlights') @@ -133,7 +133,7 @@ def load_configs(self): #self.load_tab_cfg() self.load_theme_cfg() self.load_key_cfg() - self.load_general_cfg() + # self.load_general_cfg() # note: extension page handled separately def create_action_buttons(self): @@ -1198,256 +1198,6 @@ def activate_config_changes(self): instance.ApplyKeybindings() instance.reset_help_menu_entries() - - def create_page_general(self): - """Return frame of widgets for General tab. - - Enable users to provisionally change general options. Function - load_general_cfg intializes tk variables and helplist using - idleConf. Radiobuttons startup_shell_on and startup_editor_on - set var startup_edit. Radiobuttons save_ask_on and save_auto_on - set var autosave. Entry boxes win_width_int and win_height_int - set var win_width and win_height. Setting var_name invokes the - default callback that adds option to changes. - - Helplist: load_general_cfg loads list user_helplist with - name, position pairs and copies names to listbox helplist. - Clicking a name invokes help_source selected. Clicking - button_helplist_name invokes helplist_item_name, which also - changes user_helplist. These functions all call - set_add_delete_state. All but load call update_help_changes to - rewrite changes['main']['HelpFiles']. - - Widget Structure: (*) widgets bound to self - frame - frame_run: LabelFrame - startup_title: Label - (*)startup_editor_on: Radiobutton - startup_edit - (*)startup_shell_on: Radiobutton - startup_edit - frame_save: LabelFrame - run_save_title: Label - (*)save_ask_on: Radiobutton - autosave - (*)save_auto_on: Radiobutton - autosave - frame_win_size: LabelFrame - win_size_title: Label - win_width_title: Label - (*)win_width_int: Entry - win_width - win_height_title: Label - (*)win_height_int: Entry - win_height - frame_help: LabelFrame - frame_helplist: Frame - frame_helplist_buttons: Frame - (*)button_helplist_edit - (*)button_helplist_add - (*)button_helplist_remove - (*)helplist: ListBox - scroll_helplist: Scrollbar - """ - parent = self.parent - self.startup_edit = tracers.add( - IntVar(parent), ('main', 'General', 'editor-on-startup')) - self.autosave = tracers.add( - IntVar(parent), ('main', 'General', 'autosave')) - self.win_width = tracers.add( - StringVar(parent), ('main', 'EditorWindow', 'width')) - self.win_height = tracers.add( - StringVar(parent), ('main', 'EditorWindow', 'height')) - - # Create widgets: - # body and section frames. - frame = Frame(self.note) - frame_run = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frame_save = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' autosave Preferences ') - frame_win_size = Frame(frame, borderwidth=2, relief=GROOVE) - frame_help = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') - # frame_run. - startup_title = Label(frame_run, text='At Startup') - self.startup_editor_on = Radiobutton( - frame_run, variable=self.startup_edit, value=1, - text="Open Edit Window") - self.startup_shell_on = Radiobutton( - frame_run, variable=self.startup_edit, value=0, - text='Open Shell Window') - # frame_save. - run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.save_ask_on = Radiobutton( - frame_save, variable=self.autosave, value=0, - text="Prompt to Save") - self.save_auto_on = Radiobutton( - frame_save, variable=self.autosave, value=1, - text='No Prompt') - # frame_win_size. - win_size_title = Label( - frame_win_size, text='Initial Window Size (in characters)') - win_width_title = Label(frame_win_size, text='Width') - self.win_width_int = Entry( - frame_win_size, textvariable=self.win_width, width=3) - win_height_title = Label(frame_win_size, text='Height') - self.win_height_int = Entry( - frame_win_size, textvariable=self.win_height, width=3) - # frame_help. - frame_helplist = Frame(frame_help) - frame_helplist_buttons = Frame(frame_helplist) - self.helplist = Listbox( - frame_helplist, height=5, takefocus=True, - exportselection=FALSE) - scroll_helplist = Scrollbar(frame_helplist) - scroll_helplist['command'] = self.helplist.yview - self.helplist['yscrollcommand'] = scroll_helplist.set - self.helplist.bind('', self.help_source_selected) - self.button_helplist_edit = Button( - frame_helplist_buttons, text='Edit', state=DISABLED, - width=8, command=self.helplist_item_edit) - self.button_helplist_add = Button( - frame_helplist_buttons, text='Add', - width=8, command=self.helplist_item_add) - self.button_helplist_remove = Button( - frame_helplist_buttons, text='Remove', state=DISABLED, - width=8, command=self.helplist_item_remove) - - # Pack widgets: - # body. - frame_run.pack(side=TOP, padx=5, pady=5, fill=X) - frame_save.pack(side=TOP, padx=5, pady=5, fill=X) - frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) - frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - # frame_run. - startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_save. - run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_win_size. - win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) - win_height_title.pack(side=RIGHT, anchor=E, pady=5) - self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) - win_width_title.pack(side=RIGHT, anchor=E, pady=5) - # frame_help. - frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) - self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) - self.button_helplist_add.pack(side=TOP, anchor=W) - self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) - - return frame - - def load_general_cfg(self): - "Load current configuration settings for the general options." - # Set startup state. - self.startup_edit.set(idleConf.GetOption( - 'main', 'General', 'editor-on-startup', default=0, type='bool')) - # Set autosave state. - self.autosave.set(idleConf.GetOption( - 'main', 'General', 'autosave', default=0, type='bool')) - # Set initial window size. - self.win_width.set(idleConf.GetOption( - 'main', 'EditorWindow', 'width', type='int')) - self.win_height.set(idleConf.GetOption( - 'main', 'EditorWindow', 'height', type='int')) - # Set additional help sources. - self.user_helplist = idleConf.GetAllExtraHelpSourcesList() - self.helplist.delete(0, 'end') - for help_item in self.user_helplist: - self.helplist.insert(END, help_item[0]) - self.set_add_delete_state() - - def var_changed_startup_edit(self, *params): - "Store change to toggle for starting IDLE in the editor or shell." - value = self.startup_edit.get() - changes.add_option('main', 'General', 'editor-on-startup', value) - - def var_changed_autosave(self, *params): - "Store change to autosave." - value = self.autosave.get() - changes.add_option('main', 'General', 'autosave', value) - - def var_changed_win_width(self, *params): - "Store change to window width." - value = self.win_width.get() - changes.add_option('main', 'EditorWindow', 'width', value) - - def var_changed_win_height(self, *params): - "Store change to window height." - value = self.win_height.get() - changes.add_option('main', 'EditorWindow', 'height', value) - - def help_source_selected(self, event): - "Handle event for selecting additional help." - self.set_add_delete_state() - - def set_add_delete_state(self): - "Toggle the state for the help list buttons based on list entries." - if self.helplist.size() < 1: # No entries in list. - self.button_helplist_edit['state'] = DISABLED - self.button_helplist_remove['state'] = DISABLED - else: # Some entries. - if self.helplist.curselection(): # There currently is a selection. - self.button_helplist_edit['state'] = NORMAL - self.button_helplist_remove['state'] = NORMAL - else: # There currently is not a selection. - self.button_helplist_edit['state'] = DISABLED - self.button_helplist_remove['state'] = DISABLED - - def helplist_item_add(self): - """Handle add button for the help list. - - Query for name and location of new help sources and add - them to the list. - """ - help_source = HelpSource(self, 'New Help Source').result - if help_source: - self.user_helplist.append(help_source) - self.helplist.insert(END, help_source[0]) - self.update_help_changes() - - def helplist_item_edit(self): - """Handle edit button for the help list. - - Query with existing help source information and update - config if the values are changed. - """ - item_index = self.helplist.index(ANCHOR) - help_source = self.user_helplist[item_index] - new_help_source = HelpSource( - self, 'Edit Help Source', - menuitem=help_source[0], - filepath=help_source[1], - ).result - if new_help_source and new_help_source != help_source: - self.user_helplist[item_index] = new_help_source - self.helplist.delete(item_index) - self.helplist.insert(item_index, new_help_source[0]) - self.update_help_changes() - self.set_add_delete_state() # Selected will be un-selected - - def helplist_item_remove(self): - """Handle remove button for the help list. - - Delete the help list item from config. - """ - item_index = self.helplist.index(ANCHOR) - del(self.user_helplist[item_index]) - self.helplist.delete(item_index) - self.update_help_changes() - self.set_add_delete_state() - - def update_help_changes(self): - "Clear and rebuild the HelpFiles section in changes" - changes['main']['HelpFiles'] = {} - for num in range(1, len(self.user_helplist) + 1): - changes.add_option( - 'main', 'HelpFiles', str(num), - ';'.join(self.user_helplist[num-1][:2])) - - def create_page_extensions(self): """Part of the config dialog used for configuring IDLE extensions. @@ -1845,6 +1595,238 @@ def var_changed_space_num(self, *params): changes.add_option('main', 'Indent', 'num-spaces', value) +class GenPage(Frame): + + def __init__(self, parent): + super().__init__(parent) + self.create_page_general() + self.load_general_cfg() + + def create_page_general(self): + """Return frame of widgets for General tab. + + Enable users to provisionally change general options. Function + load_general_cfg intializes tk variables and helplist using + idleConf. Radiobuttons startup_shell_on and startup_editor_on + set var startup_edit. Radiobuttons save_ask_on and save_auto_on + set var autosave. Entry boxes win_width_int and win_height_int + set var win_width and win_height. Setting var_name invokes the + default callback that adds option to changes. + + Helplist: load_general_cfg loads list user_helplist with + name, position pairs and copies names to listbox helplist. + Clicking a name invokes help_source selected. Clicking + button_helplist_name invokes helplist_item_name, which also + changes user_helplist. These functions all call + set_add_delete_state. All but load call update_help_changes to + rewrite changes['main']['HelpFiles']. + + Widget Structure: (*) widgets bound to self + frame + frame_run: LabelFrame + startup_title: Label + (*)startup_editor_on: Radiobutton - startup_edit + (*)startup_shell_on: Radiobutton - startup_edit + frame_save: LabelFrame + run_save_title: Label + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave + frame_win_size: LabelFrame + win_size_title: Label + win_width_title: Label + (*)win_width_int: Entry - win_width + win_height_title: Label + (*)win_height_int: Entry - win_height + frame_help: LabelFrame + frame_helplist: Frame + frame_helplist_buttons: Frame + (*)button_helplist_edit + (*)button_helplist_add + (*)button_helplist_remove + (*)helplist: ListBox + scroll_helplist: Scrollbar + """ + self.startup_edit = tracers.add( + IntVar(self), ('main', 'General', 'editor-on-startup')) + self.autosave = tracers.add( + IntVar(self), ('main', 'General', 'autosave')) + self.win_width = tracers.add( + StringVar(self), ('main', 'EditorWindow', 'width')) + self.win_height = tracers.add( + StringVar(self), ('main', 'EditorWindow', 'height')) + + # Create widgets: + # Section frames. + frame_run = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Startup Preferences ') + frame_save = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' autosave Preferences ') + frame_win_size = Frame(self, borderwidth=2, relief=GROOVE) + frame_help = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Additional Help Sources ') + # frame_run. + startup_title = Label(frame_run, text='At Startup') + self.startup_editor_on = Radiobutton( + frame_run, variable=self.startup_edit, value=1, + text="Open Edit Window") + self.startup_shell_on = Radiobutton( + frame_run, variable=self.startup_edit, value=0, + text='Open Shell Window') + # frame_save. + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + self.save_ask_on = Radiobutton( + frame_save, variable=self.autosave, value=0, + text="Prompt to Save") + self.save_auto_on = Radiobutton( + frame_save, variable=self.autosave, value=1, + text='No Prompt') + # frame_win_size. + win_size_title = Label( + frame_win_size, text='Initial Window Size (in characters)') + win_width_title = Label(frame_win_size, text='Width') + self.win_width_int = Entry( + frame_win_size, textvariable=self.win_width, width=3) + win_height_title = Label(frame_win_size, text='Height') + self.win_height_int = Entry( + frame_win_size, textvariable=self.win_height, width=3) + # frame_help. + frame_helplist = Frame(frame_help) + frame_helplist_buttons = Frame(frame_helplist) + self.helplist = Listbox( + frame_helplist, height=5, takefocus=True, + exportselection=FALSE) + scroll_helplist = Scrollbar(frame_helplist) + scroll_helplist['command'] = self.helplist.yview + self.helplist['yscrollcommand'] = scroll_helplist.set + self.helplist.bind('', self.help_source_selected) + self.button_helplist_edit = Button( + frame_helplist_buttons, text='Edit', state=DISABLED, + width=8, command=self.helplist_item_edit) + self.button_helplist_add = Button( + frame_helplist_buttons, text='Add', + width=8, command=self.helplist_item_add) + self.button_helplist_remove = Button( + frame_helplist_buttons, text='Remove', state=DISABLED, + width=8, command=self.helplist_item_remove) + + # Pack widgets: + # body. + frame_run.pack(side=TOP, padx=5, pady=5, fill=X) + frame_save.pack(side=TOP, padx=5, pady=5, fill=X) + frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) + frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + # frame_run. + startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_save. + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_win_size. + win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_height_title.pack(side=RIGHT, anchor=E, pady=5) + self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) + win_width_title.pack(side=RIGHT, anchor=E, pady=5) + # frame_help. + frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) + frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) + self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) + self.button_helplist_add.pack(side=TOP, anchor=W) + self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) + + def load_general_cfg(self): + "Load current configuration settings for the general options." + # Set startup state. + self.startup_edit.set(idleConf.GetOption( + 'main', 'General', 'editor-on-startup', default=0, type='bool')) + # Set autosave state. + self.autosave.set(idleConf.GetOption( + 'main', 'General', 'autosave', default=0, type='bool')) + # Set initial window size. + self.win_width.set(idleConf.GetOption( + 'main', 'EditorWindow', 'width', type='int')) + self.win_height.set(idleConf.GetOption( + 'main', 'EditorWindow', 'height', type='int')) + # Set additional help sources. + self.user_helplist = idleConf.GetAllExtraHelpSourcesList() + self.helplist.delete(0, 'end') + for help_item in self.user_helplist: + self.helplist.insert(END, help_item[0]) + self.set_add_delete_state() + + def help_source_selected(self, event): + "Handle event for selecting additional help." + self.set_add_delete_state() + + def set_add_delete_state(self): + "Toggle the state for the help list buttons based on list entries." + if self.helplist.size() < 1: # No entries in list. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED + else: # Some entries. + if self.helplist.curselection(): # There currently is a selection. + self.button_helplist_edit['state'] = NORMAL + self.button_helplist_remove['state'] = NORMAL + else: # There currently is not a selection. + self.button_helplist_edit['state'] = DISABLED + self.button_helplist_remove['state'] = DISABLED + + def helplist_item_add(self): + """Handle add button for the help list. + + Query for name and location of new help sources and add + them to the list. + """ + help_source = HelpSource(self, 'New Help Source').result + if help_source: + self.user_helplist.append(help_source) + self.helplist.insert(END, help_source[0]) + self.update_help_changes() + + def helplist_item_edit(self): + """Handle edit button for the help list. + + Query with existing help source information and update + config if the values are changed. + """ + item_index = self.helplist.index(ANCHOR) + help_source = self.user_helplist[item_index] + new_help_source = HelpSource( + self, 'Edit Help Source', + menuitem=help_source[0], + filepath=help_source[1], + ).result + if new_help_source and new_help_source != help_source: + self.user_helplist[item_index] = new_help_source + self.helplist.delete(item_index) + self.helplist.insert(item_index, new_help_source[0]) + self.update_help_changes() + self.set_add_delete_state() # Selected will be un-selected + + def helplist_item_remove(self): + """Handle remove button for the help list. + + Delete the help list item from config. + """ + item_index = self.helplist.index(ANCHOR) + del(self.user_helplist[item_index]) + self.helplist.delete(item_index) + self.update_help_changes() + self.set_add_delete_state() + + def update_help_changes(self): + "Clear and rebuild the HelpFiles section in changes" + changes['main']['HelpFiles'] = {} + for num in range(1, len(self.user_helplist) + 1): + changes.add_option( + 'main', 'HelpFiles', str(num), + ';'.join(self.user_helplist[num-1][:2])) + + class VarTrace: """Maintain Tk variables trace state.""" diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 8da726b80c0773..caeb2b5678778d 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -237,7 +237,7 @@ def setUp(self): changes.clear() -class GeneralTest(unittest.TestCase): +class GenPageTest(unittest.TestCase): """Test that general tab widgets enable users to make changes. Test that widget actions set vars, that var changes add @@ -245,20 +245,18 @@ class GeneralTest(unittest.TestCase): """ @classmethod def setUpClass(cls): - d = dialog - # Select General tab so can force focus on helplist. - d.note.select(d.genpage) - # Mask instance methods used by help functions. - d.set = d.set_add_delete_state = Func() - d.upc = d.update_help_changes = Func() + page = cls.page = dialog.genpage + dialog.note.select(page) + page.set = page.set_add_delete_state = Func() + page.upc = page.update_help_changes = Func() @classmethod def tearDownClass(cls): - d = dialog - del d.set, d.set_add_delete_state - del d.upc, d.update_help_changes - d.helplist.delete(0, 'end') - d.user_helplist.clear() + page = cls.page + del page.set, page.set_add_delete_state + del page.upc, page.update_help_changes + page.helplist.delete(0, 'end') + page.user_helplist.clear() def setUp(self): changes.clear() @@ -266,7 +264,7 @@ def setUp(self): def test_load_general_cfg(self): # Set to wrong values, load, check right values. eq = self.assertEqual - d = dialog + d = self.page d.startup_edit.set(1) d.autosave.set(1) d.win_width.set(1) @@ -283,29 +281,32 @@ def test_load_general_cfg(self): eq(d.user_helplist, [('name', 'file', '1')]) def test_startup(self): - dialog.startup_editor_on.invoke() + d = self.page + d.startup_editor_on.invoke() self.assertEqual(mainpage, {'General': {'editor-on-startup': '1'}}) changes.clear() - dialog.startup_shell_on.invoke() + d.startup_shell_on.invoke() self.assertEqual(mainpage, {'General': {'editor-on-startup': '0'}}) def test_autosave(self): - dialog.save_auto_on.invoke() + d = self.page + d.save_auto_on.invoke() self.assertEqual(mainpage, {'General': {'autosave': '1'}}) - dialog.save_ask_on.invoke() + d.save_ask_on.invoke() self.assertEqual(mainpage, {'General': {'autosave': '0'}}) def test_editor_size(self): - dialog.win_height_int.insert(0, '1') + d = self.page + d.win_height_int.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'height': '140'}}) changes.clear() - dialog.win_width_int.insert(0, '1') + d.win_width_int.insert(0, '1') self.assertEqual(mainpage, {'EditorWindow': {'width': '180'}}) def test_source_selected(self): - d = dialog + d = self.page d.set = d.set_add_delete_state d.upc = d.update_help_changes helplist = d.helplist @@ -331,7 +332,7 @@ def test_source_selected(self): def test_set_add_delete_state(self): # Call with 0 items, 1 unselected item, 1 selected item. eq = self.assertEqual - d = dialog + d = self.page del d.set_add_delete_state # Unmask method. sad = d.set_add_delete_state h = d.helplist @@ -358,7 +359,7 @@ def test_helplist_item_add(self): eq = self.assertEqual orig_helpsource = configdialog.HelpSource hs = configdialog.HelpSource = Func(return_self=True) - d = dialog + d = self.page d.helplist.delete(0, 'end') d.user_helplist.clear() d.set.called = d.upc.called = 0 @@ -385,7 +386,7 @@ def test_helplist_item_edit(self): eq = self.assertEqual orig_helpsource = configdialog.HelpSource hs = configdialog.HelpSource = Func(return_self=True) - d = dialog + d = self.page d.helplist.delete(0, 'end') d.helplist.insert(0, 'name1') d.helplist.selection_set(0) @@ -412,7 +413,7 @@ def test_helplist_item_edit(self): def test_helplist_item_remove(self): eq = self.assertEqual - d = dialog + d = self.page d.helplist.delete(0, 'end') d.helplist.insert(0, 'name1') d.helplist.selection_set(0) @@ -427,7 +428,7 @@ def test_helplist_item_remove(self): self.assertTrue(d.upc.called == d.set.called == 1) def test_update_help_changes(self): - d = dialog + d = self.page del d.update_help_changes d.user_helplist.clear() d.user_helplist.append(('name1', 'file1')) @@ -435,7 +436,7 @@ def test_update_help_changes(self): d.update_help_changes() self.assertEqual(mainpage['HelpFiles'], - {'1': 'name1;file1', '2': 'name2;file2'}) + {'1': 'name1;file1', '2': 'name2;file2'}) d.update_help_changes = Func() diff --git a/Misc/NEWS.d/next/IDLE/2017-07-30-17-39-59.bpo-31050.AXR3kP.rst b/Misc/NEWS.d/next/IDLE/2017-07-30-17-39-59.bpo-31050.AXR3kP.rst new file mode 100644 index 00000000000000..e33b2e231d6733 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-30-17-39-59.bpo-31050.AXR3kP.rst @@ -0,0 +1,2 @@ +Factor GenPage(Frame) class from ConfigDialog. The slightly modified tests +continue to pass. Patch by Cheryl Sabella. From 2bf1586e60a6639b532cd8e3442ae33064523eb1 Mon Sep 17 00:00:00 2001 From: csabella Date: Mon, 31 Jul 2017 15:10:13 -0400 Subject: [PATCH 0617/1345] [3.6] bpo-25684: ttk.OptionMenu radiobuttons weren't unique (GH-2276) (#2959) between instances of OptionMenu. (cherry picked from commit a568e5273382a5dca0c27274f7d8e34c41a87d4d) * Update Misc/ACKS --- Lib/tkinter/test/test_ttk/test_extensions.py | 25 +++++++++++++++++++ Lib/tkinter/ttk.py | 3 ++- Misc/ACKS | 1 + .../2017-07-17-11-35-00.bpo-25684.usELVx.rst | 2 ++ 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-07-17-11-35-00.bpo-25684.usELVx.rst diff --git a/Lib/tkinter/test/test_ttk/test_extensions.py b/Lib/tkinter/test/test_ttk/test_extensions.py index 218b27fc30ea1b..a45f882bb00d48 100644 --- a/Lib/tkinter/test/test_ttk/test_extensions.py +++ b/Lib/tkinter/test/test_ttk/test_extensions.py @@ -291,6 +291,31 @@ def cb_test(item): optmenu.destroy() + def test_unique_radiobuttons(self): + # check that radiobuttons are unique across instances (bpo25684) + items = ('a', 'b', 'c') + default = 'a' + optmenu = ttk.OptionMenu(self.root, self.textvar, default, *items) + textvar2 = tkinter.StringVar(self.root) + optmenu2 = ttk.OptionMenu(self.root, textvar2, default, *items) + optmenu.pack() + optmenu.wait_visibility() + optmenu2.pack() + optmenu2.wait_visibility() + optmenu['menu'].invoke(1) + optmenu2['menu'].invoke(2) + optmenu_stringvar_name = optmenu['menu'].entrycget(0, 'variable') + optmenu2_stringvar_name = optmenu2['menu'].entrycget(0, 'variable') + self.assertNotEqual(optmenu_stringvar_name, + optmenu2_stringvar_name) + self.assertEqual(self.root.tk.globalgetvar(optmenu_stringvar_name), + items[1]) + self.assertEqual(self.root.tk.globalgetvar(optmenu2_stringvar_name), + items[2]) + + optmenu.destroy() + optmenu2.destroy() + tests_gui = (LabeledScaleTest, OptionMenuTest) diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c474e60713f75e..9cd0c2073a9f59 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1638,7 +1638,8 @@ def set_menu(self, default=None, *values): menu.delete(0, 'end') for val in values: menu.add_radiobutton(label=val, - command=tkinter._setit(self._variable, val, self._callback)) + command=tkinter._setit(self._variable, val, self._callback), + variable=self._variable) if default: self._variable.set(default) diff --git a/Misc/ACKS b/Misc/ACKS index b6b355b614b820..7bc13ff0c9e55a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1340,6 +1340,7 @@ Chris Ryland Bernt Røskar Brenna Constantina S. Matthieu S +Cheryl Sabella Patrick Sabin Sébastien Sablé Amit Saha diff --git a/Misc/NEWS.d/next/Library/2017-07-17-11-35-00.bpo-25684.usELVx.rst b/Misc/NEWS.d/next/Library/2017-07-17-11-35-00.bpo-25684.usELVx.rst new file mode 100644 index 00000000000000..61d6b29cafc3ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-17-11-35-00.bpo-25684.usELVx.rst @@ -0,0 +1,2 @@ +Change ``ttk.OptionMenu`` radiobuttons to be unique across instances of +``OptionMenu``. From 330cdac5174e3dc6c1deb876bc12a39d7af9af84 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 31 Jul 2017 19:59:50 -0700 Subject: [PATCH 0618/1345] bpo-25910: Update LICENSE (GH-2873) (GH-2967) Use the copy provided in https://bugs.python.org/issue25910GH-msg295200 (cherry picked from commit 8474d87165593bac2bc231287f42c4cff3fd6aaf) --- LICENSE | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index f5d0b39a0cdddb..529349e4b38c05 100644 --- a/LICENSE +++ b/LICENSE @@ -13,12 +13,11 @@ software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. All Python releases are Open Source (see http://www.opensource.org for the Open Source Definition). Historically, most, but not all, Python From f9463b8cc9e2906d3f457d59286665bee08f626f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 31 Jul 2017 21:34:36 -0700 Subject: [PATCH 0619/1345] Remove .mention-bot (GH-2923) (GH-2970) (cherry picked from commit e1e60bd95c376f9b18cbaf9481364281c6cc1bc7) --- .mention-bot | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .mention-bot diff --git a/.mention-bot b/.mention-bot deleted file mode 100644 index cb53b993fb1e8a..00000000000000 --- a/.mention-bot +++ /dev/null @@ -1,3 +0,0 @@ -{ - "findPotentialReviewers": false -} From 48fcc72c83e1b47b200dd39f9dcc9f62fa0d4d17 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 1 Aug 2017 01:00:33 -0400 Subject: [PATCH 0620/1345] [3.6] bpo-31083: IDLE: Describe the Page classes in configdialog (GH-2965) (#2973) Add template as comment. Update existing classes to match outline. Initial patch by Cheryl Sabella. (cherry picked from commit 6f446be) --- Lib/idlelib/configdialog.py | 42 ++++++++++++------- .../2017-07-31-23-20-51.bpo-31083.991FXm.rst | 3 ++ 2 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-07-31-23-20-51.bpo-31083.991FXm.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 28070a51709dd2..6d8a03b1c00bef 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1382,11 +1382,27 @@ def save_all_changed_extensions(self): self.ext_userCfg.Save() +# class TabPage(Frame): # A template for Page classes. +# def __init__(self, master): +# super().__init__(master) +# self.create_page_tab() +# self.load_tab_cfg() +# def create_page_tab(self): +# # Define tk vars and register var and callback with tracers. +# # Create subframes and widgets. +# # Pack widgets. +# def load_tab_cfg(self): +# # Initialize widgets with data from idleConf. +# def var_changed_var_name(): +# # For each tk var that needs other than default callback. +# def other_methods(): +# # Define tab-specific behavior. + + class FontPage(Frame): - def __init__(self, parent, highpage): - super().__init__(parent) - self.parent = parent + def __init__(self, master, highpage): + super().__init__(master) self.highlight_sample = highpage.highlight_sample self.create_page_font_tab() self.load_font_cfg() @@ -1439,19 +1455,17 @@ def create_page_font_tab(self): indent_title: Label (*)indent_scale: Scale - space_num """ - parent = self.parent - self.font_name = tracers.add(StringVar(parent), self.var_changed_font) - self.font_size = tracers.add(StringVar(parent), self.var_changed_font) - self.font_bold = tracers.add(BooleanVar(parent), self.var_changed_font) + self.font_name = tracers.add(StringVar(self), self.var_changed_font) + self.font_size = tracers.add(StringVar(self), self.var_changed_font) + self.font_bold = tracers.add(BooleanVar(self), self.var_changed_font) self.space_num = tracers.add(IntVar(self), ('main', 'Indent', 'num-spaces')) # Create widgets: # body and body section frames. - frame = self frame_font = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') + self, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') frame_indent = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') + self, borderwidth=2, relief=GROOVE, text=' Indentation Width ') # frame_font. frame_font_name = Frame(frame_font) frame_font_param = Frame(frame_font) @@ -1471,7 +1485,7 @@ def create_page_font_tab(self): frame_font_param, variable=self.font_bold, onvalue=1, offvalue=0, text='Bold') frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) - temp_font = tkFont.Font(parent, ('courier', 10, 'normal')) + temp_font = tkFont.Font(self, ('courier', 10, 'normal')) self.font_sample = Label( frame_font_sample, justify=LEFT, font=temp_font, text='AaBbCcDdEe\nFfGgHhIiJj\n1234567890\n#:+=(){}[]') @@ -1503,8 +1517,6 @@ def create_page_font_tab(self): indent_title.pack(side=TOP, anchor=W, padx=5) self.indent_scale.pack(side=TOP, padx=5, fill=X) - return frame - def load_font_cfg(self): """Load current configuration settings for the font options. @@ -1597,8 +1609,8 @@ def var_changed_space_num(self, *params): class GenPage(Frame): - def __init__(self, parent): - super().__init__(parent) + def __init__(self, master): + super().__init__(master) self.create_page_general() self.load_general_cfg() diff --git a/Misc/NEWS.d/next/IDLE/2017-07-31-23-20-51.bpo-31083.991FXm.rst b/Misc/NEWS.d/next/IDLE/2017-07-31-23-20-51.bpo-31083.991FXm.rst new file mode 100644 index 00000000000000..3bb337562de851 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-07-31-23-20-51.bpo-31083.991FXm.rst @@ -0,0 +1,3 @@ +IDLE - Add an outline of a TabPage class in configdialog. +Update existing classes to match outline. +Initial patch by Cheryl Sabella. From f142e85d22ba135d5205280240f3a2fe1df2649f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 2 Aug 2017 16:50:39 +0900 Subject: [PATCH 0621/1345] bpo-31061: fix crash in asyncio speedup module (GH-2984) (cherry picked from commit de34cbe9cdaaf7b85fed86f99c2fd071e1a7b1d2) --- Lib/test/test_asyncio/test_futures.py | 12 ++++++++++++ Lib/test/test_asyncio/test_tasks.py | 15 +++++++++++++++ .../2017-08-01-09-32-58.bpo-31061.husAYX.rst | 1 + Modules/_asynciomodule.c | 4 ++++ 4 files changed, 32 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-08-01-09-32-58.bpo-31061.husAYX.rst diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index ce657fc1b6a058..ebedfec7fa3f5f 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -1,6 +1,7 @@ """Tests for futures.py.""" import concurrent.futures +import gc import re import sys import threading @@ -19,9 +20,11 @@ def _fakefunc(f): return f + def first_cb(): pass + def last_cb(): pass @@ -483,6 +486,15 @@ def test_future_iter_throw(self): Exception("elephant"), Exception("elephant")) self.assertRaises(TypeError, fi.throw, list) + def test_future_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + for i in range(100): + fut = self._new_future(loop=self.loop) + fut.set_result(Evil()) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 195a1ed40770dd..243faf6b049958 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3,6 +3,7 @@ import collections import contextlib import functools +import gc import io import os import re @@ -92,6 +93,20 @@ def setUp(self): self.loop.set_task_factory(self.new_task) self.loop.create_future = lambda: self.new_future(self.loop) + def test_task_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + @asyncio.coroutine + def run(): + return Evil() + + self.loop.run_until_complete( + asyncio.gather(*[ + self.new_task(self.loop, run()) for _ in range(100) + ], loop=self.loop)) + def test_other_loop_future(self): other_loop = asyncio.new_event_loop() fut = self.new_future(other_loop) diff --git a/Misc/NEWS.d/next/Library/2017-08-01-09-32-58.bpo-31061.husAYX.rst b/Misc/NEWS.d/next/Library/2017-08-01-09-32-58.bpo-31061.husAYX.rst new file mode 100644 index 00000000000000..650e5f95f94567 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-01-09-32-58.bpo-31061.husAYX.rst @@ -0,0 +1 @@ +Fixed a crash when using asyncio and threads. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8fbd565dd9c26c..c48cbbc263ddda 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -971,6 +971,8 @@ FutureObj_dealloc(PyObject *self) } } + PyObject_GC_UnTrack(self); + if (fut->fut_weakreflist != NULL) { PyObject_ClearWeakRefs(self); } @@ -1845,6 +1847,8 @@ TaskObj_dealloc(PyObject *self) } } + PyObject_GC_UnTrack(self); + if (task->task_weakreflist != NULL) { PyObject_ClearWeakRefs(self); } From f08b2be4416163e3d18b8d09891e7cda0d8a21d4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 3 Aug 2017 12:14:07 +0300 Subject: [PATCH 0622/1345] [3.6] bpo-30978: str.format_map() now passes key lookup exceptions through. (GH-2790) (#2992) Previously any exception was replaced with a KeyError exception. (cherry picked from commit 5075416) --- Lib/test/test_re.py | 2 +- Lib/test/test_unicode.py | 7 +++++++ .../2017-07-21-07-39-05.bpo-30978.f0jODc.rst | 2 ++ Objects/stringlib/unicode_format.h | 16 ++++++++++------ 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-07-21-07-39-05.bpo-30978.f0jODc.rst diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a6cbbd0b67abf8..e23e5a9a6caf11 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -471,7 +471,7 @@ def test_match_getitem(self): m[(0,)] with self.assertRaisesRegex(IndexError, 'no such group'): m[(0, 1)] - with self.assertRaisesRegex(KeyError, 'a2'): + with self.assertRaisesRegex(IndexError, 'no such group'): 'a1={a2}'.format_map(m) m = pat.match('ac') diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 2844bc5540c38b..56ab0ad7329908 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1279,6 +1279,13 @@ def __format__(self, spec): self.assertRaises(ValueError, '{}'.format_map, 'a') self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1}) + class BadMapping: + def __getitem__(self, key): + return 1/0 + self.assertRaises(KeyError, '{a}'.format_map, {}) + self.assertRaises(TypeError, '{a}'.format_map, []) + self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) + def test_format_huge_precision(self): format_string = ".{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-21-07-39-05.bpo-30978.f0jODc.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-21-07-39-05.bpo-30978.f0jODc.rst new file mode 100644 index 00000000000000..bb67a9fb2442cb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-21-07-39-05.bpo-30978.f0jODc.rst @@ -0,0 +1,2 @@ +str.format_map() now passes key lookup exceptions through. +Previously any exception was replaced with a KeyError exception. diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 14fa28ea54a5c8..7314af02a4a669 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -412,18 +412,22 @@ get_field_object(SubString *input, PyObject *args, PyObject *kwargs, if (index == -1) { /* look up in kwargs */ PyObject *key = SubString_new_object(&first); - if (key == NULL) + if (key == NULL) { goto error; - - /* Use PyObject_GetItem instead of PyDict_GetItem because this - code is no longer just used with kwargs. It might be passed - a non-dict when called through format_map. */ - if ((kwargs == NULL) || (obj = PyObject_GetItem(kwargs, key)) == NULL) { + } + if (kwargs == NULL) { PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); goto error; } + /* Use PyObject_GetItem instead of PyDict_GetItem because this + code is no longer just used with kwargs. It might be passed + a non-dict when called through format_map. */ + obj = PyObject_GetItem(kwargs, key); Py_DECREF(key); + if (obj == NULL) { + goto error; + } } else { /* If args is NULL, we have a format string with a positional field From 946a0b69e217ff22a6c056047eab42053e9a2d5f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 3 Aug 2017 12:14:35 +0300 Subject: [PATCH 0623/1345] [3.6] bpo-31071: Avoid masking original TypeError in call with * unpacking (GH-2957) (#2991) when other arguments are passed. (cherry picked from commit 25e4f77) --- Lib/test/test_extcall.py | 16 +++++ .../2017-07-31-13-28-53.bpo-31071.P9UBDy.rst | 2 + Python/ceval.c | 63 ++++++++++--------- 3 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-07-31-13-28-53.bpo-31071.P9UBDy.rst diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 043df013112517..2c1848337b2d7f 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -163,6 +163,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *(broken() for i in range(1))) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenIterable1: ... def __iter__(self): @@ -172,6 +176,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenIterable1()) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenIterable2: ... def __iter__(self): @@ -182,6 +190,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenIterable2()) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenSequence: ... def __getitem__(self, idx): @@ -191,6 +203,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenSequence()) + Traceback (most recent call last): + ... + TypeError: myerror Make sure that the function doesn't stomp the dictionary diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-31-13-28-53.bpo-31071.P9UBDy.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-31-13-28-53.bpo-31071.P9UBDy.rst new file mode 100644 index 00000000000000..c34d4751bfa9a7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-31-13-28-53.bpo-31071.P9UBDy.rst @@ -0,0 +1,2 @@ +Avoid masking original TypeError in call with * unpacking when other +arguments are passed. diff --git a/Python/ceval.c b/Python/ceval.c index 770dfcba89e513..8eb78bf4d46588 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -64,6 +64,8 @@ static void format_exc_unbound(PyCodeObject *co, int oparg); static PyObject * unicode_concatenate(PyObject *, PyObject *, PyFrameObject *, const _Py_CODEUNIT *); static PyObject * special_lookup(PyObject *, _Py_Identifier *); +static int check_args_iterable(PyObject *func, PyObject *vararg); +static void format_kwargs_mapping_error(PyObject *func, PyObject *kwargs); #define NAME_ERROR_MSG \ "name '%.200s' is not defined" @@ -2578,14 +2580,9 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) none_val = _PyList_Extend((PyListObject *)sum, PEEK(i)); if (none_val == NULL) { if (opcode == BUILD_TUPLE_UNPACK_WITH_CALL && - PyErr_ExceptionMatches(PyExc_TypeError)) { - PyObject *func = PEEK(1 + oparg); - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after * " - "must be an iterable, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - PEEK(i)->ob_type->tp_name); + PyErr_ExceptionMatches(PyExc_TypeError)) + { + check_args_iterable(PEEK(1 + oparg), PEEK(i)); } Py_DECREF(sum); goto error; @@ -2798,12 +2795,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) if (_PyDict_MergeEx(sum, arg, 2) < 0) { PyObject *func = PEEK(2 + oparg); if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - arg->ob_type->tp_name); + format_kwargs_mapping_error(func, arg); } else if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyObject *exc, *val, *tb; @@ -3370,13 +3362,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) * is not a mapping. */ if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - func = SECOND(); - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - kwargs->ob_type->tp_name); + format_kwargs_mapping_error(SECOND(), kwargs); } Py_DECREF(kwargs); goto error; @@ -3389,14 +3375,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) callargs = POP(); func = TOP(); if (!PyTuple_CheckExact(callargs)) { - if (Py_TYPE(callargs)->tp_iter == NULL && - !PySequence_Check(callargs)) { - PyErr_Format(PyExc_TypeError, - "%.200s%.200s argument after * " - "must be an iterable, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - callargs->ob_type->tp_name); + if (check_args_iterable(func, callargs) < 0) { Py_DECREF(callargs); goto error; } @@ -5351,6 +5330,32 @@ import_all_from(PyObject *locals, PyObject *v) return err; } +static int +check_args_iterable(PyObject *func, PyObject *args) +{ + if (args->ob_type->tp_iter == NULL && !PySequence_Check(args)) { + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after * " + "must be an iterable, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + args->ob_type->tp_name); + return -1; + } + return 0; +} + +static void +format_kwargs_mapping_error(PyObject *func, PyObject *kwargs) +{ + PyErr_Format(PyExc_TypeError, + "%.200s%.200s argument after ** " + "must be a mapping, not %.200s", + PyEval_GetFuncName(func), + PyEval_GetFuncDesc(func), + kwargs->ob_type->tp_name); +} + static void format_exc_check_arg(PyObject *exc, const char *format_str, PyObject *obj) { From 9b3abacf87653fab707e83b31afd73ac4140228e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 4 Aug 2017 00:33:33 +1000 Subject: [PATCH 0624/1345] [3.6] Improve grammar in asyncio documentation (GH-2993) (GH-2994) "not only is it .." is the correct form, as opposed to: "not only it is ..." (cherry picked from commit 87c3c5de731af18a271f4559cd69ccb8d050208f) --- Doc/library/asyncio-dev.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index cc4a14b2016226..1838eb95a7d522 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -52,7 +52,7 @@ Cancellation ------------ Cancellation of tasks is not common in classic programming. In asynchronous -programming, not only it is something common, but you have to prepare your +programming, not only is it something common, but you have to prepare your code to handle it. Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel` From a0cb7db8e4b928b4c06da4bde8695555276958f0 Mon Sep 17 00:00:00 2001 From: Yuan Chao Chou Date: Fri, 4 Aug 2017 10:53:42 -0700 Subject: [PATCH 0625/1345] Fix a shadow-compatible-local warning (#2181) Change the shadowing naming, 'value' (Python-ast.c:4686), to 'val' to prevent the variables from being misused. --- Parser/asdl_c.py | 6 +- Python/Python-ast.c | 330 ++++++++++++++++++++++---------------------- 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 725d73bad47b2f..13124bbd3ad8e6 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -522,8 +522,8 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("%s = _Py_asdl_seq_new(len, arena);" % field.name, depth+1) self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) self.emit("for (i = 0; i < len; i++) {", depth+1) - self.emit("%s value;" % ctype, depth+2) - self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % + self.emit("%s val;" % ctype, depth+2) + self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &val, arena);" % field.type, depth+2, reflow=False) self.emit("if (res != 0) goto failed;", depth+2) self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2) @@ -533,7 +533,7 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): depth+3, reflow=False) self.emit("goto failed;", depth+3) self.emit("}", depth+2) - self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) + self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2) self.emit("}", depth+1) else: self.emit("res = obj2ast_%s(tmp, &%s, arena);" % diff --git a/Python/Python-ast.c b/Python/Python-ast.c index e0607ba9ae6d68..b78a0fc714e981 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -3994,14 +3994,14 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Module field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4033,14 +4033,14 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Interactive field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4094,14 +4094,14 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Suite field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4201,14 +4201,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4229,14 +4229,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) decorator_list = _Py_asdl_seq_new(len, arena); if (decorator_list == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"decorator_list\" changed size during iteration"); goto failed; } - asdl_seq_SET(decorator_list, i, value); + asdl_seq_SET(decorator_list, i, val); } Py_CLEAR(tmp); } else { @@ -4305,14 +4305,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4333,14 +4333,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) decorator_list = _Py_asdl_seq_new(len, arena); if (decorator_list == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"decorator_list\" changed size during iteration"); goto failed; } - asdl_seq_SET(decorator_list, i, value); + asdl_seq_SET(decorator_list, i, val); } Py_CLEAR(tmp); } else { @@ -4398,14 +4398,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) bases = _Py_asdl_seq_new(len, arena); if (bases == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"bases\" changed size during iteration"); goto failed; } - asdl_seq_SET(bases, i, value); + asdl_seq_SET(bases, i, val); } Py_CLEAR(tmp); } else { @@ -4426,14 +4426,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) keywords = _Py_asdl_seq_new(len, arena); if (keywords == NULL) goto failed; for (i = 0; i < len; i++) { - keyword_ty value; - res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); + keyword_ty val; + res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"keywords\" changed size during iteration"); goto failed; } - asdl_seq_SET(keywords, i, value); + asdl_seq_SET(keywords, i, val); } Py_CLEAR(tmp); } else { @@ -4454,14 +4454,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4482,14 +4482,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) decorator_list = _Py_asdl_seq_new(len, arena); if (decorator_list == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"decorator_list\" changed size during iteration"); goto failed; } - asdl_seq_SET(decorator_list, i, value); + asdl_seq_SET(decorator_list, i, val); } Py_CLEAR(tmp); } else { @@ -4543,14 +4543,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) targets = _Py_asdl_seq_new(len, arena); if (targets == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Delete field \"targets\" changed size during iteration"); goto failed; } - asdl_seq_SET(targets, i, value); + asdl_seq_SET(targets, i, val); } Py_CLEAR(tmp); } else { @@ -4583,14 +4583,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) targets = _Py_asdl_seq_new(len, arena); if (targets == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Assign field \"targets\" changed size during iteration"); goto failed; } - asdl_seq_SET(targets, i, value); + asdl_seq_SET(targets, i, val); } Py_CLEAR(tmp); } else { @@ -4762,14 +4762,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "For field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4790,14 +4790,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) orelse = _Py_asdl_seq_new(len, arena); if (orelse == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "For field \"orelse\" changed size during iteration"); goto failed; } - asdl_seq_SET(orelse, i, value); + asdl_seq_SET(orelse, i, val); } Py_CLEAR(tmp); } else { @@ -4854,14 +4854,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncFor field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4882,14 +4882,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) orelse = _Py_asdl_seq_new(len, arena); if (orelse == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncFor field \"orelse\" changed size during iteration"); goto failed; } - asdl_seq_SET(orelse, i, value); + asdl_seq_SET(orelse, i, val); } Py_CLEAR(tmp); } else { @@ -4934,14 +4934,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "While field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -4962,14 +4962,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) orelse = _Py_asdl_seq_new(len, arena); if (orelse == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "While field \"orelse\" changed size during iteration"); goto failed; } - asdl_seq_SET(orelse, i, value); + asdl_seq_SET(orelse, i, val); } Py_CLEAR(tmp); } else { @@ -5014,14 +5014,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "If field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -5042,14 +5042,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) orelse = _Py_asdl_seq_new(len, arena); if (orelse == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "If field \"orelse\" changed size during iteration"); goto failed; } - asdl_seq_SET(orelse, i, value); + asdl_seq_SET(orelse, i, val); } Py_CLEAR(tmp); } else { @@ -5082,14 +5082,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) items = _Py_asdl_seq_new(len, arena); if (items == NULL) goto failed; for (i = 0; i < len; i++) { - withitem_ty value; - res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena); + withitem_ty val; + res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "With field \"items\" changed size during iteration"); goto failed; } - asdl_seq_SET(items, i, value); + asdl_seq_SET(items, i, val); } Py_CLEAR(tmp); } else { @@ -5110,14 +5110,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "With field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -5150,14 +5150,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) items = _Py_asdl_seq_new(len, arena); if (items == NULL) goto failed; for (i = 0; i < len; i++) { - withitem_ty value; - res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena); + withitem_ty val; + res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncWith field \"items\" changed size during iteration"); goto failed; } - asdl_seq_SET(items, i, value); + asdl_seq_SET(items, i, val); } Py_CLEAR(tmp); } else { @@ -5178,14 +5178,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "AsyncWith field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -5252,14 +5252,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Try field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -5280,14 +5280,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) handlers = _Py_asdl_seq_new(len, arena); if (handlers == NULL) goto failed; for (i = 0; i < len; i++) { - excepthandler_ty value; - res = obj2ast_excepthandler(PyList_GET_ITEM(tmp, i), &value, arena); + excepthandler_ty val; + res = obj2ast_excepthandler(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Try field \"handlers\" changed size during iteration"); goto failed; } - asdl_seq_SET(handlers, i, value); + asdl_seq_SET(handlers, i, val); } Py_CLEAR(tmp); } else { @@ -5308,14 +5308,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) orelse = _Py_asdl_seq_new(len, arena); if (orelse == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Try field \"orelse\" changed size during iteration"); goto failed; } - asdl_seq_SET(orelse, i, value); + asdl_seq_SET(orelse, i, val); } Py_CLEAR(tmp); } else { @@ -5336,14 +5336,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) finalbody = _Py_asdl_seq_new(len, arena); if (finalbody == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Try field \"finalbody\" changed size during iteration"); goto failed; } - asdl_seq_SET(finalbody, i, value); + asdl_seq_SET(finalbody, i, val); } Py_CLEAR(tmp); } else { @@ -5409,14 +5409,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) names = _Py_asdl_seq_new(len, arena); if (names == NULL) goto failed; for (i = 0; i < len; i++) { - alias_ty value; - res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); + alias_ty val; + res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Import field \"names\" changed size during iteration"); goto failed; } - asdl_seq_SET(names, i, value); + asdl_seq_SET(names, i, val); } Py_CLEAR(tmp); } else { @@ -5460,14 +5460,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) names = _Py_asdl_seq_new(len, arena); if (names == NULL) goto failed; for (i = 0; i < len; i++) { - alias_ty value; - res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); + alias_ty val; + res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ImportFrom field \"names\" changed size during iteration"); goto failed; } - asdl_seq_SET(names, i, value); + asdl_seq_SET(names, i, val); } Py_CLEAR(tmp); } else { @@ -5509,14 +5509,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) names = _Py_asdl_seq_new(len, arena); if (names == NULL) goto failed; for (i = 0; i < len; i++) { - identifier value; - res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); + identifier val; + res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Global field \"names\" changed size during iteration"); goto failed; } - asdl_seq_SET(names, i, value); + asdl_seq_SET(names, i, val); } Py_CLEAR(tmp); } else { @@ -5548,14 +5548,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) names = _Py_asdl_seq_new(len, arena); if (names == NULL) goto failed; for (i = 0; i < len; i++) { - identifier value; - res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); + identifier val; + res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Nonlocal field \"names\" changed size during iteration"); goto failed; } - asdl_seq_SET(names, i, value); + asdl_seq_SET(names, i, val); } Py_CLEAR(tmp); } else { @@ -5693,14 +5693,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) values = _Py_asdl_seq_new(len, arena); if (values == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "BoolOp field \"values\" changed size during iteration"); goto failed; } - asdl_seq_SET(values, i, value); + asdl_seq_SET(values, i, val); } Py_CLEAR(tmp); } else { @@ -5893,14 +5893,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) keys = _Py_asdl_seq_new(len, arena); if (keys == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Dict field \"keys\" changed size during iteration"); goto failed; } - asdl_seq_SET(keys, i, value); + asdl_seq_SET(keys, i, val); } Py_CLEAR(tmp); } else { @@ -5921,14 +5921,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) values = _Py_asdl_seq_new(len, arena); if (values == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Dict field \"values\" changed size during iteration"); goto failed; } - asdl_seq_SET(values, i, value); + asdl_seq_SET(values, i, val); } Py_CLEAR(tmp); } else { @@ -5960,14 +5960,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) elts = _Py_asdl_seq_new(len, arena); if (elts == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Set field \"elts\" changed size during iteration"); goto failed; } - asdl_seq_SET(elts, i, value); + asdl_seq_SET(elts, i, val); } Py_CLEAR(tmp); } else { @@ -6011,14 +6011,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) generators = _Py_asdl_seq_new(len, arena); if (generators == NULL) goto failed; for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + comprehension_ty val; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ListComp field \"generators\" changed size during iteration"); goto failed; } - asdl_seq_SET(generators, i, value); + asdl_seq_SET(generators, i, val); } Py_CLEAR(tmp); } else { @@ -6062,14 +6062,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) generators = _Py_asdl_seq_new(len, arena); if (generators == NULL) goto failed; for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + comprehension_ty val; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "SetComp field \"generators\" changed size during iteration"); goto failed; } - asdl_seq_SET(generators, i, value); + asdl_seq_SET(generators, i, val); } Py_CLEAR(tmp); } else { @@ -6125,14 +6125,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) generators = _Py_asdl_seq_new(len, arena); if (generators == NULL) goto failed; for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + comprehension_ty val; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "DictComp field \"generators\" changed size during iteration"); goto failed; } - asdl_seq_SET(generators, i, value); + asdl_seq_SET(generators, i, val); } Py_CLEAR(tmp); } else { @@ -6176,14 +6176,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) generators = _Py_asdl_seq_new(len, arena); if (generators == NULL) goto failed; for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + comprehension_ty val; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "GeneratorExp field \"generators\" changed size during iteration"); goto failed; } - asdl_seq_SET(generators, i, value); + asdl_seq_SET(generators, i, val); } Py_CLEAR(tmp); } else { @@ -6293,14 +6293,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) ops = _Py_asdl_int_seq_new(len, arena); if (ops == NULL) goto failed; for (i = 0; i < len; i++) { - cmpop_ty value; - res = obj2ast_cmpop(PyList_GET_ITEM(tmp, i), &value, arena); + cmpop_ty val; + res = obj2ast_cmpop(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Compare field \"ops\" changed size during iteration"); goto failed; } - asdl_seq_SET(ops, i, value); + asdl_seq_SET(ops, i, val); } Py_CLEAR(tmp); } else { @@ -6321,14 +6321,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) comparators = _Py_asdl_seq_new(len, arena); if (comparators == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Compare field \"comparators\" changed size during iteration"); goto failed; } - asdl_seq_SET(comparators, i, value); + asdl_seq_SET(comparators, i, val); } Py_CLEAR(tmp); } else { @@ -6373,14 +6373,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) args = _Py_asdl_seq_new(len, arena); if (args == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Call field \"args\" changed size during iteration"); goto failed; } - asdl_seq_SET(args, i, value); + asdl_seq_SET(args, i, val); } Py_CLEAR(tmp); } else { @@ -6401,14 +6401,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) keywords = _Py_asdl_seq_new(len, arena); if (keywords == NULL) goto failed; for (i = 0; i < len; i++) { - keyword_ty value; - res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); + keyword_ty val; + res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Call field \"keywords\" changed size during iteration"); goto failed; } - asdl_seq_SET(keywords, i, value); + asdl_seq_SET(keywords, i, val); } Py_CLEAR(tmp); } else { @@ -6529,14 +6529,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) values = _Py_asdl_seq_new(len, arena); if (values == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "JoinedStr field \"values\" changed size during iteration"); goto failed; } - asdl_seq_SET(values, i, value); + asdl_seq_SET(values, i, val); } Py_CLEAR(tmp); } else { @@ -6805,14 +6805,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) elts = _Py_asdl_seq_new(len, arena); if (elts == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "List field \"elts\" changed size during iteration"); goto failed; } - asdl_seq_SET(elts, i, value); + asdl_seq_SET(elts, i, val); } Py_CLEAR(tmp); } else { @@ -6856,14 +6856,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) elts = _Py_asdl_seq_new(len, arena); if (elts == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "Tuple field \"elts\" changed size during iteration"); goto failed; } - asdl_seq_SET(elts, i, value); + asdl_seq_SET(elts, i, val); } Py_CLEAR(tmp); } else { @@ -7025,14 +7025,14 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) dims = _Py_asdl_seq_new(len, arena); if (dims == NULL) goto failed; for (i = 0; i < len; i++) { - slice_ty value; - res = obj2ast_slice(PyList_GET_ITEM(tmp, i), &value, arena); + slice_ty val; + res = obj2ast_slice(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ExtSlice field \"dims\" changed size during iteration"); goto failed; } - asdl_seq_SET(dims, i, value); + asdl_seq_SET(dims, i, val); } Py_CLEAR(tmp); } else { @@ -7389,14 +7389,14 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) ifs = _Py_asdl_seq_new(len, arena); if (ifs == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "comprehension field \"ifs\" changed size during iteration"); goto failed; } - asdl_seq_SET(ifs, i, value); + asdl_seq_SET(ifs, i, val); } Py_CLEAR(tmp); } else { @@ -7499,14 +7499,14 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) body = _Py_asdl_seq_new(len, arena); if (body == NULL) goto failed; for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + stmt_ty val; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "ExceptHandler field \"body\" changed size during iteration"); goto failed; } - asdl_seq_SET(body, i, value); + asdl_seq_SET(body, i, val); } Py_CLEAR(tmp); } else { @@ -7549,14 +7549,14 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) args = _Py_asdl_seq_new(len, arena); if (args == NULL) goto failed; for (i = 0; i < len; i++) { - arg_ty value; - res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); + arg_ty val; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "arguments field \"args\" changed size during iteration"); goto failed; } - asdl_seq_SET(args, i, value); + asdl_seq_SET(args, i, val); } Py_CLEAR(tmp); } else { @@ -7587,14 +7587,14 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) kwonlyargs = _Py_asdl_seq_new(len, arena); if (kwonlyargs == NULL) goto failed; for (i = 0; i < len; i++) { - arg_ty value; - res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); + arg_ty val; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "arguments field \"kwonlyargs\" changed size during iteration"); goto failed; } - asdl_seq_SET(kwonlyargs, i, value); + asdl_seq_SET(kwonlyargs, i, val); } Py_CLEAR(tmp); } else { @@ -7615,14 +7615,14 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) kw_defaults = _Py_asdl_seq_new(len, arena); if (kw_defaults == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "arguments field \"kw_defaults\" changed size during iteration"); goto failed; } - asdl_seq_SET(kw_defaults, i, value); + asdl_seq_SET(kw_defaults, i, val); } Py_CLEAR(tmp); } else { @@ -7653,14 +7653,14 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) defaults = _Py_asdl_seq_new(len, arena); if (defaults == NULL) goto failed; for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + expr_ty val; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { PyErr_SetString(PyExc_RuntimeError, "arguments field \"defaults\" changed size during iteration"); goto failed; } - asdl_seq_SET(defaults, i, value); + asdl_seq_SET(defaults, i, val); } Py_CLEAR(tmp); } else { From 4795ba857ef2a89e6b477285df961672106a1792 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Sat, 5 Aug 2017 08:01:10 -0700 Subject: [PATCH 0626/1345] [3.6] bpo-31107: Fix copyreg mangled slot names calculation. (GH-2989) (#3003) (cherry picked from commit c4c9866064f03646c686d7e08b00aeb203c35c19) --- Lib/copyreg.py | 6 +++++- Lib/test/test_copyreg.py | 10 ++++++++++ Misc/ACKS | 1 + .../Library/2017-08-02-12-48-15.bpo-31107.1t2hn5.rst | 2 ++ 4 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2017-08-02-12-48-15.bpo-31107.1t2hn5.rst diff --git a/Lib/copyreg.py b/Lib/copyreg.py index 67f5bb029ee8f1..cef9984b1928c9 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -128,7 +128,11 @@ class found there. (This assumes classes don't modify their continue # mangled names elif name.startswith('__') and not name.endswith('__'): - names.append('_%s%s' % (c.__name__, name)) + stripped = c.__name__.lstrip('_') + if stripped: + names.append('_%s%s' % (stripped, name)) + else: + names.append(name) else: names.append(name) diff --git a/Lib/test/test_copyreg.py b/Lib/test/test_copyreg.py index 52e887cb36c41e..e3f1cd81aab205 100644 --- a/Lib/test/test_copyreg.py +++ b/Lib/test/test_copyreg.py @@ -16,6 +16,12 @@ class WithWeakref(object): class WithPrivate(object): __slots__ = ('__spam',) +class _WithLeadingUnderscoreAndPrivate(object): + __slots__ = ('__spam',) + +class ___(object): + __slots__ = ('__spam',) + class WithSingleString(object): __slots__ = 'spam' @@ -104,6 +110,10 @@ def test_slotnames(self): self.assertEqual(copyreg._slotnames(WithWeakref), []) expected = ['_WithPrivate__spam'] self.assertEqual(copyreg._slotnames(WithPrivate), expected) + expected = ['_WithLeadingUnderscoreAndPrivate__spam'] + self.assertEqual(copyreg._slotnames(_WithLeadingUnderscoreAndPrivate), + expected) + self.assertEqual(copyreg._slotnames(___), ['__spam']) self.assertEqual(copyreg._slotnames(WithSingleString), ['spam']) expected = ['eggs', 'spam'] expected.sort() diff --git a/Misc/ACKS b/Misc/ACKS index 7bc13ff0c9e55a..20e174384ba2c3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -592,6 +592,7 @@ David Harrigan Brian Harring Jonathan Hartley Travis B. Hartwell +Shane Harvey Larry Hastings Tim Hatch Shane Hathaway diff --git a/Misc/NEWS.d/next/Library/2017-08-02-12-48-15.bpo-31107.1t2hn5.rst b/Misc/NEWS.d/next/Library/2017-08-02-12-48-15.bpo-31107.1t2hn5.rst new file mode 100644 index 00000000000000..3c2a15528d8018 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-02-12-48-15.bpo-31107.1t2hn5.rst @@ -0,0 +1,2 @@ +Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +name begins with an underscore. Patch by Shane Harvey. From 9cd0ef8f3b93818f0ee5318cb1fc9a1ea32e95b6 Mon Sep 17 00:00:00 2001 From: Rajath Agasthya Date: Sat, 5 Aug 2017 11:28:17 -0700 Subject: [PATCH 0627/1345] bpo-31029: test_tokenize Add missing import unittest (#2998) --- Lib/test/test_tokenize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5a81a5f11a4adb..10e0ad807798e1 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3,6 +3,7 @@ STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, open as tokenize_open, Untokenizer) from io import BytesIO +import unittest from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) From 57e836cb41656ac23042886db6b5985c036a7ce2 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 6 Aug 2017 19:32:39 +0200 Subject: [PATCH 0628/1345] [3.6] bpo-30808: Use _Py_atomic API for concurrency-sensitive signal state (GH-2417) (#3007) * Improve signal delivery Avoid using Py_AddPendingCall from signal handler, to avoid calling signal-unsafe functions. * Remove unused function * Improve comments * Use _Py_atomic API for concurrency-sensitive signal state * Add blurb (cherry picked from commit 2c8a5e4c968217f9672340e520942c4ed788d8de) --- .../2017-07-17-12-12-59.bpo-30808.bA3zOv.rst | 1 + Modules/signalmodule.c | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-07-17-12-12-59.bpo-30808.bA3zOv.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-17-12-12-59.bpo-30808.bA3zOv.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-17-12-12-59.bpo-30808.bA3zOv.rst new file mode 100644 index 00000000000000..8adbbe7dc3a07e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-17-12-12-59.bpo-30808.bA3zOv.rst @@ -0,0 +1 @@ +Use _Py_atomic API for concurrency-sensitive signal state. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 30990f89c3eeaf..0e6099fd361cb5 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -93,7 +93,7 @@ static pid_t main_pid; #endif static volatile struct { - sig_atomic_t tripped; + _Py_atomic_int tripped; PyObject *func; } Handlers[NSIG]; @@ -113,7 +113,7 @@ static volatile sig_atomic_t wakeup_fd = -1; #endif /* Speed up sigcheck() when none tripped */ -static volatile sig_atomic_t is_tripped = 0; +static _Py_atomic_int is_tripped; static PyObject *DefaultHandler; static PyObject *IgnoreHandler; @@ -240,11 +240,13 @@ trip_signal(int sig_num) int fd; Py_ssize_t rc; - Handlers[sig_num].tripped = 1; + _Py_atomic_store_relaxed(&Handlers[sig_num].tripped, 1); /* Set is_tripped after setting .tripped, as it gets cleared in PyErr_CheckSignals() before .tripped. */ - is_tripped = 1; + _Py_atomic_store(&is_tripped, 1); + + /* Notify ceval.c */ _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and @@ -465,7 +467,7 @@ signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) return NULL; } old_handler = Handlers[signalnum].func; - Handlers[signalnum].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[signalnum].tripped, 0); Py_INCREF(handler); Handlers[signalnum].func = handler; if (old_handler != NULL) @@ -1264,11 +1266,11 @@ PyInit__signal(void) goto finally; Py_INCREF(IntHandler); - Handlers[0].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[0].tripped, 0); for (i = 1; i < NSIG; i++) { void (*t)(int); t = PyOS_getsig(i); - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); if (t == SIG_DFL) Handlers[i].func = DefaultHandler; else if (t == SIG_IGN) @@ -1492,7 +1494,7 @@ finisignal(void) for (i = 1; i < NSIG; i++) { func = Handlers[i].func; - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); Handlers[i].func = NULL; if (i != SIGINT && func != NULL && func != Py_None && func != DefaultHandler && func != IgnoreHandler) @@ -1513,7 +1515,7 @@ PyErr_CheckSignals(void) int i; PyObject *f; - if (!is_tripped) + if (!_Py_atomic_load(&is_tripped)) return 0; #ifdef WITH_THREAD @@ -1535,16 +1537,16 @@ PyErr_CheckSignals(void) * we receive a signal i after we zero is_tripped and before we * check Handlers[i].tripped. */ - is_tripped = 0; + _Py_atomic_store(&is_tripped, 0); if (!(f = (PyObject *)PyEval_GetFrame())) f = Py_None; for (i = 1; i < NSIG; i++) { - if (Handlers[i].tripped) { + if (_Py_atomic_load_relaxed(&Handlers[i].tripped)) { PyObject *result = NULL; PyObject *arglist = Py_BuildValue("(iO)", i, f); - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); if (arglist) { result = PyEval_CallObject(Handlers[i].func, @@ -1552,7 +1554,7 @@ PyErr_CheckSignals(void) Py_DECREF(arglist); } if (!result) { - is_tripped = 1; + _Py_atomic_store(&is_tripped, 1); return -1; } @@ -1591,12 +1593,12 @@ PyOS_FiniInterrupts(void) int PyOS_InterruptOccurred(void) { - if (Handlers[SIGINT].tripped) { + if (_Py_atomic_load_relaxed(&Handlers[SIGINT].tripped)) { #ifdef WITH_THREAD if (PyThread_get_thread_ident() != main_thread) return 0; #endif - Handlers[SIGINT].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[SIGINT].tripped, 0); return 1; } return 0; @@ -1606,11 +1608,11 @@ static void _clear_pending_signals(void) { int i; - if (!is_tripped) + if (!_Py_atomic_load(&is_tripped)) return; - is_tripped = 0; + _Py_atomic_store(&is_tripped, 0); for (i = 1; i < NSIG; ++i) { - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); } } From b61de2d46596ac00dcf2d33bbce2023d0bf4207b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 7 Aug 2017 14:23:07 -0400 Subject: [PATCH 0629/1345] [3.6] bpo-29910: IDLE - revert `break`s that disabled calltip close. (GH-2997) (#3017) (cherry picked from commit 8922587) --- Lib/idlelib/parenmatch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/parenmatch.py b/Lib/idlelib/parenmatch.py index c15cb818cd8fbd..7f25880ae5ec1b 100644 --- a/Lib/idlelib/parenmatch.py +++ b/Lib/idlelib/parenmatch.py @@ -104,18 +104,18 @@ def paren_closed_event(self, event): # If user bound non-closer to <>, quit. closer = self.text.get("insert-1c") if closer not in _openers: - return "break" + return hp = HyperParser(self.editwin, "insert-1c") if not hp.is_in_code(): - return "break" + return indices = hp.get_surrounding_brackets(_openers[closer], True) if indices is None: self.bell() - return "break" + return self.activate_restore() self.create_tag(indices) self.set_timeout() - return "break" + return def restore_event(self, event=None): "Remove effect of doing match." From 9d7d928b5853b921ed27f1e535dfe8174169854c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 7 Aug 2017 15:20:03 -0400 Subject: [PATCH 0630/1345] [3.6] bpo-31130: IDLE -- stop leaks in test_configdialog. (GH-3016) (#3018) Initial patch by Victor Stinner. (cherry picked from commit 733d0f6) --- Lib/idlelib/configdialog.py | 1 + Lib/idlelib/idle_test/test_configdialog.py | 86 +++++++++++-------- .../2017-08-07-14-02-56.bpo-31130.FbsC7f.rst | 1 + 3 files changed, 50 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-08-07-14-02-56.bpo-31130.FbsC7f.rst diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 6d8a03b1c00bef..e634d5f359929f 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1856,6 +1856,7 @@ def __init__(self): def clear(self): "Clear lists (for tests)." + # Call after all tests in a module to avoid memory leaks. self.untraced.clear() self.traced.clear() diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index caeb2b5678778d..cd78482150ca6c 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -9,7 +9,7 @@ import unittest from unittest import mock from idlelib.idle_test.mock_idle import Func -from tkinter import Tk, Frame, IntVar, BooleanVar, DISABLED, NORMAL +from tkinter import Tk, Frame, StringVar, IntVar, BooleanVar, DISABLED, NORMAL from idlelib import config from idlelib.configdialog import idleConf, changes, tracers @@ -41,6 +41,8 @@ def tearDownModule(): global root, dialog idleConf.userCfg = usercfg tracers.detach() + tracers.clear() + changes.clear() del dialog root.update_idletasks() root.destroy() @@ -442,16 +444,20 @@ def test_update_help_changes(self): class VarTraceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tracers = configdialog.VarTrace() + cls.iv = IntVar(root) + cls.bv = BooleanVar(root) + + @classmethod + def tearDownClass(cls): + del cls.tracers, cls.iv, cls.bv + def setUp(self): - changes.clear() - tracers.clear() - self.v1 = IntVar(root) - self.v2 = BooleanVar(root) + self.tracers.clear() self.called = 0 - def tearDown(self): - del self.v1, self.v2 - def var_changed_increment(self, *params): self.called += 13 @@ -459,64 +465,68 @@ def var_changed_boolean(self, *params): pass def test_init(self): - tracers.__init__() - self.assertEqual(tracers.untraced, []) - self.assertEqual(tracers.traced, []) + tr = self.tracers + tr.__init__() + self.assertEqual(tr.untraced, []) + self.assertEqual(tr.traced, []) def test_clear(self): - tracers.untraced.append(0) - tracers.traced.append(1) - tracers.clear() - self.assertEqual(tracers.untraced, []) - self.assertEqual(tracers.traced, []) + tr = self.tracers + tr.untraced.append(0) + tr.traced.append(1) + tr.clear() + self.assertEqual(tr.untraced, []) + self.assertEqual(tr.traced, []) def test_add(self): - tr = tracers + tr = self.tracers func = Func() cb = tr.make_callback = mock.Mock(return_value=func) - v1 = tr.add(self.v1, self.var_changed_increment) - self.assertIsInstance(v1, IntVar) - v2 = tr.add(self.v2, self.var_changed_boolean) - self.assertIsInstance(v2, BooleanVar) + iv = tr.add(self.iv, self.var_changed_increment) + self.assertIs(iv, self.iv) + bv = tr.add(self.bv, self.var_changed_boolean) + self.assertIs(bv, self.bv) - v3 = IntVar(root) - v3 = tr.add(v3, ('main', 'section', 'option')) + sv = StringVar(root) + sv2 = tr.add(sv, ('main', 'section', 'option')) + self.assertIs(sv2, sv) cb.assert_called_once() - cb.assert_called_with(v3, ('main', 'section', 'option')) + cb.assert_called_with(sv, ('main', 'section', 'option')) - expected = [(v1, self.var_changed_increment), - (v2, self.var_changed_boolean), - (v3, func)] + expected = [(iv, self.var_changed_increment), + (bv, self.var_changed_boolean), + (sv, func)] self.assertEqual(tr.traced, []) self.assertEqual(tr.untraced, expected) del tr.make_callback def test_make_callback(self): - cb = tracers.make_callback(self.v1, ('main', 'section', 'option')) + cb = self.tracers.make_callback(self.iv, ('main', 'section', 'option')) self.assertTrue(callable(cb)) - self.v1.set(42) + self.iv.set(42) # Not attached, so set didn't invoke the callback. self.assertNotIn('section', changes['main']) # Invoke callback manually. cb() self.assertIn('section', changes['main']) self.assertEqual(changes['main']['section']['option'], '42') + changes.clear() def test_attach_detach(self): - tr = tracers - v1 = tr.add(self.v1, self.var_changed_increment) - v2 = tr.add(self.v2, self.var_changed_boolean) - expected = [(v1, self.var_changed_increment), - (v2, self.var_changed_boolean)] + tr = self.tracers + iv = tr.add(self.iv, self.var_changed_increment) + bv = tr.add(self.bv, self.var_changed_boolean) + expected = [(iv, self.var_changed_increment), + (bv, self.var_changed_boolean)] # Attach callbacks and test call increment. tr.attach() self.assertEqual(tr.untraced, []) self.assertCountEqual(tr.traced, expected) - v1.set(1) - self.assertEqual(v1.get(), 1) + iv.set(1) + self.assertEqual(iv.get(), 1) self.assertEqual(self.called, 13) # Check that only one callback is attached to a variable. @@ -524,7 +534,7 @@ def test_attach_detach(self): # would be called twice and the counter would be 2. self.called = 0 tr.attach() - v1.set(1) + iv.set(1) self.assertEqual(self.called, 13) # Detach callbacks. @@ -532,7 +542,7 @@ def test_attach_detach(self): tr.detach() self.assertEqual(tr.traced, []) self.assertCountEqual(tr.untraced, expected) - v1.set(1) + iv.set(1) self.assertEqual(self.called, 0) diff --git a/Misc/NEWS.d/next/IDLE/2017-08-07-14-02-56.bpo-31130.FbsC7f.rst b/Misc/NEWS.d/next/IDLE/2017-08-07-14-02-56.bpo-31130.FbsC7f.rst new file mode 100644 index 00000000000000..c57728fda22312 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-07-14-02-56.bpo-31130.FbsC7f.rst @@ -0,0 +1 @@ +IDLE -- stop leaks in test_configdialog. Initial patch by Victor Stinner. From e93135dbb265e801c0000a19cd9c5428d57d1cf4 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 8 Aug 2017 14:13:53 +0200 Subject: [PATCH 0631/1345] bpo-31045: Language switch (#2652) (#3023) * Doc: Indicate the language * Renaming version_switcher to switchers (to add language_switcher). * Adding language switch. * Doc switchers: Enhance readability of regex parsing versions. * Doc switchers: Desambiguate the need of a replace(/\/+$/g, '') by proper naming. * Doc switchers: py3k can't reach js, it's redirected server-side by nginx. * Doc switchers: Examples matching actual regexes. * Doc switchers: Better fallback on unexisting translated version. (cherry picked from commit dff9b5f9d62aa0b23f8a255867d09d11890efd1b) --- Doc/Makefile | 4 +- Doc/tools/static/switchers.js | 146 +++++++++++++++++++++++++++++ Doc/tools/static/version_switch.js | 67 ------------- Doc/tools/templates/layout.html | 5 +- 4 files changed, 151 insertions(+), 71 deletions(-) create mode 100644 Doc/tools/static/switchers.js delete mode 100644 Doc/tools/static/version_switch.js diff --git a/Doc/Makefile b/Doc/Makefile index ae59f3294f1c06..526269d1e19d9e 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -167,12 +167,12 @@ serve: # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js new file mode 100644 index 00000000000000..470ff6d33b72b9 --- /dev/null +++ b/Doc/tools/static/switchers.js @@ -0,0 +1,146 @@ +(function() { + 'use strict'; + + // Parses versions in URL segments like: + // "3", "dev", "release/2.7" or "3.6rc2" + var version_regexs = [ + '(?:\\d)', + '(?:\\d\\.\\d[\\w\\d\\.]*)', + '(?:dev)', + '(?:release/\\d.\\d[\\x\\d\\.]*)']; + + var all_versions = { + '3.7': 'dev (3.7)', + '3.6': '3.6', + '3.5': '3.5', + '3.4': '3.4', + '3.3': '3.3', + '2.7': '2.7', + }; + + var all_languages = { + 'en': 'English', + 'fr': 'Français', + }; + + function build_version_select(current_version, current_release) { + var buf = [''); + return buf.join(''); + } + + function build_language_select(current_language) { + var buf = [''); + return buf.join(''); + } + + function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + var url = urls.shift(); + if (urls.length == 0) { + window.location.href = url; + return; + } + $.ajax({ + url: url, + success: function() { + window.location.href = url; + }, + error: function() { + navigate_to_first_existing(urls); + } + }); + } + + function on_version_switch() { + var selected_version = $(this).children('option:selected').attr('value') + '/'; + var url = window.location.href; + var current_language = language_segment_from_url(url); + var current_version = version_segment_in_url(url); + var new_url = url.replace('.org/' + current_language + current_version, + '.org/' + current_language + selected_version); + if (new_url != url) { + navigate_to_first_existing([ + new_url, + url.replace('.org/' + current_language + current_version, + '.org/' + selected_version), + 'https://docs.python.org/' + current_language + selected_version, + 'https://docs.python.org/' + selected_version, + 'https://docs.python.org/' + ]); + } + } + + function on_language_switch() { + var selected_language = $(this).children('option:selected').attr('value') + '/'; + var url = window.location.href; + var current_language = language_segment_from_url(url); + var current_version = version_segment_in_url(url); + if (selected_language == 'en/') // Special 'default' case for english. + selected_language = ''; + var new_url = url.replace('.org/' + current_language + current_version, + '.org/' + selected_language + current_version); + if (new_url != url) { + navigate_to_first_existing([ + new_url, + 'https://docs.python.org/' + ]); + } + } + + // Returns the path segment of the language as a string, like 'fr/' + // or '' if not found. + function language_segment_from_url(url) { + var language_regexp = '\.org/(' + Object.keys(all_languages).join('|') + '/)'; + var match = url.match(language_regexp); + if (match !== null) + return match[1]; + return ''; + } + + // Returns the path segment of the version as a string, like '3.6/' + // or '' if not found. + function version_segment_in_url(url) { + var language_segment = '(?:(?:' + Object.keys(all_languages).join('|') + ')/)'; + var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; + var match = url.match(version_regexp); + if (match !== null) + return match[1]; + return '' + } + + $(document).ready(function() { + var release = DOCUMENTATION_OPTIONS.VERSION; + var language_segment = language_segment_from_url(window.location.href); + var current_language = language_segment.replace(/\/+$/g, '') || 'en'; + var version = release.substr(0, 3); + var version_select = build_version_select(version, release); + + $('.version_switcher_placeholder').html(version_select); + $('.version_switcher_placeholder select').bind('change', on_version_switch); + + var language_select = build_language_select(current_language); + + $('.language_switcher_placeholder').html(language_select); + $('.language_switcher_placeholder select').bind('change', on_language_switch); + }); +})(); diff --git a/Doc/tools/static/version_switch.js b/Doc/tools/static/version_switch.js deleted file mode 100644 index 8b36a61671831d..00000000000000 --- a/Doc/tools/static/version_switch.js +++ /dev/null @@ -1,67 +0,0 @@ -(function() { - 'use strict'; - - var all_versions = { - '3.7': 'dev (3.7)', - '3.6': '3.6', - '3.5': '3.5', - '3.4': '3.4', - '3.3': '3.3', - '2.7': '2.7', - }; - - function build_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function patch_url(url, new_version) { - var url_re = /\.org\/(\d|py3k|dev|((release\/)?\d\.\d[\w\d\.]*))\//, - new_url = url.replace(url_re, '.org/' + new_version + '/'); - - if (new_url == url && !new_url.match(url_re)) { - // python 2 url without version? - new_url = url.replace(/\.org\//, '.org/' + new_version + '/'); - } - return new_url; - } - - function on_switch() { - var selected = $(this).children('option:selected').attr('value'); - - var url = window.location.href, - new_url = patch_url(url, selected); - - if (new_url != url) { - // check beforehand if url exists, else redirect to version's start page - $.ajax({ - url: new_url, - success: function() { - window.location.href = new_url; - }, - error: function() { - window.location.href = 'https://docs.python.org/' + selected; - } - }); - } - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var version = release.substr(0, 3); - var select = build_select(version, release); - - $('.version_switcher_placeholder').html(select); - $('.version_switcher_placeholder select').bind('change', on_switch); - }); -})(); diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 640d8b30493299..c2106678ac60f4 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -4,7 +4,8 @@ style="vertical-align: middle; margin-top: -1px"/>
  • Python{{ reldelim1 }}
  • - {%- if versionswitcher is defined %} + {%- if switchers is defined %} + {{ language or 'en' }} {{ release }} {% trans %}Documentation {% endtrans %}{{ reldelim1 }} {%- else %} @@ -41,7 +42,7 @@ {% if builder != "htmlhelp" %} {% if not embedded %}{% endif %} - {% if versionswitcher is defined and not embedded %}{% endif %} + {% if switchers is defined and not embedded %}{% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %}