[2.7] bpo-24960: use pkgutil.get_data in lib2to3 to read pickled grammar files (GH-4977) (#4980)

This is more complicated than it should be because we need to preserve the
useful mtime-based regeneration feature that lib2to3.pgen2.driver.load_grammar
has. We only look for the pickled grammar file with pkgutil.get_data and only if
the source file does not exist..
(cherry picked from commit 8a58771)

Author: benjaminp

Lib/lib2to3/pgen2/driver.py

@@ -19,6 +19,7 @@
 import codecs
 import os
 import logging
+import pkgutil
 import StringIO
 import sys
 
@@ -143,6 +144,26 @@ def _newer(a, b):
     return os.path.getmtime(a) >= os.path.getmtime(b)
 
 
+def load_packaged_grammar(package, grammar_source):
+    """Normally, loads a pickled grammar by doing
+        pkgutil.get_data(package, pickled_grammar)
+    where *pickled_grammar* is computed from *grammar_source* by adding the
+    Python version and using a ``.pickle`` extension.
+
+    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
+    is called instead. This facilities using a packaged grammar file when needed
+    but preserves load_grammar's automatic regeneration behavior when possible.
+
+    """
+    if os.path.isfile(grammar_source):
+        return load_grammar(grammar_source)
+    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
+    data = pkgutil.get_data(package, pickled_name)
+    g = grammar.Grammar()
+    g.loads(data)
+    return g
+
+
 def main(*args):
     """Main program, when run as a script: produce grammar pickle files.
 

Lib/lib2to3/pgen2/grammar.py

@@ -109,6 +109,10 @@ def load(self, filename):
         f.close()
         self.__dict__.update(d)
 
+    def loads(self, pkl):
+        """Load the grammar tables from a pickle bytes object."""
+        self.__dict__.update(pickle.loads(pkl))
+
     def copy(self):
         """
         Copy the grammar.

Lib/lib2to3/pygram.py

@@ -29,12 +29,12 @@ def __init__(self, grammar):
             setattr(self, name, symbol)
 
 
-python_grammar = driver.load_grammar(_GRAMMAR_FILE)
+python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE)
 
 python_symbols = Symbols(python_grammar)
 
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 
-pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)
+pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)

Lib/lib2to3/tests/test_parser.py

@@ -11,11 +11,14 @@
 from .support import driver, test_dir
 
 # Python imports
+import operator
 import os
+import pickle
 import shutil
 import subprocess
 import sys
 import tempfile
+import types
 import unittest
 
 # Local imports
@@ -97,6 +100,18 @@ def test_load_grammar_from_subprocess(self):
         finally:
             shutil.rmtree(tmpdir)
 
+    def test_load_packaged_grammar(self):
+        modname = __name__ + '.load_test'
+        class MyLoader:
+            def get_data(self, where):
+                return pickle.dumps({'elephant': 19})
+        class MyModule(types.ModuleType):
+            __file__ = 'parsertestmodule'
+            __loader__ = MyLoader()
+        sys.modules[modname] = MyModule(modname)
+        self.addCleanup(operator.delitem, sys.modules, modname)
+        g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
+        self.assertEqual(g.elephant, 19)
 
 
 class GrammarTest(support.TestCase):

Misc/NEWS.d/next/Tools-Demos/2017-12-22-09-25-51.bpo-24960.TGdAgO.rst

@@ -0,0 +1,3 @@
+2to3 and lib2to3 can now read pickled grammar files using pkgutil.get_data()
+rather than probing the filesystem. This lets 2to3 and lib2to3 work when run
+from a zipfile.