Initial part of abstracting out python specifics from sample generation (#182)

Initial movement of manifest generation to a new file
Lift out python specifics
Move some unrelated code to utility file

Author: software-dov

packages/gapic-generator/gapic/generator/generator.py

@@ -19,9 +19,10 @@
 from typing import (Any, DefaultDict, Dict, Mapping, List)
 from hashlib import sha256
 from collections import (OrderedDict, defaultdict)
-from gapic.samplegen_utils.utils import is_valid_sample_cfg
+from gapic.samplegen_utils.utils import (
+    coerce_response_name, is_valid_sample_cfg)
 from gapic.samplegen_utils.types import InvalidConfig
-from gapic.samplegen import samplegen
+from gapic.samplegen import (manifest, samplegen)
 from gapic.generator import options
 from gapic.generator import formatter
 from gapic.schema import api
@@ -56,7 +57,7 @@ def __init__(self, opts: options.Options) -> None:
         self._env.filters['snake_case'] = utils.to_snake_case
         self._env.filters['sort_lines'] = utils.sort_lines
         self._env.filters['wrap'] = utils.wrap
-        self._env.filters['coerce_response_name'] = samplegen.coerce_response_name
+        self._env.filters['coerce_response_name'] = coerce_response_name
 
         self._sample_configs = opts.sample_configs
 
@@ -76,7 +77,8 @@ def get_response(self, api_schema: api.API) -> CodeGeneratorResponse:
         output_files: Dict[str, CodeGeneratorResponse.File] = OrderedDict()
 
         sample_templates, client_templates = utils.partition(
-            lambda fname: os.path.basename(fname) == samplegen.TEMPLATE_NAME,
+            lambda fname: os.path.basename(
+                fname) == samplegen.DEFAULT_TEMPLATE_NAME,
             self._env.loader.list_templates())
 
         # Iterate over each template and add the appropriate output files
@@ -138,7 +140,6 @@ def _generate_samples_and_manifest(
                 spec["id"] = sample_id
                 id_to_samples[sample_id].append(spec)
 
-        # Interpolate the special variables in the sample_out_dir template.
         out_dir = "samples"
         fpath_to_spec_and_rendered = {}
         for samples in id_to_samples.values():
@@ -167,10 +168,9 @@ def _generate_samples_and_manifest(
 
         # Only generate a manifest if we generated samples.
         if output_files:
-            manifest_fname, manifest_doc = samplegen.generate_manifest(
+            manifest_fname, manifest_doc = manifest.generate(
                 ((fname, spec)
                  for fname, (spec, _) in fpath_to_spec_and_rendered.items()),
-                out_dir,
                 api_schema
             )
 

packages/gapic-generator/gapic/samplegen/__init__.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 from gapic.samplegen import samplegen
+from gapic.samplegen import manifest
 
 __all__ = (
+    'manifest',
     'samplegen',
 )

packages/gapic-generator/gapic/samplegen/manifest.py

@@ -0,0 +1,125 @@
+# Copyright (C) 2019  Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from typing import Tuple
+
+from gapic.samplegen_utils import (types, yaml)
+
+BASE_PATH_KEY = "base_path"
+DEFAULT_SAMPLE_DIR = "samples"
+
+# The default environment for executing python samples.
+# Custom environments must adhere to the following pattern:
+# they must be a yaml.Map with a defined anchor_name field,
+# and 'environment', 'base_path', and 'invocation' keys must be present.
+# The 'invocation' key must map to an interpolable commandline
+# that will invoke the given sample.
+PYTHON3_ENVIRONMENT = yaml.Map(
+    name="python",
+    anchor_name="python",
+    elements=[
+        yaml.KeyVal("environment", "python"),
+        yaml.KeyVal("bin", "python3"),
+        yaml.KeyVal(BASE_PATH_KEY, DEFAULT_SAMPLE_DIR),
+        yaml.KeyVal("invocation", "'{bin} {path} @args'"),
+    ],
+)
+
+
+def generate(
+        fpaths_and_samples,
+        api_schema,
+        *,
+        environment: yaml.Map = PYTHON3_ENVIRONMENT,
+        manifest_time: int = None
+) -> Tuple[str, yaml.Doc]:
+    """Generate a samplegen manifest for use by sampletest
+
+    Args:
+        fpaths_and_samples (Iterable[Tuple[str, Mapping[str, Any]]]):
+                         The file paths and samples to be listed in the manifest
+        api_schema (~.api.API): An API schema object.
+        environment (yaml.Map): Optional custom sample execution environment.
+                                Set this if the samples are being generated for
+                                a custom language.
+        manifest_time (int): Optional. An override for the timestamp in the name of the manifest filename.
+                             Primarily used for testing.
+
+    Returns:
+        Tuple[str, yaml.Doc]: The filename of the manifest and the manifest data as a dictionary.
+
+    Raises:
+        types.InvalidSampleFpath: If any of the paths in fpaths_and_samples do not
+                                  begin with the base_path from the environment.
+
+    """
+    base_path = environment.get(BASE_PATH_KEY, DEFAULT_SAMPLE_DIR)
+
+    def transform_path(fpath):
+        fpath = os.path.normpath(fpath)
+        if not fpath.startswith(base_path):
+            raise types.InvalidSampleFpath(
+                f"Sample fpath does not start with '{base_path}': {fpath}")
+
+        return "'{base_path}/%s'" % os.path.relpath(fpath, base_path)
+
+    doc = yaml.Doc(
+        [
+            yaml.KeyVal("type", "manifest/samples"),
+            yaml.KeyVal("schema_version", "3"),
+            environment,
+            yaml.Collection(
+                name="samples",
+                elements=[
+                    [  # type: ignore
+                        # Mypy doesn't correctly intuit the type of the
+                        # "region_tag" conditional expression.
+                        yaml.Alias(environment.anchor_name or ""),
+                        yaml.KeyVal("sample", sample["id"]),
+                        yaml.KeyVal(
+                            "path", transform_path(fpath)
+                        ),
+                        (yaml.KeyVal("region_tag", sample["region_tag"])
+                         if "region_tag" in sample else
+                         yaml.Null),
+                    ]
+                    for fpath, sample in fpaths_and_samples
+                ],
+            ),
+        ]
+    )
+
+    dt = time.gmtime(manifest_time)
+    manifest_fname_template = (
+        "{api}.{version}.{language}."
+        "{year:04d}{month:02d}{day:02d}."
+        "{hour:02d}{minute:02d}{second:02d}."
+        "manifest.yaml"
+    )
+
+    manifest_fname = manifest_fname_template.format(
+        api=api_schema.naming.name,
+        version=api_schema.naming.version,
+        language=environment.name,
+        year=dt.tm_year,
+        month=dt.tm_mon,
+        day=dt.tm_mday,
+        hour=dt.tm_hour,
+        minute=dt.tm_min,
+        second=dt.tm_sec,
+    )
+
+    return manifest_fname, doc

packages/gapic-generator/gapic/samplegen/samplegen.py

@@ -20,7 +20,7 @@
 import re
 import time
 
-from gapic.samplegen_utils import (types, yaml)
+from gapic.samplegen_utils import types
 from gapic.schema import (api, wrappers)
 
 from collections import (defaultdict, namedtuple, ChainMap as chainmap)
@@ -55,9 +55,7 @@
     )
 )
 
-# TODO: configure the base template name so that
-#       e.g. other languages can use the same machinery.
-TEMPLATE_NAME = "sample.py.j2"
+DEFAULT_TEMPLATE_NAME = "sample.py.j2"
 
 
 @dataclasses.dataclass(frozen=True)
@@ -106,20 +104,6 @@ class TransformedRequest:
     body: Optional[List[AttributeRequestSetup]]
 
 
-def coerce_response_name(s: str) -> str:
-    # In the sample config, the "$resp" keyword is used to refer to the
-    # item of interest as received by the corresponding calling form.
-    # For a 'regular', i.e. unary, synchronous, non-long-running method,
-    # it's the return value; for a server-streaming method, it's the iteration
-    # variable in the for loop that iterates over the return value, and for
-    # a long running promise, the user calls result on the method return value to
-    # resolve the future.
-    #
-    # The sample schema uses '$resp' as the special variable,
-    # but in the samples the 'response' variable is used instead.
-    return s.replace("$resp", "response")
-
-
 class Validator:
     """Class that validates a sample.
 
@@ -160,6 +144,17 @@ def __init__(self, method: wrappers.Method):
             }
         )
 
+    @staticmethod
+    def preprocess_sample(sample, api_schema):
+        """Modify a sample to set default or missing fields.
+
+        Args:
+           sample (Any): A definition for a single sample generated from parsed yaml.
+           api_schema (api.API): The schema that defines the API to which the sample belongs.
+        """
+        sample["package_name"] = api_schema.naming.warehouse_package_name
+        sample.setdefault("response", [{"print": ["%s", "$resp"]}])
+
     def var_field(self, var_name: str) -> Optional[wrappers.Field]:
         return self.var_defs_.get(var_name)
 
@@ -661,7 +656,8 @@ def _validate_loop(self, loop):
 
 def generate_sample(sample,
                     env: jinja2.environment.Environment,
-                    api_schema: api.API) -> str:
+                    api_schema: api.API,
+                    template_name: str = DEFAULT_TEMPLATE_NAME) -> str:
     """Generate a standalone, runnable sample.
 
     Rendering and writing the rendered output is left for the caller.
@@ -671,11 +667,13 @@ def generate_sample(sample,
         env (jinja2.environment.Environment): The jinja environment used to generate
                                               the filled template for the sample.
         api_schema (api.API): The schema that defines the API to which the sample belongs.
+        template_name (str): An optional override for the name of the template
+                             used to generate the sample.
 
     Returns:
         str: The rendered sample.
     """
-    sample_template = env.get_template(TEMPLATE_NAME)
+    sample_template = env.get_template(template_name)
 
     service_name = sample["service"]
     service = api_schema.services.get(service_name)
@@ -693,99 +691,17 @@ def generate_sample(sample,
     calling_form = types.CallingForm.method_default(rpc)
 
     v = Validator(rpc)
+    # Tweak some small aspects of the sample to set sane defaults for optional
+    # fields, add fields that are required for the template, and so forth.
+    v.preprocess_sample(sample, api_schema)
     sample["request"] = v.validate_and_transform_request(calling_form,
                                                          sample["request"])
     v.validate_response(sample["response"])
 
-    sample["package_name"] = api_schema.naming.warehouse_package_name
-
     return sample_template.render(
         file_header=FILE_HEADER,
         sample=sample,
         imports=[],
         calling_form=calling_form,
         calling_form_enum=types.CallingForm,
     )
-
-
-def generate_manifest(
-        fpaths_and_samples,
-        base_path: str,
-        api_schema,
-        *,
-        manifest_time: int = None
-) -> Tuple[str, yaml.Doc]:
-    """Generate a samplegen manifest for use by sampletest
-
-    Args:
-        fpaths_and_samples (Iterable[Tuple[str, Mapping[str, Any]]]):
-                         The file paths and samples to be listed in the manifest
-        base_path (str): The base directory where the samples are generated.
-        api_schema (~.api.API): An API schema object.
-        manifest_time (int): Optional. An override for the timestamp in the name of the manifest filename.
-                             Primarily used for testing.
-
-    Returns:
-        Tuple[str, yaml.Doc]: The filename of the manifest and the manifest data as a dictionary.
-
-    """
-
-    doc = yaml.Doc(
-        [
-            yaml.KeyVal("type", "manifest/samples"),
-            yaml.KeyVal("schema_version", "3"),
-            # TODO: make the environment configurable to allow other languages
-            #       to use the same basic machinery.
-            yaml.Map(
-                name="python",
-                anchor_name="python",
-                elements=[
-                    yaml.KeyVal("environment", "python"),
-                    yaml.KeyVal("bin", "python3"),
-                    yaml.KeyVal("base_path", base_path),
-                    yaml.KeyVal("invocation", "'{bin} {path} @args'"),
-                ],
-            ),
-            yaml.Collection(
-                name="samples",
-                elements=[
-                    [   # type: ignore
-                        # Mypy doesn't correctly intuit the type of the
-                        # "region_tag" conditional expression.
-                        yaml.Alias("python"),
-                        yaml.KeyVal("sample", sample["id"]),
-                        yaml.KeyVal("path",
-                                    "'{base_path}/%s'" % os.path.relpath(fpath,
-                                                                         base_path)),
-                        (yaml.KeyVal("region_tag", sample["region_tag"])
-                         if "region_tag" in sample else
-                         yaml.Null),
-
-                    ]
-                    for fpath, sample in fpaths_and_samples
-                ],
-            ),
-        ]
-    )
-
-    dt = time.gmtime(manifest_time)
-    # TODO: allow other language configuration
-    manifest_fname_template = (
-        "{api}.{version}.python."
-        "{year:04d}{month:02d}{day:02d}."
-        "{hour:02d}{minute:02d}{second:02d}."
-        "manifest.yaml"
-    )
-
-    manifest_fname = manifest_fname_template.format(
-        api=api_schema.naming.name,
-        version=api_schema.naming.version,
-        year=dt.tm_year,
-        month=dt.tm_mon,
-        day=dt.tm_mday,
-        hour=dt.tm_hour,
-        minute=dt.tm_min,
-        second=dt.tm_sec,
-    )
-
-    return manifest_fname, doc

packages/gapic-generator/gapic/samplegen_utils/types.py

@@ -76,6 +76,10 @@ class InvalidEnumVariant(SampleError):
     pass
 
 
+class InvalidSampleFpath(SampleError):
+    pass
+
+
 class CallingForm(Enum):
     Request = auto()
     RequestPaged = auto()