diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py deleted file mode 100644 index 08a5f75fe394..000000000000 --- a/utilities/batch_generate_apis.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Instructions: -# -# Check out the googleapis and discovery-artifact-manager repo somewhere locally -# (e.g. from https://github.com/googleapis/googleapis): -# -# $ git clone https://github.com/googleapis/googleapis.git -# $ git clone https://github.com/googleapis/discovery-artifact-manager.git -# -# Run this script from the top-level google-cloud-java directory: -# -# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS PATH_TO_DISCOVERY_ARTIFACT_MANAGER - -import argparse -import os -import sys -from subprocess import check_output - -import generate_api - - -def run_gapic_gen(googleapis): - def generate(artman_yaml, artifact_name=generate_api.JAVA_GAPIC): - generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), - artifact_name) - - generate('google/datastore/artman_datastore.yaml', generate_api.JAVA_PROTO) - - generate('google/cloud/asset/artman_cloudasset_v1beta1.yaml') - generate('google/cloud/automl/artman_automl_v1beta1.yaml') - generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') - generate('google/bigtable/artman_bigtable.yaml') - generate('google/bigtable/admin/artman_bigtableadmin.yaml') - generate('google/container/artman_container.yaml') - generate('google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml') - generate('google/cloud/dataproc/artman_dataproc_v1.yaml') - generate('google/cloud/dataproc/artman_dataproc_v1beta2.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') - generate('google/privacy/dlp/artman_dlp_v2.yaml') - generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') - generate('google/firestore/artman_firestore.yaml') - generate('google/cloud/iot/artman_cloudiot.yaml') - generate('google/cloud/kms/artman_cloudkms.yaml') - generate('google/cloud/language/artman_language_v1.yaml') - generate('google/cloud/language/artman_language_v1beta2.yaml') - generate('google/logging/artman_logging.yaml') - generate('google/monitoring/artman_monitoring.yaml') - generate('google/pubsub/artman_pubsub.yaml') - generate('google/cloud/oslogin/artman_oslogin_v1.yaml') - generate('google/cloud/redis/artman_redis_v1.yaml') - generate('google/cloud/redis/artman_redis_v1beta1.yaml') - generate('google/spanner/artman_spanner.yaml') - generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') - generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') - generate('google/cloud/speech/artman_speech_v1.yaml') - generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') - generate('google/cloud/tasks/artman_cloudtasks_v2beta2.yaml') - generate('google/cloud/tasks/artman_cloudtasks_v2beta3.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1p2beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') - generate('google/cloud/vision/artman_vision_v1.yaml') - generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') - generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') - generate('google/cloud/vision/artman_vision_v1p3beta1.yaml') - generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') - - -def run_discogapic_gen(discovery_repo): - def generate(artman_yaml): - # Run java_discogapic task. No proto or grpc libraries are generated. - generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), - generate_api.JAVA_DISCOGAPIC) - - generate('gapic/google/compute/artman_compute.yaml') - - -def verify_protoc_version(): - protobuf_version_node = check_output( - ['grep', '-zohr', '--include=pom.xml', - '.*']) - version_start_index = protobuf_version_node.find('>') + 1 - version_end_index = protobuf_version_node.rfind('<') - protobuf_version = protobuf_version_node[version_start_index : version_end_index].strip() - - # This will be something like 'libprotoc 3.6.0' - protoc_version_str = check_output(['protoc', '--version']) - - if not (protobuf_version in protoc_version_str): - sys.exit("ERROR: Local version of protoc is %s" - " (see output of `which protoc`)." - " Please use protoc version %s" - " to match the version of protobuf-java used in this repo." - % (protoc_version_str, protobuf_version)) - - -def main(): - verify_protoc_version() - # TODO Make the docker image the default, add --local option - parser = argparse.ArgumentParser(description='Batch generate all APIs.') - parser.add_argument('googleapis', help='The path to the googleapis repo') - parser.add_argument('discovery_repo', - help='The path to the discovery-artifact-manager repo') - args = parser.parse_args() - - generate_api.dump_versions(googleapis=args.googleapis, discovery_repo=args.discovery_repo) - - run_gapic_gen(args.googleapis) - run_discogapic_gen(args.discovery_repo) - -if __name__ == '__main__': - main() diff --git a/utilities/bump_versions.py b/utilities/bump_versions.py deleted file mode 100644 index 774d48b9da1f..000000000000 --- a/utilities/bump_versions.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use case: Bump to the next patch release version in preparation for a release -# -# python utilities/bump_versions.py next_release patch -# -# Use case: Bump to the next minor release version in preparation for a release -# -# python utilities/bump_versions.py next_release minor -# -# Use case: Bump to the next snapshot version in preparation for new development -# -# python utilities/bump_versions.py next_snapshot patch -# -# This script will update versions.txt in place. -# The script must be run at the root of google-cloud-java. - -import argparse -import copy -import re - -version_regex = re.compile(r'(\d+)\.(\d+)\.(\d+)(-\w+)?(-\w+)?') - -class Version: - def __init__(self, version_str): - match = version_regex.match(version_str) - major = int(match.group(1)) - minor = int(match.group(2)) - patch = int(match.group(3)) - qualifier1 = match.group(4) - qualifier2 = match.group(5) - variant = '' - snapshot = False - if qualifier1 and qualifier2: - if qualifier2 == '-SNAPSHOT': - variant = qualifier1 - snapshot = True - else: - variant = qualifier1 + qualifier2 - elif qualifier1: - if qualifier1 == '-SNAPSHOT': - snapshot = True - else: - variant = qualifier1 - self.major = major - self.minor = minor - self.patch = patch - self.variant = variant - self.snapshot = snapshot - - def bump(self, bump_type): - if bump_type == 'minor': - self.bump_minor() - elif bump_type == 'patch': - self.bump_patch() - else: - raise ValueError('invalid bump_type: {}'.format(bump_type)) - - def bump_minor(self): - self.minor += 1 - self.patch = 0 - - def bump_patch(self): - self.patch += 1 - - def set_snapshot(self, snapshot): - self.snapshot = snapshot - - def __str__(self): - mmp = '{}.{}.{}'.format(self.major, self.minor, self.patch) - postfix = self.variant - if self.snapshot: - postfix += '-SNAPSHOT' - return mmp + postfix - -def bump_versions(next_version_type, bump_type): - newlines = [] - with open('versions.txt') as f: - for line in f: - version_line = line.strip() - if not version_line or version_line.startswith('#'): - newlines.append(line) - continue - - (module, released_version_str, current_version_str) = version_line.split(':') - released_version = Version(released_version_str) - - if next_version_type == 'next_release': - released_version.bump(bump_type) - released_version.set_snapshot(False) - current_version = copy.deepcopy(released_version) - elif next_version_type == 'next_snapshot': - current_version = copy.deepcopy(released_version) - current_version.bump(bump_type) - current_version.set_snapshot(True) - else: - raise ValueError('invalid next_version_type: {}'.format(next_version_type)) - - newlines.append('{}:{}:{}\n'.format(module, released_version, current_version)) - - with open('versions.txt', 'w') as f: - for line in newlines: - f.write(line) - -def main(): - parser = argparse.ArgumentParser(description='Add snippets to Javadoc.') - parser.add_argument('next_version_type', help='Either next_snapshot or next_release') - parser.add_argument('bump_type', help='Either minor or patch') - args = parser.parse_args() - - bump_versions(args.next_version_type, args.bump_type) - -if __name__ == '__main__': - main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py deleted file mode 100644 index 870f40ec491d..000000000000 --- a/utilities/generate_api.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Instructions: -# -# Find the artman config file the describes the API you want to generate a client for. -# Specifiy the artman ARTIFACT_NAME to generate, e.g. "java_gapic" -# -# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE ARTIFACT_NAME - -import argparse -import collections -import io -import os -import re -import shutil -import subprocess - -from distutils import dir_util -from ruamel import yaml - - -dir_overrides = { - 'error-reporting': 'google-cloud-errorreporting', - 'spanner-admin-instance': 'google-cloud-spanner', - 'spanner-admin-database': 'google-cloud-spanner' -} - -JAVA_PROTO="java_proto" -JAVA_GRPC="java_grpc" -JAVA_GAPIC="java_gapic" -JAVA_DISCOGAPIC="java_discogapic" - -def get_git_repo_version(path): - commit = subprocess.check_output(['git', '-C', path, 'rev-parse', 'HEAD']).strip() - suffix = '' - - changes = subprocess.check_output(['git', '-C', path, 'diff', '--stat']) - - if changes: - suffix = " ({})".format(changes.splitlines()[-1]) - print([commit,suffix]) - return ''.join([commit.decode('UTF-8'), suffix]) - - -def dump_versions(googleapis=None, discovery_repo=None): - print("Component versions:") - - print(subprocess.check_output(['artman', '--version'], stderr=subprocess.STDOUT).strip()) - - with io.open(os.path.expanduser("~/.artman/config.yaml"), encoding='UTF-8') as config_file: - artman_config_data = yaml.load(config_file, Loader=yaml.Loader) - toolkit_path = artman_config_data['local']['toolkit'] - print("gapic_generator {}".format(get_git_repo_version(toolkit_path))) - - print("google-cloud-java {}".format(get_git_repo_version(os.path.dirname(__file__)))) - - if googleapis: - print("googleapis {}".format(get_git_repo_version(googleapis))) - - if discovery_repo: - print("discovery_repo {}".format(get_git_repo_version(discovery_repo))) - - -def run_generate_api(config_path, artifact_name, noisy=False): - """ Generate an API client library. - - :param config_path: (str) Path to directory containing artman config file. - :param artifact_name: (str) artman target, e.g "java_gapic". - :param noisy: (bool) if console output should be verbose. - - """ - api_repo_index = config_path.rfind('/google/') - if artifact_name == JAVA_DISCOGAPIC: - api_repo_index = config_path.rfind('/gapic/') - if api_repo_index == -1: - raise ValueError('Didn\'t find the API repo in config file path; need absolute path to the artman config file.') - root_dir = config_path[0:api_repo_index] - api_dir = config_path[api_repo_index+1:] - - extra_options = [] - if noisy: - extra_options = ['-v'] - - subprocess.check_call( - ['artman', '--config', api_dir, '--local', '--root-dir', root_dir] - + extra_options + ['generate', artifact_name, '--aspect', 'CODE']) - - with io.open(config_path, encoding='UTF-8') as config_file: - artman_config_data = yaml.load(config_file, Loader=yaml.Loader) - - api_name = artman_config_data['common']['api_name'] - api_version = artman_config_data['common']['api_version'] - org_name = artman_config_data['common']['organization_name'] - - api_full_name = '{}-{}-{}'.format(org_name, api_name, api_version) - proto_dirname = 'proto-{}'.format(api_full_name) - grpc_dirname = 'grpc-{}'.format(api_full_name) - gapic_dirname = 'gapic-{}'.format(api_full_name) - - generating_gapic = artifact_name == JAVA_GAPIC or artifact_name == JAVA_DISCOGAPIC - generating_grpc = generating_gapic or artifact_name == JAVA_GRPC - - gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname) - if generating_gapic and not os.path.exists(gapic_dir): - raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir)) - - if artifact_name != JAVA_DISCOGAPIC: - proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) - grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) - - if not os.path.exists(proto_dir): - raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) - if generating_grpc and not os.path.exists(grpc_dir): - raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir)) - - target_proto_dir = os.path.join('google-api-grpc', proto_dirname) - target_proto_code_dir = os.path.join(target_proto_dir, 'src') - if os.path.exists(target_proto_code_dir): - print('{} already exists, removing & replacing it.'.format(target_proto_code_dir)) - shutil.rmtree(target_proto_code_dir) - dir_util.copy_tree(proto_dir, target_proto_dir) - - if generating_grpc: - target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname) - target_grpc_code_dir = os.path.join(target_grpc_dir, 'src') - if os.path.exists(target_grpc_code_dir): - print('{} already exists, removing & replacing it.'.format(target_grpc_code_dir)) - shutil.rmtree(target_grpc_code_dir) - dir_util.copy_tree(grpc_dir, target_grpc_dir) - - if generating_gapic: - api_unversioned_name = '{}-{}'.format(org_name, api_name) - if api_name in dir_overrides: - api_unversioned_name = dir_overrides[api_name] - - target_gapic_dir = os.path.join('google-cloud-clients', api_unversioned_name) - dir_util.copy_tree(os.path.join(gapic_dir, 'src'), os.path.join(target_gapic_dir, 'src')) - - if noisy: - print('**** REMAINING MANUAL WORK: *****') - print('This script doesn\'t set up new clients. If this is a new client, you need to:') - print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml') - print('2. Add version declarations for the proto and grpc modules in google-api-grpc/pom.xml') - print('3. Copy an existing client pom.xml to the new client directory in google-cloud-clients/google-cloud-[api], update its text') - print('4. Copy an existing client README.md to the same new client directory, update its text') - print('5. Add the new proto, grpc, and client modules into versions.txt') - print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml') - print('7. Add the API to the list in the root README.md') - print('8. Add the client module into the list in google-cloud-clients/pom.xml') - print('9. Add the stub package for the client into the stubs list in google-cloud-clients/pom.xml') - print('10. Run `utilities/replace_versions.py` to update the module versions') - -def main(): - parser = argparse.ArgumentParser(description='Regenerate a single API.') - parser.add_argument('config_file', help='The artman config file for the API') - parser.add_argument('artifact_name', help='The artman artifact type', - default="java_gapic") - parser.add_argument('--quiet', action="store_true", default=False, - help='Don\'t print informational instructions') - args = parser.parse_args() - - if not args.quiet: - dump_versions(googleapis=os.path.dirname(args.config_file)) - - run_generate_api(args.config_file, args.artifact_name, not args.quiet) - -if __name__ == '__main__': - main() diff --git a/utilities/new_client.py b/utilities/new_client.py index 43fb4b776514..a95573ca33d6 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -33,6 +33,8 @@ class Context: artman_config: str = None google_cloud_artifact: str = None google_cloud_version: releasetool.Version = None + google_cloud_parent_version: releasetool.Version = None + grpc_api_parent_version: releasetool.Version = None grpc_artifact: str = None grpc_version: releasetool.Version = None proto_artifact: str = None @@ -41,6 +43,8 @@ class Context: pathlib.Path(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))) description: str = "FIXME" name: str = "FIXME" + product_overview_url: str = "FIXME" + product_docs_url: str = "FIXME" versions: List[str] = None jinja_env: Environment = None @@ -66,20 +70,23 @@ def add_to_versions(ctx: Context) -> None: versions.append(releasetool.ArtifactVersions(version_line)) + ctx.google_cloud_parent_version = next((v for v in versions if v.module == "google-cloud-clients"), None) + ctx.grpc_api_parent_version = next((v for v in versions if v.module == "google-api-grpc"), None) + # Add new versions unless the artifacts already exist in the versions.txt manifest ctx.google_cloud_version = next((v for v in versions if v.module == ctx.google_cloud_artifact), None) if not ctx.google_cloud_version: - ctx.google_cloud_version = releasetool.ArtifactVersions(f"{ctx.google_cloud_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + ctx.google_cloud_version = releasetool.ArtifactVersions(f"{ctx.google_cloud_artifact}:0.0.0:0.0.1-SNAPSHOT") versions.append(ctx.google_cloud_version) ctx.proto_version = next((v for v in versions if v.module == ctx.proto_artifact), None) if not ctx.proto_version: - ctx.proto_version = releasetool.ArtifactVersions(f"{ctx.proto_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + ctx.proto_version = releasetool.ArtifactVersions(f"{ctx.proto_artifact}:0.0.0:0.0.1-SNAPSHOT") versions.append(ctx.proto_version) ctx.grpc_version = next((v for v in versions if v.module == ctx.grpc_artifact), None) if not ctx.grpc_version: - ctx.grpc_version = releasetool.ArtifactVersions(f"{ctx.grpc_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + ctx.grpc_version = releasetool.ArtifactVersions(f"{ctx.grpc_artifact}:0.0.0:0.0.1-SNAPSHOT") versions.append(ctx.grpc_version) # sort by name @@ -163,13 +170,14 @@ def write_synthfile(ctx: Context) -> None: os.makedirs(directory) synth.dump(str(path)) -def write_pom(template: str, path: str, ctx: Context, version: str) -> None: +def write_pom(template: str, path: str, ctx: Context, version: str, parent_version: str) -> None: """Creates a pom.xml file from a template.""" template = ctx.jinja_env.get_template(template) pom = template.stream( api_version=ctx.api_version, description=ctx.description, name=ctx.name, + parent_version=parent_version, service=ctx.service, version=version ) @@ -212,7 +220,9 @@ def write_readme(ctx: Context) -> None: description=ctx.description, name=ctx.name, service=ctx.service, - version=ctx.google_cloud_version + version=ctx.google_cloud_version, + product_overview_url=ctx.product_overview_url, + product_docs_url=ctx.product_docs_url, ) path = ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "README.md" directory = os.path.dirname(path) @@ -223,14 +233,22 @@ def write_readme(ctx: Context) -> None: def main(): parser = argparse.ArgumentParser(description="Create a new client") parser.add_argument("-v", required=True, help="API version (i.e. v1)") - parser.add_argument("-c", required=True, help="Path to config in googleapis/googleapis") - parser.add_argument("-s", required=True, help="Service name") + parser.add_argument("-c", help="Path to config in googleapis/googleapis") + parser.add_argument("-s", required=True, help="Service name (i.e. dialogflow)") + parser.add_argument("-n", required=True, help="Client name (i.e. Google Cloud Pub/Sub)") + parser.add_argument("-d", required=True, help="API description for README") + parser.add_argument("--docs-url", required=True, help="URL to the product docs page") + parser.add_argument("--overview-url", required=True, help="URL to the product overview page") args = parser.parse_args() ctx = Context( api_version=args.v, service=args.s, - artman_config=args.c + artman_config=args.c, + name=args.n, + description=args.d, + product_docs_url=args.docs_url, + product_overview_url=args.overview_url ) add_to_versions(ctx) @@ -240,11 +258,12 @@ def main(): ctx=ctx, template="cloud_pom.xml", path=ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "pom.xml", + parent_version=ctx.google_cloud_parent_version.current, version=ctx.google_cloud_version.current ) add_module_to_pom( pom=ctx.root_directory / "google-cloud-clients/pom.xml", - module_name="google-cloud-iamcredentials" + module_name=ctx.google_cloud_artifact ) add_dependency_management_to_pom( pom=ctx.root_directory / "google-api-grpc/pom.xml", @@ -280,12 +299,14 @@ def main(): ctx=ctx, template="proto_pom.xml", path=ctx.root_directory / "google-api-grpc" / ctx.proto_artifact / "pom.xml", + parent_version=ctx.grpc_api_parent_version.current, version=ctx.proto_version.current ) write_pom( ctx=ctx, template="grpc_pom.xml", path=ctx.root_directory / "google-api-grpc" / ctx.grpc_artifact / "pom.xml", + parent_version=ctx.grpc_api_parent_version.current, version=ctx.grpc_version.current ) add_module_to_pom( diff --git a/utilities/replace_versions.py b/utilities/replace_versions.py deleted file mode 100644 index 5867ed3d8afa..000000000000 --- a/utilities/replace_versions.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use case: Update all the versions in README.md and pom.xml files based on -# the versions in versions.txt -# -# python utilities/replace_versions.py -# -# Use case: Update the versions in a particular file -# -# python utilities/replace_versions.py my-file -# -# The script must be run at the root of google-cloud-java. - -import argparse -import os -import re - -version_update_marker = re.compile(r'\{x-version-update:([^:]+):([^}]+)\}') -version_update_start_marker = re.compile(r'\{x-version-update-start:([^:]+):([^}]+)\}') -version_update_end_marker = re.compile(r'\{x-version-update-end\}') -version_regex_str = r'\d+\.\d+\.\d+(?:-\w+)?(?:-\w+)?' -version_regex = re.compile(version_regex_str) - -class CodeModule: - def __init__(self, module_str): - (name, released, current) = module_str.split(':') - self.name = name - self.released = released - self.current = current.strip() - -def replace_versions(version_map, target): - newlines = [] - repl_open, repl_thisline = False, False - with open(target) as f: - for line in f: - repl_thisline = repl_open - match = version_update_marker.search(line) - if match: - module_name, version_type = match.group(1), match.group(2) - repl_thisline = True - else: - match = version_update_start_marker.search(line) - if match: - module_name, version_type = match.group(1), match.group(2) - repl_open, repl_thisline = True, True - else: - match = version_update_end_marker.search(line) - if match: - repl_open, repl_thisline = False, False - - if repl_thisline: - if module_name not in version_map: - raise ValueError('module not found in version.txt: {}'.format(module_name)) - module = version_map[module_name] - new_version = '' - if version_type == 'current': - new_version = module.current - elif version_type == 'released': - new_version = module.released - else: - raise ValueError('invalid version type: {}'.format(version_type)) - - newline = re.sub(version_regex_str, new_version, line) - newlines.append(newline) - else: - newlines.append(line) - - if not repl_open: - module_name, version_type = '', '' - - with open(target, 'w') as f: - for line in newlines: - f.write(line) - -def replace_versions_all(target): - version_map = {} - with open('versions.txt') as f: - for line in f: - version_line = line.strip() - if not version_line or version_line.startswith('#'): - continue - module = CodeModule(version_line) - version_map[module.name] = module - - if target: - replace_versions(version_map, target) - else: - for root, _, files in os.walk("."): - for file_name in files: - file_path = root + os.sep + file_name - if file_name == 'README.md' or file_name == 'pom.xml': - replace_versions(version_map, file_path) - -def main(): - parser = argparse.ArgumentParser(description='Replace version numbers in poms and READMEs.') - parser.add_argument('target', nargs='?', help='File to update - all files scanned if omitted') - args = parser.parse_args() - - replace_versions_all(args.target) - -if __name__ == '__main__': - main() diff --git a/utilities/requirements.txt b/utilities/requirements.txt index 6ce1600b1b6e..707abc5777ab 100644 --- a/utilities/requirements.txt +++ b/utilities/requirements.txt @@ -1,5 +1,6 @@ attr gcp-releasetool +gcp-synthtool jinja2 lxml~=4.2 typing~=3.6 diff --git a/utilities/templates/README.md.tmpl b/utilities/templates/README.md.tmpl index 8818c0874ff6..45934136fd2e 100644 --- a/utilities/templates/README.md.tmpl +++ b/utilities/templates/README.md.tmpl @@ -43,7 +43,7 @@ See the [Authentication](https://github.com/googleapis/google-cloud-java#authent About {{name}} ---------------------------- -[{{name}}][product-overview] FIXME +[{{name}}][product-overview] {{description}} See the [{{name}} client library docs][lib-docs] to learn how to use this {{name}} Client Library. @@ -95,6 +95,6 @@ Apache 2.0 - See [LICENSE] for more information. [code-of-conduct]:https://github.com/googleapis/google-cloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct [LICENSE]: https://github.com/googleapis/google-cloud-java/blob/master/LICENSE [cloud-platform]: https://cloud.google.com/ -[product-overview]: FIXME -[product-docs]: FIXME +[product-overview]: {{product_overview_url}} +[product-docs]: {{product_docs_url}} [lib-docs]: https://googleapis.github.io/google-cloud-java/google-cloud-clients/apidocs/index.html?com/google/cloud/{{service}}/{{api_version}}/package-summary.html diff --git a/utilities/templates/cloud_pom.xml b/utilities/templates/cloud_pom.xml index 400dd5650913..6d0b0981f571 100644 --- a/utilities/templates/cloud_pom.xml +++ b/utilities/templates/cloud_pom.xml @@ -12,7 +12,7 @@ com.google.cloud google-cloud-clients - {{version}} + {{parent_version}} google-cloud-{{service}} diff --git a/utilities/templates/grpc_pom.xml b/utilities/templates/grpc_pom.xml index 96c1e3ef2a49..6b11e24bdb60 100644 --- a/utilities/templates/grpc_pom.xml +++ b/utilities/templates/grpc_pom.xml @@ -9,7 +9,7 @@ com.google.api.grpc google-api-grpc - {{version}} + {{parent_version}} diff --git a/utilities/templates/proto_pom.xml b/utilities/templates/proto_pom.xml index 7567238d0488..a52cef78df4d 100644 --- a/utilities/templates/proto_pom.xml +++ b/utilities/templates/proto_pom.xml @@ -9,7 +9,7 @@ com.google.api.grpc google-api-grpc - {{version}} + {{parent_version}} diff --git a/utilities/templates/synth.py b/utilities/templates/synth.py index 1c97d7f6fb34..509381238fc8 100644 --- a/utilities/templates/synth.py +++ b/utilities/templates/synth.py @@ -23,12 +23,13 @@ versions = ['{{version}}'] service = '{{service}}' +config_pattern = {% if config_path %}"{{config_path}}"{% else %}'/google/cloud/{{service}}/artman_{{service}}_{version}.yaml'{% endif %} for version in versions: library = gapic.java_library( service=service, version=version, - config_path=f'{{config_path}}', + config_path=config_pattern.format(version=version), artman_output_name='') s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src')