diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index a7ca2c031089..32ebf9bbd140 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -14,14 +14,15 @@ # Instructions: # -# Check out the googleapis repo somewhere locally (e.g. from -# https://github.com/googleapis/googleapis): +# Check out the googleapis and discovery-artifact-manager repo somewhere locally +# (e.g. from https://github.com/googleapis/googleapis): # # $ git checkout https://github.com/googleapis/googleapis.git +# $ git checkout https://github.com/googleapis/discovery-artifact-manager.git # # Run this script: # -# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS +# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS PATH_TO_DISCOVERY_ARTIFACT_MANAGER import argparse import os @@ -29,9 +30,10 @@ import generate_api -def run(googleapis): +def run_gapic_gen(googleapis): def generate(artman_yaml): - generate_api.run_generate_api(os.path.join(googleapis, artman_yaml)) + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), + generate_api.JAVA_GAPIC) # TODO Needs to have java_proto called instead of java_grpc #generate('google/datastore/artman_datastore.yaml') @@ -73,13 +75,25 @@ def generate(artman_yaml): generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') +def run_discogapic_gen(discovery_repo): + def generate(artman_yaml): + # Run java_discogapic task. No proto or grpc libraries are generated. + generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), + generate_api.JAVA_DISCOGAPIC) + + generate('gapic/google/compute/artman_compute.yaml') + + def main(): # TODO Make the docker image the default, add --local option parser = argparse.ArgumentParser(description='Batch generate all APIs.') parser.add_argument('googleapis', help='The path to the googleapis repo') + parser.add_argument('discovery_repo', + help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - run(args.googleapis) + run_gapic_gen(args.googleapis) + run_discogapic_gen(args.discovery_repo) if __name__ == '__main__': main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py index ff0f810c058b..e60027f38855 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -15,8 +15,9 @@ # Instructions: # # Find the artman config file the describes the API you want to generate a client for. +# Specifiy the artman ARTIFACT_TYPE to generate, e.g. "java_gapic" # -# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE +# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE ARTIFACT_TYPE import argparse import io @@ -35,19 +36,32 @@ 'spanner-admin-database': 'google-cloud-spanner' } +JAVA_GAPIC="java_gapic" +JAVA_DISCOGAPIC="java_discogapic" -def run_generate_api(config_path, noisy=False): - googleapis_index = config_path.rfind('/google/') - if googleapis_index == -1: - raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') - root_dir = config_path[0:googleapis_index] - api_dir = config_path[googleapis_index+1:] +def run_generate_api(config_path, artifact_type, noisy=False): + """ Generate an API client library. + + :param config_path: (str) Path to directory containing artman config file. + :param artifact_type: (str) artman target, e.g "java_gapic". + :param noisy: (bool) if console output should be verbose. + + """ + api_repo_index = config_path.rfind('/google/') + if artifact_type == JAVA_DISCOGAPIC: + api_repo_index = config_path.rfind('/gapic/') + if api_repo_index == -1: + raise ValueError('Didn\'t find the API repo in config file path; need absolute path to the artman config file.') + root_dir = config_path[0:api_repo_index] + api_dir = config_path[api_repo_index+1:] extra_options = [] if noisy: extra_options = ['-v'] - subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic']) + subprocess.check_call( + ['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + + extra_options + ['generate', artifact_type]) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader) @@ -60,38 +74,42 @@ def run_generate_api(config_path, noisy=False): proto_dirname = 'proto-{}'.format(api_full_name) grpc_dirname = 'grpc-{}'.format(api_full_name) gapic_dirname = 'gapic-{}'.format(api_full_name) - proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) - grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) + gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname) - if not os.path.exists(proto_dir): - raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) - if not os.path.exists(grpc_dir): - raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir)) if not os.path.exists(gapic_dir): raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir)) - target_proto_dir = os.path.join('google-api-grpc', proto_dirname) - target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname) - if os.path.exists(target_proto_dir): - print('{} already exists, removing & replacing it.'.format(target_proto_dir)) - if os.path.exists(target_grpc_dir): - print('{} already exists, removing & replacing it.'.format(target_grpc_dir)) - - print('-- ignore any pathspec errors that follow') - - if os.path.exists(target_proto_dir): - shutil.rmtree(target_proto_dir) - shutil.copytree(proto_dir, target_proto_dir) - os.remove(os.path.join(target_proto_dir, 'LICENSE')) - os.remove(os.path.join(target_proto_dir, 'build.gradle')) - subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')]) - - if os.path.exists(target_grpc_dir): - shutil.rmtree(target_grpc_dir) - shutil.copytree(grpc_dir, target_grpc_dir) - os.remove(os.path.join(target_grpc_dir, 'LICENSE')) - os.remove(os.path.join(target_grpc_dir, 'build.gradle')) - subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) + if artifact_type != JAVA_DISCOGAPIC: + proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) + grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) + + if not os.path.exists(proto_dir): + raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) + if not os.path.exists(grpc_dir): + raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir)) + + target_proto_dir = os.path.join('google-api-grpc', proto_dirname) + target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname) + if os.path.exists(target_proto_dir): + print('{} already exists, removing & replacing it.'.format(target_proto_dir)) + if os.path.exists(target_grpc_dir): + print('{} already exists, removing & replacing it.'.format(target_grpc_dir)) + + print('-- ignore any pathspec errors that follow') + + if os.path.exists(target_proto_dir): + shutil.rmtree(target_proto_dir) + shutil.copytree(proto_dir, target_proto_dir) + os.remove(os.path.join(target_proto_dir, 'LICENSE')) + os.remove(os.path.join(target_proto_dir, 'build.gradle')) + subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')]) + + if os.path.exists(target_grpc_dir): + shutil.rmtree(target_grpc_dir) + shutil.copytree(grpc_dir, target_grpc_dir) + os.remove(os.path.join(target_grpc_dir, 'LICENSE')) + os.remove(os.path.join(target_grpc_dir, 'build.gradle')) + subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) api_unversioned_name = '{}-{}'.format(org_name, api_name) if api_name in dir_overrides: @@ -113,11 +131,13 @@ def run_generate_api(config_path, noisy=False): def main(): parser = argparse.ArgumentParser(description='Regenerate a single API.') parser.add_argument('config_file', help='The artman config file for the API') + parser.add_argument('artifact_type', help='The artman artifact type', + default="java_gapic") parser.add_argument('--quiet', action="store_true", default=False, help='Don\'t print informational instructions') args = parser.parse_args() - run_generate_api(args.config_file, not args.quiet) + run_generate_api(args.config_file, args.artifact_type, not args.quiet) if __name__ == '__main__': main()