From 021046b66c473e2bd722127c8d53d0259bfbd963 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 17:14:39 -0700 Subject: [PATCH 01/12] first draft --- utilities/batch_generate_apis.py | 91 ++++++++++++++++++-------------- utilities/generate_api.py | 13 +++-- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index a7ca2c031089..a07a8f5bd483 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -14,14 +14,15 @@ # Instructions: # -# Check out the googleapis repo somewhere locally (e.g. from -# https://github.com/googleapis/googleapis): +# Check out the googleapis and discovery-artifact-manager repo somewhere locally +# (e.g. from https://github.com/googleapis/googleapis): # # $ git checkout https://github.com/googleapis/googleapis.git +# $ git checkout https://github.com/googleapis/discovery-artifact-manager # # Run this script: # -# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS +# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS PATH_TO_DISCOVERY_ARTIFACT_MANAGER import argparse import os @@ -29,57 +30,67 @@ import generate_api -def run(googleapis): +def run_gapic_gen(googleapis): def generate(artman_yaml): - generate_api.run_generate_api(os.path.join(googleapis, artman_yaml)) + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), "java_gapic") # TODO Needs to have java_proto called instead of java_grpc #generate('google/datastore/artman_datastore.yaml') generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') - generate('google/bigtable/artman_bigtable.yaml') - generate('google/bigtable/admin/artman_bigtableadmin.yaml') - generate('google/container/artman_container.yaml') - generate('google/cloud/dataproc/artman_dataproc_v1.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') - generate('google/privacy/dlp/artman_dlp_v2.yaml') - generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') - generate('google/firestore/artman_firestore.yaml') - generate('google/cloud/language/artman_language_v1.yaml') - generate('google/cloud/language/artman_language_v1beta2.yaml') - generate('google/logging/artman_logging.yaml') - generate('google/monitoring/artman_monitoring.yaml') - generate('google/pubsub/artman_pubsub.yaml') - generate('google/cloud/oslogin/artman_oslogin_v1.yaml') - generate('google/cloud/redis/artman_redis_v1beta1.yaml') - generate('google/spanner/artman_spanner.yaml') - generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') - generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') - generate('google/cloud/speech/artman_speech_v1.yaml') - generate('google/cloud/speech/artman_speech_v1beta1.yaml') - generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') - generate('google/cloud/vision/artman_vision_v1.yaml') - generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') - generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') - generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + # generate('google/bigtable/artman_bigtable.yaml') + # generate('google/bigtable/admin/artman_bigtableadmin.yaml') + # generate('google/container/artman_container.yaml') + # generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + # generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + # generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + # generate('google/privacy/dlp/artman_dlp_v2.yaml') + # generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + # generate('google/firestore/artman_firestore.yaml') + # generate('google/cloud/language/artman_language_v1.yaml') + # generate('google/cloud/language/artman_language_v1beta2.yaml') + # generate('google/logging/artman_logging.yaml') + # generate('google/monitoring/artman_monitoring.yaml') + # generate('google/pubsub/artman_pubsub.yaml') + # generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + # generate('google/cloud/redis/artman_redis_v1beta1.yaml') + # generate('google/spanner/artman_spanner.yaml') + # generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + # generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + # generate('google/cloud/speech/artman_speech_v1.yaml') + # generate('google/cloud/speech/artman_speech_v1beta1.yaml') + # generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + # generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + # generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + # generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + # generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + # generate('google/cloud/vision/artman_vision_v1.yaml') + # generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + # generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + # generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + + +def run_discogapic_gen(discovery_repo): + def generate(artman_yaml): + generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), "java_discogapic") + + generate('gapic/google/compute/artman_compute.yaml') def main(): # TODO Make the docker image the default, add --local option parser = argparse.ArgumentParser(description='Batch generate all APIs.') parser.add_argument('googleapis', help='The path to the googleapis repo') + parser.add_argument('discovery', help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - run(args.googleapis) + run_gapic_gen(args.discovery) + run_discogapic_gen(args.discovery) + if __name__ == '__main__': main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py index ff0f810c058b..7b8c6adff2f0 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -35,8 +35,13 @@ 'spanner-admin-database': 'google-cloud-spanner' } +def run_generate_api(config_path, artifact_type, noisy=False): + """ Generate an API client library. -def run_generate_api(config_path, noisy=False): + :param config_path (str): Path to directory containing artman config file. + :param artifact_type (str): artman target, e.g "java_gapic". + :param noisy (bool): if console output should be verbose. + """ googleapis_index = config_path.rfind('/google/') if googleapis_index == -1: raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') @@ -47,7 +52,7 @@ def run_generate_api(config_path, noisy=False): if noisy: extra_options = ['-v'] - subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic']) + subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', artifact_type]) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader) @@ -113,11 +118,13 @@ def run_generate_api(config_path, noisy=False): def main(): parser = argparse.ArgumentParser(description='Regenerate a single API.') parser.add_argument('config_file', help='The artman config file for the API') + parser.add_argument('artifact_type', help='The artman artifact type', + default="java_gapic") parser.add_argument('--quiet', action="store_true", default=False, help='Don\'t print informational instructions') args = parser.parse_args() - run_generate_api(args.config_file, not args.quiet) + run_generate_api(args.config_file, args.artifact_type, not args.quiet) if __name__ == '__main__': main() From f55a65aa0f1590954c35a65319903fa0cc9ff0a7 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 17:18:31 -0700 Subject: [PATCH 02/12] bug --- utilities/batch_generate_apis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index a07a8f5bd483..afffc9680aa9 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -88,7 +88,7 @@ def main(): parser.add_argument('discovery', help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - run_gapic_gen(args.discovery) + # run_gapic_gen(args.googleapis) run_discogapic_gen(args.discovery) From 1fb9d5644262484b7f1a00fe92dcfdd13e602cbb Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 17:27:36 -0700 Subject: [PATCH 03/12] don't worry about proto or grpc libraries for discogapi --- utilities/batch_generate_apis.py | 6 ++- utilities/generate_api.py | 73 +++++++++++++++++--------------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index afffc9680aa9..b27697107dbb 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -76,8 +76,12 @@ def generate(artman_yaml): def run_discogapic_gen(discovery_repo): def generate(artman_yaml): - generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), "java_discogapic") + # Run java_discogapic task. No proto or grpc libraries are generated. + generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), + "java_discogapic", + replace_proto_libs=False) + os.chdir(discovery_repo) generate('gapic/google/compute/artman_compute.yaml') diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 7b8c6adff2f0..36c1e3fc5b98 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -16,7 +16,7 @@ # # Find the artman config file the describes the API you want to generate a client for. # -# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE +# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE java_gapic import argparse import io @@ -35,12 +35,15 @@ 'spanner-admin-database': 'google-cloud-spanner' } -def run_generate_api(config_path, artifact_type, noisy=False): + +def run_generate_api(config_path, artifact_type, noisy=False, replace_proto_libs=True): """ Generate an API client library. - :param config_path (str): Path to directory containing artman config file. - :param artifact_type (str): artman target, e.g "java_gapic". - :param noisy (bool): if console output should be verbose. + :param config_path: (str) Path to directory containing artman config file. + :param artifact_type: (str) artman target, e.g "java_gapic". + :param noisy: (bool) if console output should be verbose. + :param replace_proto_libs: (bool) if the generated proto and grpc libraries + should replace the existing ones """ googleapis_index = config_path.rfind('/google/') if googleapis_index == -1: @@ -65,38 +68,42 @@ def run_generate_api(config_path, artifact_type, noisy=False): proto_dirname = 'proto-{}'.format(api_full_name) grpc_dirname = 'grpc-{}'.format(api_full_name) gapic_dirname = 'gapic-{}'.format(api_full_name) - proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) - grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) + gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname) - if not os.path.exists(proto_dir): - raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) - if not os.path.exists(grpc_dir): - raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir)) if not os.path.exists(gapic_dir): raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir)) - target_proto_dir = os.path.join('google-api-grpc', proto_dirname) - target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname) - if os.path.exists(target_proto_dir): - print('{} already exists, removing & replacing it.'.format(target_proto_dir)) - if os.path.exists(target_grpc_dir): - print('{} already exists, removing & replacing it.'.format(target_grpc_dir)) - - print('-- ignore any pathspec errors that follow') - - if os.path.exists(target_proto_dir): - shutil.rmtree(target_proto_dir) - shutil.copytree(proto_dir, target_proto_dir) - os.remove(os.path.join(target_proto_dir, 'LICENSE')) - os.remove(os.path.join(target_proto_dir, 'build.gradle')) - subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')]) - - if os.path.exists(target_grpc_dir): - shutil.rmtree(target_grpc_dir) - shutil.copytree(grpc_dir, target_grpc_dir) - os.remove(os.path.join(target_grpc_dir, 'LICENSE')) - os.remove(os.path.join(target_grpc_dir, 'build.gradle')) - subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) + if replace_proto_libs: + proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) + grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) + + if not os.path.exists(proto_dir): + raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) + if not os.path.exists(grpc_dir): + raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir)) + + target_proto_dir = os.path.join('google-api-grpc', proto_dirname) + target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname) + if os.path.exists(target_proto_dir): + print('{} already exists, removing & replacing it.'.format(target_proto_dir)) + if os.path.exists(target_grpc_dir): + print('{} already exists, removing & replacing it.'.format(target_grpc_dir)) + + print('-- ignore any pathspec errors that follow') + + if os.path.exists(target_proto_dir): + shutil.rmtree(target_proto_dir) + shutil.copytree(proto_dir, target_proto_dir) + os.remove(os.path.join(target_proto_dir, 'LICENSE')) + os.remove(os.path.join(target_proto_dir, 'build.gradle')) + subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')]) + + if os.path.exists(target_grpc_dir): + shutil.rmtree(target_grpc_dir) + shutil.copytree(grpc_dir, target_grpc_dir) + os.remove(os.path.join(target_grpc_dir, 'LICENSE')) + os.remove(os.path.join(target_grpc_dir, 'build.gradle')) + subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) api_unversioned_name = '{}-{}'.format(org_name, api_name) if api_name in dir_overrides: From 42f0d60cc8071b98e6b02bef9db4b021580409c7 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 18:29:15 -0700 Subject: [PATCH 04/12] works with compute and bigquerydt --- utilities/batch_generate_apis.py | 6 +++--- utilities/generate_api.py | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index b27697107dbb..5e2d8637b6a7 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -79,9 +79,9 @@ def generate(artman_yaml): # Run java_discogapic task. No proto or grpc libraries are generated. generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), "java_discogapic", - replace_proto_libs=False) + root_dir=discovery_repo, + has_proto_libs=False) - os.chdir(discovery_repo) generate('gapic/google/compute/artman_compute.yaml') @@ -92,7 +92,7 @@ def main(): parser.add_argument('discovery', help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - # run_gapic_gen(args.googleapis) + run_gapic_gen(args.googleapis) run_discogapic_gen(args.discovery) diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 36c1e3fc5b98..6a6584a5918f 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -36,25 +36,32 @@ } -def run_generate_api(config_path, artifact_type, noisy=False, replace_proto_libs=True): +def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=True, noisy=False): """ Generate an API client library. :param config_path: (str) Path to directory containing artman config file. :param artifact_type: (str) artman target, e.g "java_gapic". + :param root_dir: (str) Path to the directory containing the API definitions. + :param has_proto_libs: (bool) if the generated proto and grpc libraries + should replace the existing ones :param noisy: (bool) if console output should be verbose. - :param replace_proto_libs: (bool) if the generated proto and grpc libraries - should replace the existing ones + """ googleapis_index = config_path.rfind('/google/') if googleapis_index == -1: raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') - root_dir = config_path[0:googleapis_index] - api_dir = config_path[googleapis_index+1:] + if has_proto_libs: + root_dir = config_path[0:googleapis_index] + api_dir = config_path[googleapis_index+1:] + else: + api_dir = config_path extra_options = [] if noisy: extra_options = ['-v'] + print("api_dir: "+ api_dir) + print("root_dir: "+ root_dir) subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', artifact_type]) with io.open(config_path, encoding='UTF-8') as config_file: @@ -73,7 +80,7 @@ def run_generate_api(config_path, artifact_type, noisy=False, replace_proto_libs if not os.path.exists(gapic_dir): raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir)) - if replace_proto_libs: + if has_proto_libs: proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) From 66775ed6c4299d61137eb1524a3ff411f8b79bc0 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 18:29:51 -0700 Subject: [PATCH 05/12] uncomment all other java_gapic targets --- utilities/batch_generate_apis.py | 68 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index 5e2d8637b6a7..c869156f0254 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -38,40 +38,40 @@ def generate(artman_yaml): #generate('google/datastore/artman_datastore.yaml') generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') - # generate('google/bigtable/artman_bigtable.yaml') - # generate('google/bigtable/admin/artman_bigtableadmin.yaml') - # generate('google/container/artman_container.yaml') - # generate('google/cloud/dataproc/artman_dataproc_v1.yaml') - # generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') - # generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') - # generate('google/privacy/dlp/artman_dlp_v2.yaml') - # generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') - # generate('google/firestore/artman_firestore.yaml') - # generate('google/cloud/language/artman_language_v1.yaml') - # generate('google/cloud/language/artman_language_v1beta2.yaml') - # generate('google/logging/artman_logging.yaml') - # generate('google/monitoring/artman_monitoring.yaml') - # generate('google/pubsub/artman_pubsub.yaml') - # generate('google/cloud/oslogin/artman_oslogin_v1.yaml') - # generate('google/cloud/redis/artman_redis_v1beta1.yaml') - # generate('google/spanner/artman_spanner.yaml') - # generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') - # generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') - # generate('google/cloud/speech/artman_speech_v1.yaml') - # generate('google/cloud/speech/artman_speech_v1beta1.yaml') - # generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') - # generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') - # generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') - # generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') - # generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') - # generate('google/cloud/vision/artman_vision_v1.yaml') - # generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') - # generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') - # generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + generate('google/bigtable/artman_bigtable.yaml') + generate('google/bigtable/admin/artman_bigtableadmin.yaml') + generate('google/container/artman_container.yaml') + generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + generate('google/privacy/dlp/artman_dlp_v2.yaml') + generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + generate('google/firestore/artman_firestore.yaml') + generate('google/cloud/language/artman_language_v1.yaml') + generate('google/cloud/language/artman_language_v1beta2.yaml') + generate('google/logging/artman_logging.yaml') + generate('google/monitoring/artman_monitoring.yaml') + generate('google/pubsub/artman_pubsub.yaml') + generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + generate('google/cloud/redis/artman_redis_v1beta1.yaml') + generate('google/spanner/artman_spanner.yaml') + generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + generate('google/cloud/speech/artman_speech_v1.yaml') + generate('google/cloud/speech/artman_speech_v1beta1.yaml') + generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + generate('google/cloud/vision/artman_vision_v1.yaml') + generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') def run_discogapic_gen(discovery_repo): From 7185ddcd748ff75f982cf1d00d48202d787e2227 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Tue, 10 Jul 2018 18:31:28 -0700 Subject: [PATCH 06/12] formatting --- utilities/batch_generate_apis.py | 4 ++-- utilities/generate_api.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index c869156f0254..83ce548111d3 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -89,12 +89,12 @@ def main(): # TODO Make the docker image the default, add --local option parser = argparse.ArgumentParser(description='Batch generate all APIs.') parser.add_argument('googleapis', help='The path to the googleapis repo') - parser.add_argument('discovery', help='The path to the discovery-artifact-manager repo') + parser.add_argument('discovery', + help='The path to the discovery-artifact-manager repo') args = parser.parse_args() run_gapic_gen(args.googleapis) run_discogapic_gen(args.discovery) - if __name__ == '__main__': main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 6a6584a5918f..1d394bb0a86d 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -42,8 +42,7 @@ def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=T :param config_path: (str) Path to directory containing artman config file. :param artifact_type: (str) artman target, e.g "java_gapic". :param root_dir: (str) Path to the directory containing the API definitions. - :param has_proto_libs: (bool) if the generated proto and grpc libraries - should replace the existing ones + :param has_proto_libs: (bool) if this target creates proto and grpc libraries :param noisy: (bool) if console output should be verbose. """ @@ -60,8 +59,6 @@ def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=T if noisy: extra_options = ['-v'] - print("api_dir: "+ api_dir) - print("root_dir: "+ root_dir) subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', artifact_type]) with io.open(config_path, encoding='UTF-8') as config_file: From 89298fa63cbfd362295586806e79816c1dbf4f9c Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 10:58:10 -0700 Subject: [PATCH 07/12] add .git --- utilities/batch_generate_apis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index 83ce548111d3..800b9807f883 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -18,7 +18,7 @@ # (e.g. from https://github.com/googleapis/googleapis): # # $ git checkout https://github.com/googleapis/googleapis.git -# $ git checkout https://github.com/googleapis/discovery-artifact-manager +# $ git checkout https://github.com/googleapis/discovery-artifact-manager.git # # Run this script: # From 481169460bd0da5451acf4a38052ff5e363d7dcf Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 11:43:00 -0700 Subject: [PATCH 08/12] fix --- utilities/batch_generate_apis.py | 76 ++++++++++++++++---------------- utilities/generate_api.py | 23 +++++----- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index 800b9807f883..844982520b80 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -32,55 +32,53 @@ def run_gapic_gen(googleapis): def generate(artman_yaml): - generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), "java_gapic") + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), generate_api.JAVA_GAPIC) # TODO Needs to have java_proto called instead of java_grpc #generate('google/datastore/artman_datastore.yaml') generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') - generate('google/bigtable/artman_bigtable.yaml') - generate('google/bigtable/admin/artman_bigtableadmin.yaml') - generate('google/container/artman_container.yaml') - generate('google/cloud/dataproc/artman_dataproc_v1.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') - generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') - generate('google/privacy/dlp/artman_dlp_v2.yaml') - generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') - generate('google/firestore/artman_firestore.yaml') - generate('google/cloud/language/artman_language_v1.yaml') - generate('google/cloud/language/artman_language_v1beta2.yaml') - generate('google/logging/artman_logging.yaml') - generate('google/monitoring/artman_monitoring.yaml') - generate('google/pubsub/artman_pubsub.yaml') - generate('google/cloud/oslogin/artman_oslogin_v1.yaml') - generate('google/cloud/redis/artman_redis_v1beta1.yaml') - generate('google/spanner/artman_spanner.yaml') - generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') - generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') - generate('google/cloud/speech/artman_speech_v1.yaml') - generate('google/cloud/speech/artman_speech_v1beta1.yaml') - generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') - generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') - generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') - generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') - generate('google/cloud/vision/artman_vision_v1.yaml') - generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') - generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') - generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + # generate('google/bigtable/artman_bigtable.yaml') + # generate('google/bigtable/admin/artman_bigtableadmin.yaml') + # generate('google/container/artman_container.yaml') + # generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + # generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + # generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + # generate('google/privacy/dlp/artman_dlp_v2.yaml') + # generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + # generate('google/firestore/artman_firestore.yaml') + # generate('google/cloud/language/artman_language_v1.yaml') + # generate('google/cloud/language/artman_language_v1beta2.yaml') + # generate('google/logging/artman_logging.yaml') + # generate('google/monitoring/artman_monitoring.yaml') + # generate('google/pubsub/artman_pubsub.yaml') + # generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + # generate('google/cloud/redis/artman_redis_v1beta1.yaml') + # generate('google/spanner/artman_spanner.yaml') + # generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + # generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + # generate('google/cloud/speech/artman_speech_v1.yaml') + # generate('google/cloud/speech/artman_speech_v1beta1.yaml') + # generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + # generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + # generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + # generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + # generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + # generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + # generate('google/cloud/vision/artman_vision_v1.yaml') + # generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + # generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + # generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') def run_discogapic_gen(discovery_repo): def generate(artman_yaml): # Run java_discogapic task. No proto or grpc libraries are generated. generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml), - "java_discogapic", - root_dir=discovery_repo, - has_proto_libs=False) + generate_api.JAVA_DISCOGAPIC) generate('gapic/google/compute/artman_compute.yaml') @@ -93,7 +91,7 @@ def main(): help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - run_gapic_gen(args.googleapis) + # run_gapic_gen(args.googleapis) run_discogapic_gen(args.discovery) if __name__ == '__main__': diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 1d394bb0a86d..dbfa29a3cd17 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -35,8 +35,10 @@ 'spanner-admin-database': 'google-cloud-spanner' } +JAVA_GAPIC="java_gapic" +JAVA_DISCOGAPIC="java_discogapic" -def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=True, noisy=False): +def run_generate_api(config_path, artifact_type, noisy=False): """ Generate an API client library. :param config_path: (str) Path to directory containing artman config file. @@ -46,20 +48,21 @@ def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=T :param noisy: (bool) if console output should be verbose. """ - googleapis_index = config_path.rfind('/google/') - if googleapis_index == -1: + api_repo_index = config_path.rfind('/google/') + if artifact_type == JAVA_DISCOGAPIC: + api_repo_index = config_path.rfind('/gapic/') + if api_repo_index == -1: raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') - if has_proto_libs: - root_dir = config_path[0:googleapis_index] - api_dir = config_path[googleapis_index+1:] - else: - api_dir = config_path + root_dir = config_path[0:api_repo_index] + api_dir = config_path[api_repo_index+1:] extra_options = [] if noisy: extra_options = ['-v'] - subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', artifact_type]) + subprocess.check_call( + ['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + + extra_options + ['generate', artifact_type]) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader) @@ -77,7 +80,7 @@ def run_generate_api(config_path, artifact_type, root_dir=None, has_proto_libs=T if not os.path.exists(gapic_dir): raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir)) - if has_proto_libs: + if artifact_type != JAVA_DISCOGAPIC: proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) From dfbb72624959256fa22feeb71423b87b2b086efe Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 11:44:24 -0700 Subject: [PATCH 09/12] uncomment --- utilities/batch_generate_apis.py | 70 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index 844982520b80..d5419fbd663e 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -38,40 +38,40 @@ def generate(artman_yaml): #generate('google/datastore/artman_datastore.yaml') generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') - # generate('google/bigtable/artman_bigtable.yaml') - # generate('google/bigtable/admin/artman_bigtableadmin.yaml') - # generate('google/container/artman_container.yaml') - # generate('google/cloud/dataproc/artman_dataproc_v1.yaml') - # generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') - # generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') - # generate('google/privacy/dlp/artman_dlp_v2.yaml') - # generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') - # generate('google/firestore/artman_firestore.yaml') - # generate('google/cloud/language/artman_language_v1.yaml') - # generate('google/cloud/language/artman_language_v1beta2.yaml') - # generate('google/logging/artman_logging.yaml') - # generate('google/monitoring/artman_monitoring.yaml') - # generate('google/pubsub/artman_pubsub.yaml') - # generate('google/cloud/oslogin/artman_oslogin_v1.yaml') - # generate('google/cloud/redis/artman_redis_v1beta1.yaml') - # generate('google/spanner/artman_spanner.yaml') - # generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') - # generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') - # generate('google/cloud/speech/artman_speech_v1.yaml') - # generate('google/cloud/speech/artman_speech_v1beta1.yaml') - # generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') - # generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') - # generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') - # generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') - # generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') - # generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') - # generate('google/cloud/vision/artman_vision_v1.yaml') - # generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') - # generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') - # generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + generate('google/bigtable/artman_bigtable.yaml') + generate('google/bigtable/admin/artman_bigtableadmin.yaml') + generate('google/container/artman_container.yaml') + generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + generate('google/privacy/dlp/artman_dlp_v2.yaml') + generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + generate('google/firestore/artman_firestore.yaml') + generate('google/cloud/language/artman_language_v1.yaml') + generate('google/cloud/language/artman_language_v1beta2.yaml') + generate('google/logging/artman_logging.yaml') + generate('google/monitoring/artman_monitoring.yaml') + generate('google/pubsub/artman_pubsub.yaml') + generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + generate('google/cloud/redis/artman_redis_v1beta1.yaml') + generate('google/spanner/artman_spanner.yaml') + generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + generate('google/cloud/speech/artman_speech_v1.yaml') + generate('google/cloud/speech/artman_speech_v1beta1.yaml') + generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + generate('google/cloud/vision/artman_vision_v1.yaml') + generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') def run_discogapic_gen(discovery_repo): @@ -91,7 +91,7 @@ def main(): help='The path to the discovery-artifact-manager repo') args = parser.parse_args() - # run_gapic_gen(args.googleapis) + run_gapic_gen(args.googleapis) run_discogapic_gen(args.discovery) if __name__ == '__main__': From deed41733ca6e73b6149340dfc6907e9548f95f1 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 11:49:03 -0700 Subject: [PATCH 10/12] discovery_repo --- utilities/batch_generate_apis.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py index d5419fbd663e..32ebf9bbd140 100644 --- a/utilities/batch_generate_apis.py +++ b/utilities/batch_generate_apis.py @@ -32,7 +32,8 @@ def run_gapic_gen(googleapis): def generate(artman_yaml): - generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), generate_api.JAVA_GAPIC) + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml), + generate_api.JAVA_GAPIC) # TODO Needs to have java_proto called instead of java_grpc #generate('google/datastore/artman_datastore.yaml') @@ -87,12 +88,12 @@ def main(): # TODO Make the docker image the default, add --local option parser = argparse.ArgumentParser(description='Batch generate all APIs.') parser.add_argument('googleapis', help='The path to the googleapis repo') - parser.add_argument('discovery', + parser.add_argument('discovery_repo', help='The path to the discovery-artifact-manager repo') args = parser.parse_args() run_gapic_gen(args.googleapis) - run_discogapic_gen(args.discovery) + run_discogapic_gen(args.discovery_repo) if __name__ == '__main__': main() From cdfa471d6946e3e44adbc5132ae1ec37a144c8b0 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 11:53:08 -0700 Subject: [PATCH 11/12] comments --- utilities/generate_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utilities/generate_api.py b/utilities/generate_api.py index dbfa29a3cd17..e524dc6c3700 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -15,8 +15,9 @@ # Instructions: # # Find the artman config file the describes the API you want to generate a client for. +# Specifiy the artman ARTIFACT_TYPE to generate, e.g. "java_gapic" # -# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE java_gapic +# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE ARTIFACT_TYPE import argparse import io @@ -52,7 +53,7 @@ def run_generate_api(config_path, artifact_type, noisy=False): if artifact_type == JAVA_DISCOGAPIC: api_repo_index = config_path.rfind('/gapic/') if api_repo_index == -1: - raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') + raise ValueError('Didn\'t find the API repo in config file path; need absolute path to the artman config file.') root_dir = config_path[0:api_repo_index] api_dir = config_path[api_repo_index+1:] From 45fe5c0a3c854a3c10ca996c563606df8569bd83 Mon Sep 17 00:00:00 2001 From: Andrea Lin Date: Wed, 11 Jul 2018 12:26:53 -0700 Subject: [PATCH 12/12] removing old param doc strings --- utilities/generate_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utilities/generate_api.py b/utilities/generate_api.py index e524dc6c3700..e60027f38855 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -44,8 +44,6 @@ def run_generate_api(config_path, artifact_type, noisy=False): :param config_path: (str) Path to directory containing artman config file. :param artifact_type: (str) artman target, e.g "java_gapic". - :param root_dir: (str) Path to the directory containing the API definitions. - :param has_proto_libs: (bool) if this target creates proto and grpc libraries :param noisy: (bool) if console output should be verbose. """