Various updates

src/azdev/commands.py

@@ -51,6 +51,10 @@ def operation_group(name):
         g.command('add', 'add_extension')
         g.command('remove', 'remove_extension')
         g.command('list', 'list_extensions')
+        g.command('update-index', 'update_extension_index')
+
+    with CommandGroup(self, 'group', operation_group('resource')) as g:
+        g.command('delete', 'delete_groups')
 
     # TODO: implement
     # with CommandGroup(self, operation_group('help')) as g:

src/azdev/help.py

@@ -17,12 +17,9 @@
     long-summary: Find or clones the relevant repositories and installs the necessary modules.
 """
 
-helps['configure'] = """
-    short-summary: Configure azdev for use without installing anything.
-"""
 
 helps['configure'] = """
-    short-summary: Configure `azdev` for development.
+    short-summary: Configure azdev for use without installing anything.
 """
 
 
@@ -153,3 +150,15 @@
 helps['extension list'] = """
     short-summary: List what extensions are currently visible to your development environment.
 """
+
+helps['extension update-index'] = """
+    short-summary: Update the extensions index.json from a built WHL file.
+"""
+
+helps['group delete'] = """
+    short-summary: Delete several resource groups with filters. Useful for cleaning up test resources.
+    long-summary: >
+        Can filter either by key tags used by the CLI infrastructure, or by name prefix. If name prefix
+        is used, the tag filters will be ignored. This command doesn't guarantee the resource group will
+        be deleted.
+"""

src/azdev/operations/extensions.py → src/azdev/operations/extensions/__init__.py

@@ -1,8 +1,7 @@
-# -----------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# -----------------------------------------------------------------------------
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
 
 from collections import OrderedDict
 from glob import glob
@@ -85,3 +84,58 @@ def list_extensions():
         if long_name not in installed:
             results.append({'name': long_name, 'path': folder})
     return results
+
+
+def _get_sha256sum(a_file):
+    import hashlib
+    sha256 = hashlib.sha256()
+    with open(a_file, 'rb') as f:
+        sha256.update(f.read())
+    return sha256.hexdigest()
+
+
+def update_extension_index(extension):
+    import json
+    import re
+    import tempfile
+
+    from .util import get_ext_metadata, get_whl_from_url
+
+    NAME_REGEX = r'.*/([^/]*)-\d+.\d+.\d+'
+
+    ext_path = get_ext_repo_path()
+
+    # Get extension WHL from URL
+    if not extension.endswith('.whl') or not extension.startswith('https:'):
+        raise ValueError('usage error: only URL to a WHL file currently supported.')
+
+    # Extract the extension name
+    try:
+        extension_name = re.findall(NAME_REGEX, extension)[0]
+        extension_name = extension_name.replace('_', '-')
+    except IndexError:
+        raise ValueError('unable to parse extension name')
+
+    extensions_dir = tempfile.mkdtemp()
+    ext_dir = tempfile.mkdtemp(dir=extensions_dir)
+    whl_cache_dir = tempfile.mkdtemp()
+    whl_cache = {}
+    ext_file = get_whl_from_url(whl_path, extension_name, whl_cache_dir, whl_cache)
+
+    with open(os.join(ext_path, 'src', 'index.json'), 'r') as infile:
+        curr_index = json.loads(infile.read())
+
+    try:
+        entry = curr_index['extensions'][extension_name]
+    except IndexError:
+        raise ValueError('{} not found in index.json'.format(extension_name))
+
+    entry[0]['downloadUrl'] = whl_path
+    entry[0]['sha256Digest'] = _get_sha256sum(ext_file)
+    entry[0]['filename'] = whl_path.split('/')[-1]
+    entry[0]['metadata'] = get_ext_metadata(ext_dir, ext_file, extension_name)
+
+    # update index and write back to file
+    curr_index['extensions'][extension_name] = entry
+    with open(os.join(ext_path, 'src', 'index.json'), 'w') as outfile:
+        outfile.write(json.dumps(curr_index, indent=4, sort_keys=True))

src/azdev/operations/extensions/util.py

@@ -0,0 +1,71 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import json
+import zipfile
+from wheel.install import WHEEL_INFO_RE
+
+from azdev.utilities import EXTENSION_PREFIX
+
+
+def _get_extension_modname(ext_dir):
+    # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153
+    pos_mods = [n for n in os.listdir(ext_dir)
+                if n.startswith(EXTENSION_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))]
+    if len(pos_mods) != 1:
+        raise AssertionError("Expected 1 module to load starting with "
+                             "'{}': got {}".format(EXTENSION_PREFIX, pos_mods))
+    return pos_mods[0]
+
+
+def _get_azext_metadata(ext_dir):
+    # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109
+    AZEXT_METADATA_FILENAME = 'azext_metadata.json'
+    azext_metadata = None
+    ext_modname = _get_extension_modname(ext_dir=ext_dir)
+    azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME)
+    if os.path.isfile(azext_metadata_filepath):
+        with open(azext_metadata_filepath) as f:
+            azext_metadata = json.load(f)
+    return azext_metadata
+
+
+def get_ext_metadata(ext_dir, ext_file, ext_name):
+    # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89
+    WHL_METADATA_FILENAME = 'metadata.json'
+    zip_ref = zipfile.ZipFile(ext_file, 'r')
+    zip_ref.extractall(ext_dir)
+    zip_ref.close()
+    metadata = {}
+    dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
+    azext_metadata = _get_azext_metadata(ext_dir)
+    if azext_metadata:
+        metadata.update(azext_metadata)
+    for dist_info_dirname in dist_info_dirs:
+        parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
+        if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'):
+            whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
+            if os.path.isfile(whl_metadata_filepath):
+                with open(whl_metadata_filepath) as f:
+                    metadata.update(json.load(f))
+    return metadata
+
+
+def get_whl_from_url(url, filename, tmp_dir, whl_cache=None):
+    if not whl_cache:
+        whl_cache = {}
+    if url in whl_cache:
+        return whl_cache[url]
+    import requests
+    r = requests.get(url, stream=True)
+    assert r.status_code == 200, "Request to {} failed with {}".format(url, r.status_code)
+    ext_file = os.path.join(tmp_dir, filename)
+    with open(ext_file, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024):
+            if chunk:  # ignore keep-alive new chunks
+                f.write(chunk)
+    whl_cache[url] = ext_file
+    return ext_file

src/azdev/operations/performance.py

@@ -19,7 +19,6 @@
 logger = get_logger(__name__)
 
 TOTAL = 'ALL'
-NUM_RUNS = 3
 DEFAULT_THRESHOLD = 10
 
 # TODO: Treat everything as bubble instead of specific modules
@@ -33,14 +32,14 @@
 }
 
 
-def check_load_time():
+def check_load_time(runs=3):
     heading('Module Load Performance')
 
     regex = r"[^']*'([^']*)'[\D]*([\d\.]*)"
 
     results = {TOTAL: []}
     # Time the module loading X times
-    for i in range(0, NUM_RUNS + 1):
+    for i in range(0, runs + 1):
         lines = cmd('az -h --debug', show_stderr=True).result
         if i == 0:
             # Ignore the first run since it can be longer due to *.pyc file compilation

src/azdev/operations/resource.py

@@ -0,0 +1,81 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import json
+
+from knack.log import get_logger
+from knack.prompting import prompt_y_n
+from knack.util import CLIError
+
+from azdev.utilities import (
+    cmd as run_cmd, heading, subheading, display)
+
+logger = get_logger(__name__)
+
+
+class Data(object):
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+        if 'properties' in self.__dict__:
+            self.__dict__.update(self.properties)
+            del self.properties
+
+
+def delete_groups(cmd, prefixes=None, older_than=6, product='azurecli', cause='automation', yes=False):
+    from datetime import datetime, timedelta
+
+    groups = json.loads(run_cmd('az group list -ojson').result)
+    groups_to_delete = []
+
+    def _filter_by_tags():
+        for group in groups:
+            group = Data(**group)
+
+            if not group.tags:
+                continue
+
+            tags = Data(**group.tags)
+            try:
+                date_tag = datetime.strptime(tags.date, '%Y-%m-%dT%H:%M:%SZ')
+                curr_time = datetime.utcnow()
+                if tags.product == product and tags.cause == cause and (curr_time - date_tag <= timedelta(hours=older_than + 1)):
+                    groups_to_delete.append(group.name)
+            except AttributeError:
+                continue
+
+    def _filter_by_prefix():
+        for group in groups:
+            group = Data(**group)
+
+            for prefix in prefixes:
+                if group.name.startswith(prefix):
+                    groups_to_delete.append(group.name)
+
+    def _delete():
+        for group in groups_to_delete:
+            run_cmd('az group delete -g {} -y --no-wait'.format(group), message=True)
+
+    if prefixes:
+        logger.info('Filter by prefix')
+        _filter_by_prefix()
+    else:
+        logger.info('Filter by tags')
+        _filter_by_tags
+
+    if not groups_to_delete:
+        raise CLIError('No groups meet the criteria to delete.')
+
+    if yes:
+        _delete()
+    else:
+        subheading('Groups to Delete')
+        for group in groups_to_delete:
+            display('\t{}'.format(group))
+
+        if prompt_y_n('Delete {} resource groups?'.format(len(groups_to_delete)), 'y'):
+            _delete()
+        else:
+            raise CLIError('Command cancelled.')

src/azdev/operations/setup.py

@@ -17,7 +17,7 @@
 from azdev.params import Flag
 from azdev.utilities import (
     display, heading, subheading, cmd, py_cmd, pip_cmd, find_file, IS_WINDOWS, get_path_table,
-    get_azdev_config_dir)
+    get_env_config_dir, get_env_config)
 
 logger = get_logger(__name__)
 
@@ -144,7 +144,7 @@ def _copy_config_files():
 
     config_mod = import_module('azdev.config')
     config_dir_path = config_mod.__dict__['__path__'][0]
-    dest_path = os.path.join(get_azdev_config_dir(), 'config_files')
+    dest_path = os.path.join(get_env_config_dir(), 'config_files')
     if os.path.exists(dest_path):
         rmtree(dest_path)
     copytree(config_dir_path, dest_path)
@@ -213,14 +213,14 @@ def setup(cmd, venv='env', cli_path=None, ext_path=None, yes=None):
                 _get_venv_activate_command(venv)
             )
         )
+    config = get_env_config()
 
     # save data to config files
-    config = cmd.cli_ctx.config
     if ext_path:
         from azdev.utilities import get_azure_config
         config.set_value('ext', 'repo_path', ext_path)
         az_config = get_azure_config()
-        az_config.set_value('extension', 'dir', os.path.join(ext_path, 'src'))
+        az_config.set_value('extension', 'dir', os.path.join(ext_path))
 
     if cli_path:
         config.set_value('cli', 'repo_path', cli_path)
@@ -262,12 +262,12 @@ def configure(cmd, cli_path=None, ext_path=None):
         display("Azure CLI extensions repo found at: {}".format(ext_path))
 
     # save data to config files
-    config = cmd.cli_ctx.config
+    config = get_env_config()
     if ext_path:
         from azdev.utilities import get_azure_config
         config.set_value('ext', 'repo_path', ext_path)
         az_config = get_azure_config()
-        az_config.set_value('extension', 'dir', os.path.join(ext_path, 'src'))
+        az_config.set_value('extension', 'dir', os.path.join(ext_path))
 
     if cli_path:
         config.set_value('cli', 'repo_path', cli_path)

src/azdev/operations/style.py

@@ -19,7 +19,7 @@
 
 from azdev.utilities import (
     display, heading, subheading, py_cmd, get_path_table, EXTENSION_PREFIX,
-    get_azdev_config_dir)
+    get_env_config_dir)
 
 
 def check_style(cmd, modules=None, pylint=False, pep8=False):
@@ -112,7 +112,7 @@ def _run_pylint(cli_path, ext_path, modules):
     def run(paths, rcfile, desc):
         if not paths:
             return None
-        config_path = os.path.join(get_azdev_config_dir(), 'config_files', rcfile)
+        config_path = os.path.join(get_env_config_dir(), 'config_files', rcfile)
         logger.info('Using rcfile file: %s', config_path)
         logger.info('Running on %s: %s', desc, ' '.join(paths))
         command = 'pylint {} --rcfile={} -j {}'.format(' '.join(paths),
@@ -133,7 +133,7 @@ def _run_pep8(cli_path, ext_path, modules):
     def run(paths, config_file, desc):
         if not paths:
             return
-        config_path = os.path.join(get_azdev_config_dir(), 'config_files', config_file)
+        config_path = os.path.join(get_env_config_dir(), 'config_files', config_file)
         logger.info('Using config file: %s', config_path)
         logger.info('Running on %s: %s', desc, ' '.join(paths))
         command = 'flake8 --statistics --append-config={} {}'.format(

src/azdev/operations/tests/__init__.py

@@ -20,14 +20,14 @@
     cmd, py_cmd, pip_cmd, find_file, IS_WINDOWS,
     ENV_VAR_TEST_MODULES, ENV_VAR_TEST_LIVE,
     COMMAND_MODULE_PREFIX, EXTENSION_PREFIX,
-    make_dirs, get_azdev_config_dir,
+    make_dirs, get_env_config_dir,
     get_path_table)
 
 logger = get_logger(__name__)
 
 
 DEFAULT_RESULT_FILE = 'test_results.xml'
-DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE)
+DEFAULT_RESULT_PATH = os.path.join(get_env_config_dir(), DEFAULT_RESULT_FILE)
 
 
 def run_tests(cmd, tests, xml_path=None, ci_mode=False, discover=False, in_series=False,
@@ -266,7 +266,8 @@ def add_to_index(key, path):
 
 
 def _get_test_index(cmd, profile, discover):
-    test_index_dir = os.path.join(cmd.cli_ctx.config.config_dir, 'test_index')
+    config_dir = get_env_config_dir()
+    test_index_dir = os.path.join(config_dir, 'test_index')
     make_dirs(test_index_dir)
     test_index_path = os.path.join(test_index_dir, '{}.json'.format(profile))
     test_index = {}