From 23bc9d94e0352f3150e830de55843fe3ef058eb2 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Tue, 16 Oct 2018 22:39:34 -0700 Subject: [PATCH 01/11] revert to desktop python --- build_scripts/windows/azure-cli.wixproj | 4 +- build_scripts/windows/scripts/build.cmd | 145 ++++++++---------------- 2 files changed, 48 insertions(+), 101 deletions(-) diff --git a/build_scripts/windows/azure-cli.wixproj b/build_scripts/windows/azure-cli.wixproj index 845c4d3d624..09ff14ceae3 100644 --- a/build_scripts/windows/azure-cli.wixproj +++ b/build_scripts/windows/azure-cli.wixproj @@ -2,11 +2,11 @@ - artifacts\wix\ + $(HOMEDRIVE)$(HOMEPATH)\zwix\ $(LocalWixRoot) $(WixToolPath)Wix.targets wixtasks.dll - artifacts\cli + $(HOMEDRIVE)$(HOMEPATH)\zcli -fv diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 26d69af37ca..41140d09d0f 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -3,37 +3,55 @@ SetLocal EnableDelayedExpansion echo build a msi installer using local cli sources and python executables. You need to have curl.exe, unzip.exe and msbuild.exe available under PATH echo. -set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin" - -set PYTHON_DOWNLOAD_URL="https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-win32.zip" -set GET_PIP_DOWNLOAD_URL="https://bootstrap.pypa.io/get-pip.py" -set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" +set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin;" if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) +set PYTHON_VERSION=3.6.1 +set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" :: Set up the output directory and temp. directories echo Cleaning previous build artifacts... - set OUTPUT_DIR=%~dp0..\out if exist %OUTPUT_DIR% rmdir /s /q %OUTPUT_DIR% mkdir %OUTPUT_DIR% -set ARTIFACTS_DIR=%~dp0..\artifacts -mkdir %ARTIFACTS_DIR% - -set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch -set BUILDING_DIR=%ARTIFACTS_DIR%\cli -set PYTHON_DIR=%ARTIFACTS_DIR%\Python -set WIX_DIR=%ARTIFACTS_DIR%\wix +set TEMP_SCRATCH_FOLDER=%HOMEDRIVE%%HOMEPATH%\zcli_scratch +set BUILDING_DIR=%HOMEDRIVE%%HOMEPATH%\zcli +set WIX_DIR=%HOMEDRIVE%%HOMEPATH%\zwix set REPO_ROOT=%~dp0..\..\.. +:: look for python 3.x so we can build into the installer +if not "%1"=="" ( + set PYTHON_DIR=%1 + set PYTHON_EXE=%1\python.exe + goto PYTHON_FOUND +) + +FOR /f %%i IN ('where python') DO ( + set PY_FILE_DRIVE=%%~di + set PY_FILE_PATH=%%~pi + set PY_FILE_NAME=%%~ni + set PYTHON_EXE=!PY_FILE_DRIVE!!PY_FILE_PATH!!PY_FILE_NAME!.exe + set PYTHON_DIR=!PY_FILE_DRIVE!!PY_FILE_PATH! + FOR /F "delims=" %%j IN ('!PYTHON_EXE! --version') DO ( + set PYTHON_VER=%%j + echo.!PYTHON_VER!|findstr /C:"%PYTHON_VERSION%" >nul 2>&1 + if not errorlevel 1 ( + goto PYTHON_FOUND + ) + ) +) +echo python %PYTHON_VERSION% is needed to create installer. +exit /b 1 +:PYTHON_FOUND +echo Python Executables: %PYTHON_DIR%, %PYTHON_EXE% + ::reset working folders if exist %BUILDING_DIR% rmdir /s /q %BUILDING_DIR% - ::rmdir always returns 0, so check folder's existence if exist %BUILDING_DIR% ( echo Failed to delete %BUILDING_DIR%. @@ -41,36 +59,6 @@ if exist %BUILDING_DIR% ( ) mkdir %BUILDING_DIR% -:: get Python -if exist %PYTHON_DIR% rmdir /s /q %PYTHON_DIR% - -:: rmdir always returns 0, so check folder's existence -if exist %PYTHON_DIR% ( - echo Failed to delete %PYTHON_DIR%. - goto ERROR -) - -mkdir %PYTHON_DIR% -pushd %PYTHON_DIR% - -echo Downloading Python. -curl -o python-archive.zip %PYTHON_DOWNLOAD_URL% -k -unzip -q python-archive.zip -unzip -q python36.zip -if %errorlevel% neq 0 goto ERROR - -del python-archive.zip -del python36.zip - -echo Python downloaded and extracted successfully. -echo Setting up pip -curl -o get-pip.py %GET_PIP_DOWNLOAD_URL% -k -%PYTHON_DIR%\python.exe get-pip.py -del get-pip.py -echo Pip set up successful. - -popd - if exist %TEMP_SCRATCH_FOLDER% rmdir /s /q %TEMP_SCRATCH_FOLDER% if exist %TEMP_SCRATCH_FOLDER% ( echo Failed to delete %TEMP_SCRATCH_FOLDER%. @@ -82,11 +70,10 @@ if exist %REPO_ROOT%\privates ( copy %REPO_ROOT%\privates\*.whl %TEMP_SCRATCH_FOLDER% ) -:: ensure wix is available +::ensure wix is available if exist %WIX_DIR% ( echo Using existing Wix at %WIX_DIR% ) - if not exist %WIX_DIR% ( mkdir %WIX_DIR% pushd %WIX_DIR% @@ -103,19 +90,14 @@ if not exist %WIX_DIR% ( robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL :: Build & install all the packages with bdist_wheel +%BUILDING_DIR%\python.exe -m pip install wheel echo Building CLI packages... - -:: Workaround for get bdist_wheel to complete otherwise it fails to import azure_bdist_wheel -set PYTHONPATH=%BUILDING_DIR%\Lib\site-packages -del %BUILDING_DIR%\python36._pth - set CLI_SRC=%REPO_ROOT%\src -for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg %CLI_SRC%\azure-cli-telemetry) do ( +for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg) do ( pushd %%a %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %TEMP_SCRATCH_FOLDER% popd ) - pushd %CLI_SRC%\command_modules for /D %%a in (*) do ( pushd %CLI_SRC%\command_modules\%%a @@ -123,16 +105,6 @@ for /D %%a in (*) do ( popd ) popd - -:: Undo the rest of the workaround and add site-packages to ._pth. -:: See https://docs.python.org/3/using/windows.html#finding-modules -set PYTHONPATH= -( - echo python36.zip - echo . - echo Lib\site-packages -) > %BUILDING_DIR%\python36._pth - echo Built CLI packages successfully. if %errorlevel% neq 0 goto ERROR @@ -142,9 +114,8 @@ for %%i in (%TEMP_SCRATCH_FOLDER%\*.whl) do ( set ALL_MODULES=!ALL_MODULES! %%i ) echo All modules: %ALL_MODULES% -%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall pycparser==2.18 -%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --no-cache-dir %ALL_MODULES% -%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg +%BUILDING_DIR%\python.exe -m pip install --no-cache-dir %ALL_MODULES% +%BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg echo Creating the wbin (Windows binaries) folder that will be added to the path... mkdir %BUILDING_DIR%\wbin @@ -154,49 +125,25 @@ copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% : Delete some files we don't need -rmdir /s /q %BUILDING_DIR%\Scripts -for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( - rmdir /s /q %%a -) - +::rmdir /s /q %BUILDING_DIR%\Scripts +:: for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( +:: rmdir /s /q %%a +::) +:: for /d /r %BUILDING_DIR%\Lib\site-packages %%d in (__pycache__) do ( +:: if exist %%d rmdir /s /q "%%d" +:: ) :: Use universal files and remove Py3 only files pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt for /f %%a in ('dir /b /s *_py3.py') do ( set PY3_FILE=%%a if exist !PY3_FILE! del !PY3_FILE! ) -for /f %%a in ('dir /b /s *_py3.*.pyc') do ( - set PY3_FILE=%%a - if exist !PY3_FILE! del !PY3_FILE! -) -popd - -:: Remove .py and only deploy .pyc files -pushd %BUILDING_DIR%\Lib\site-packages\azure -for /f %%f in ('dir /b /s *.pyc') do ( - set PARENT_DIR=%%~df%%~pf.. - echo !PARENT_DIR! | findstr /C:"!BUILDING_DIR!\Lib\site-packages\pip" 1>nul - if errorlevel 1 ( - set FILENAME=%%~nf - set BASE_FILENAME=!FILENAME:~0,-11! - set pyc=!BASE_FILENAME!.pyc - del !PARENT_DIR!\!BASE_FILENAME!.py - copy %%~f !PARENT_DIR!\!pyc! >nul - del %%~f - ) ELSE ( - echo --SKIP !PARENT_DIR! under pip - ) -) popd -for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do ( - if exist %%d rmdir /s /q "%%d" -) - if %errorlevel% neq 0 goto ERROR echo Building MSI... -MSBuild.exe /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj +msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj start %OUTPUT_DIR% @@ -208,4 +155,4 @@ exit /b 1 :END exit /b 0 -popd +popd \ No newline at end of file From 62487ece69be87ca19b390fdf2f45d004af402b8 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Thu, 18 Oct 2018 17:03:03 -0700 Subject: [PATCH 02/11] change before do model file concat --- build_scripts/windows/scripts/build.cmd | 32 +++++++++++++++++++++---- src/azure-cli/setup.py | 4 +++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 41140d09d0f..ee304a296d6 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -125,21 +125,45 @@ copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% : Delete some files we don't need -::rmdir /s /q %BUILDING_DIR%\Scripts +rmdir /s /q %BUILDING_DIR%\Scripts :: for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( :: rmdir /s /q %%a ::) -:: for /d /r %BUILDING_DIR%\Lib\site-packages %%d in (__pycache__) do ( -:: if exist %%d rmdir /s /q "%%d" -:: ) + :: Use universal files and remove Py3 only files pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt for /f %%a in ('dir /b /s *_py3.py') do ( set PY3_FILE=%%a if exist !PY3_FILE! del !PY3_FILE! ) +for /f %%a in ('dir /b /s *_py3.*.pyc') do ( + set PY3_FILE=%%a + if exist !PY3_FILE! del !PY3_FILE! +) +popd + +:: Remove .py and only deploy .pyc files +pushd %BUILDING_DIR%\Lib\site-packages +for /f %%f in ('dir /b /s *.pyc') do ( + set PARENT_DIR=%%~df%%~pf.. + echo !PARENT_DIR! | findstr /C:"!BUILDING_DIR!\Lib\site-packages\pip" 1>nul + if errorlevel 1 ( + set FILENAME=%%~nf + set BASE_FILENAME=!FILENAME:~0,-11! + set pyc=!BASE_FILENAME!.pyc + del !PARENT_DIR!\!BASE_FILENAME!.py + copy %%~f !PARENT_DIR!\!pyc! >nul + del %%~f + ) ELSE ( + echo --SKIP !PARENT_DIR! under pip + ) +) popd +for /d /r %BUILDING_DIR%\Lib\site-packages\pip %%d in (__pycache__) do ( + if exist %%d rmdir /s /q "%%d" +) + if %errorlevel% neq 0 goto ERROR echo Building MSI... diff --git a/src/azure-cli/setup.py b/src/azure-cli/setup.py index 90f01b7b1aa..e95fd319539 100644 --- a/src/azure-cli/setup.py +++ b/src/azure-cli/setup.py @@ -99,7 +99,9 @@ 'azure-cli-signalr', 'azure-cli-sql', 'azure-cli-storage', - 'azure-cli-vm' + 'azure-cli-vm', + 'urllib3==1.23', + 'urllib3[secure]==1.23' ] with open('README.rst', 'r', encoding='utf-8') as f: From aa7d7f333026522765e5e315698f519a7c9b7cb6 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Sat, 20 Oct 2018 20:45:48 -0700 Subject: [PATCH 03/11] add trimmer --- build_scripts/windows/scripts/build.cmd | 21 ++ .../windows/scripts/patch_models_v2.py | 277 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 build_scripts/windows/scripts/patch_models_v2.py diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index ee304a296d6..04ee9204513 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -117,6 +117,10 @@ echo All modules: %ALL_MODULES% %BUILDING_DIR%\python.exe -m pip install --no-cache-dir %ALL_MODULES% %BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg +pushd %BUILDING_DIR% +%BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py +popd + echo Creating the wbin (Windows binaries) folder that will be added to the path... mkdir %BUILDING_DIR%\wbin copy %REPO_ROOT%\build_scripts\windows\scripts\az.cmd %BUILDING_DIR%\wbin\ @@ -125,6 +129,23 @@ copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% : Delete some files we don't need +pushd %BUILDING_DIR% +for %%i in ( + Doc + include + Scripts + Tcl + Tools + libs +) do ( + if exist %%i ( + echo Deleting %%i... + rmdir /s /q %%i + ) +) +popd + +rmdir /s /q %BUILDING_DIR%\Scripts rmdir /s /q %BUILDING_DIR%\Scripts :: for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( :: rmdir /s /q %%a diff --git a/build_scripts/windows/scripts/patch_models_v2.py b/build_scripts/windows/scripts/patch_models_v2.py new file mode 100644 index 00000000000..837be604146 --- /dev/null +++ b/build_scripts/windows/scripts/patch_models_v2.py @@ -0,0 +1,277 @@ +from enum import Enum +import importlib +import inspect +import logging +from pathlib import Path +import pkgutil +import shutil +import sys +import tempfile + +from msrest.serialization import Model +from msrest.exceptions import HttpOperationError +from msrest.paging import Paged + +_LOGGER = logging.getLogger(__name__) + + +copyright_header = b"""# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is +# regenerated. +# -------------------------------------------------------------------------- + +""" + +header = copyright_header+b"""from msrest.serialization import Model +from msrest.exceptions import HttpOperationError +""" + +paging_header = copyright_header+b"""from msrest.paging import Paged +""" + +init_file = """ +try: + from .{} import * +except (SyntaxError, ImportError): + from .{} import * +from .{} import * +""" + +MODEL_PY2_NAME = "_models" +MODEL_PY3_NAME = "_models_py3" +PAGED_NAME = "_paged_models" + +def as_file_name(name): + return name + ".py" + +def parse_input(input_parameter): + """From a syntax like package_name#submodule, build a package name + and complete module name. + """ + split_package_name = input_parameter.split('#') + package_name = split_package_name[0] + module_name = package_name.replace("-", ".") + if len(split_package_name) >= 2: + module_name = ".".join([module_name, split_package_name[1]]) + return package_name, module_name + +def solve_mro(models): + for models_module in models: + models_path = models_module.__path__[0] + _LOGGER.info(f"Working on {models_path}") + if Path(models_path, as_file_name(MODEL_PY3_NAME)).exists(): + _LOGGER.info("Skipping since already patched") + return + + # Build the new files in a temp folder + with tempfile.TemporaryDirectory() as temp_folder: + final_models_path = Path(temp_folder, "models") + final_models_path.mkdir() + solve_one_model(models_module, final_models_path) + + # Swith the files + shutil.rmtree(models_path) + shutil.move(final_models_path, models_path) + +def solve_one_model(models_module, output_folder): + """Will build the compacted models in the output_folder""" + + models_classes = [ + (len(model_class.__mro__), model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Model in model_class.__mro__ + ] + # Sort on MRO size first, and then alphabetically + models_classes.sort(key=lambda x: (x[0], x[1])) + + # Just need the name of exceptions + exceptions_classes = [ + model_name for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and HttpOperationError in model_class.__mro__ + ] + + py2_models_classes = [ + (len_mro, model_name, path.replace("_py3.py", ".py"), None) + for len_mro, model_name, path, _ in models_classes + ] + + paged_models_classes = [ + (model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Paged in model_class.__mro__ + ] + + enum_models_classes = [ + (model_name, inspect.getfile(model_class), model_class) for model_name, model_class in vars(models_module).items() + if model_name[0].isupper() and Enum in model_class.__mro__ + ] + if enum_models_classes: + # Can't be more than one enum file + enum_file = Path(enum_models_classes[0][1]) + enum_file_module_name = "_"+enum_file.with_suffix('').name + shutil.copyfile(enum_file, Path(output_folder, as_file_name(enum_file_module_name))) + else: + enum_file_module_name = None + + write_model_file(Path(output_folder, as_file_name(MODEL_PY3_NAME)), models_classes) + write_model_file(Path(output_folder, as_file_name(MODEL_PY2_NAME)), py2_models_classes) + write_paging_file(Path(output_folder, as_file_name(PAGED_NAME)), paged_models_classes) + write_complete_init( + Path(output_folder, "__init__.py"), + models_classes, + exceptions_classes, + paged_models_classes, + enum_models_classes, + enum_file_module_name + ) + +def write_model_file(output_file_path, classes_to_write): + with open(output_file_path, "bw") as write_fd: + write_fd.write(header) + + for model in classes_to_write: + _, _, model_file_path, _ = model + + with open(model_file_path, "rb") as read_fd: + lines = read_fd.readlines() + # Skip until it's "class XXXX" + while lines: + if lines[0].startswith(b"class "): + break + lines.pop(0) + else: + raise ValueError("Never found any class definition!") + # Now I keep everything + write_fd.write(b'\n') + write_fd.write(b'\n') + write_fd.writelines(lines) + +def write_paging_file(output_file_path, classes_to_write): + with open(output_file_path, "bw") as write_fd: + write_fd.write(paging_header) + + for model in classes_to_write: + _, model_file_path, _ = model + + with open(model_file_path, "rb") as read_fd: + # Skip the first 15 lines (based on Autorest deterministic behavior) + # If we want this less random, look for the first line starts with "class" + lines = read_fd.readlines()[14:] + write_fd.write(b'\n') + write_fd.write(b'\n') + write_fd.writelines(lines) + +def write_init(output_file_path, model_file_name, model_file_name_py2, paging_file_name, enum_file_name): + with open(output_file_path, "bw") as write_fd: + write_fd.write(copyright_header) + + write_fd.write(init_file.format( + model_file_name, + model_file_name_py2, + paging_file_name, + ).encode('utf8')) + if enum_file_name: + write_fd.write( + "from .{} import *".format(enum_file_name).encode('utf8') + ) + +def write_complete_init(output_file_path, models, exceptions_classes, paging_models, enum_models, enum_file_module_name): + with open(output_file_path, "bw") as write_fd: + write_fd.write(copyright_header) + + write_fd.write(b"\ntry:\n") + # Write py3 import + for _, model_name, _, _ in models: + write_fd.write(f" from .{MODEL_PY3_NAME} import {model_name}\n".encode("utf-8")) + # Py 3 exceptions + for model_name in exceptions_classes: + write_fd.write(f" from .{MODEL_PY3_NAME} import {model_name}\n".encode("utf-8")) + + write_fd.write(b"except (SyntaxError, ImportError):\n") + + # Write py2 import + for _, model_name, _, _ in models: + write_fd.write(f" from .{MODEL_PY2_NAME} import {model_name}\n".encode("utf-8")) + # Py 2 exceptions + for model_name in exceptions_classes: + write_fd.write(f" from .{MODEL_PY2_NAME} import {model_name}\n".encode("utf-8")) + + # Write paged import + for model_name, _, _ in paging_models: + write_fd.write(f"from .{PAGED_NAME} import {model_name}\n".encode("utf-8")) + + if enum_models: + # Write enum model import + for model_name, _, _ in enum_models: + write_fd.write(f"from .{enum_file_module_name} import {model_name}\n".encode("utf-8")) + + write_fd.write(b"\n\n__all__=[\n") + # Write all classes name + for _, model_name, _, _ in models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + for model_name in exceptions_classes: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + for model_name, _, _ in paging_models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + if enum_models: + for model_name, _, _ in enum_models: + write_fd.write(f" '{model_name}',\n".encode("utf-8")) + + write_fd.write(b"]\n") + + +def find_models_to_change(module_name): + """Will figure out if the package is a multi-api one, + and understand what to generate. + """ + main_module = importlib.import_module(module_name) + try: + models_module = main_module.models + models_module.__path__ + # It didn't fail, that's a single API package + return [models_module] + except AttributeError: + # This means I loaded the fake module "models" + # and it's multi-api, load all models + return [ + importlib.import_module('.'+label+'.models', main_module.__name__) + for (_, label, ispkg) in pkgutil.iter_modules(main_module.__path__) + if ispkg + ] + + +def find_autorest_generated_folder(module_prefix="azure.mgmt"): + """Find all Autorest generated code in that module prefix. + + This actually looks for a "models" package only. We could be smarter if necessary. + """ + _LOGGER.info(f"Looking for Autorest generated package in {module_prefix}") + result = [] + try: + _LOGGER.debug(f"Try {module_prefix}") + importlib.import_module(".models", module_prefix) + # If not exception, we found it + _LOGGER.info(f"Found {module_prefix}") + result.append(module_prefix) + except ModuleNotFoundError: + # No model, might dig deeper + prefix_module = importlib.import_module(module_prefix) + for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."): + if ispkg: + result += find_autorest_generated_folder(sub_package) + return result + + +def main(prefix="azure.mgmt"): + packages = find_autorest_generated_folder(prefix) + for autorest_package in packages: + models_module = find_models_to_change(autorest_package) + solve_mro(models_module) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main(sys.argv[1] if len(sys.argv) >= 2 else "azure.mgmt") \ No newline at end of file From 96269a442809477572bd6a29e5887d3e1337c01b Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Sat, 20 Oct 2018 20:48:55 -0700 Subject: [PATCH 04/11] revert an unrelated change --- src/azure-cli/setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/azure-cli/setup.py b/src/azure-cli/setup.py index e95fd319539..90f01b7b1aa 100644 --- a/src/azure-cli/setup.py +++ b/src/azure-cli/setup.py @@ -99,9 +99,7 @@ 'azure-cli-signalr', 'azure-cli-sql', 'azure-cli-storage', - 'azure-cli-vm', - 'urllib3==1.23', - 'urllib3[secure]==1.23' + 'azure-cli-vm' ] with open('README.rst', 'r', encoding='utf-8') as f: From cf9f06223ecc133ab5b0c9a13e60518c6c18e403 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Mon, 22 Oct 2018 11:15:56 -0700 Subject: [PATCH 05/11] incorporate model file trimmer --- build_scripts/windows/azure-cli.wixproj | 4 +-- build_scripts/windows/scripts/build.cmd | 25 +++++++++---------- build_scripts/windows/scripts/build2.cmd | 19 ++++++++++++++ .../windows/scripts/patch_models_v2.py | 4 +++ 4 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 build_scripts/windows/scripts/build2.cmd diff --git a/build_scripts/windows/azure-cli.wixproj b/build_scripts/windows/azure-cli.wixproj index 09ff14ceae3..845c4d3d624 100644 --- a/build_scripts/windows/azure-cli.wixproj +++ b/build_scripts/windows/azure-cli.wixproj @@ -2,11 +2,11 @@ - $(HOMEDRIVE)$(HOMEPATH)\zwix\ + artifacts\wix\ $(LocalWixRoot) $(WixToolPath)Wix.targets wixtasks.dll - $(HOMEDRIVE)$(HOMEPATH)\zcli + artifacts\cli -fv diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 04ee9204513..1c661fb7004 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -9,7 +9,7 @@ if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) -set PYTHON_VERSION=3.6.1 +set PYTHON_VERSION=3.6.6 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" @@ -19,12 +19,17 @@ set OUTPUT_DIR=%~dp0..\out if exist %OUTPUT_DIR% rmdir /s /q %OUTPUT_DIR% mkdir %OUTPUT_DIR% -set TEMP_SCRATCH_FOLDER=%HOMEDRIVE%%HOMEPATH%\zcli_scratch -set BUILDING_DIR=%HOMEDRIVE%%HOMEPATH%\zcli -set WIX_DIR=%HOMEDRIVE%%HOMEPATH%\zwix +set ARTIFACTS_DIR=%~dp0..\artifacts +mkdir %ARTIFACTS_DIR% +set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch +set BUILDING_DIR=%ARTIFACTS_DIR%\cli +set WIX_DIR=%ARTIFACTS_DIR%\wix + set REPO_ROOT=%~dp0..\..\.. :: look for python 3.x so we can build into the installer +:: To prepare: install 32 bits Python 3.6.6+ at local and supply the folder path. To minimize the size, +:: turn off the option of "test" and "tcl tools" if not "%1"=="" ( set PYTHON_DIR=%1 set PYTHON_EXE=%1\python.exe @@ -39,7 +44,7 @@ FOR /f %%i IN ('where python') DO ( set PYTHON_DIR=!PY_FILE_DRIVE!!PY_FILE_PATH! FOR /F "delims=" %%j IN ('!PYTHON_EXE! --version') DO ( set PYTHON_VER=%%j - echo.!PYTHON_VER!|findstr /C:"%PYTHON_VERSION%" >nul 2>&1 + echo.!PYTHON_VER!|findstr /C:%PYTHON_VERSION% >nul 2>&1 if not errorlevel 1 ( goto PYTHON_FOUND ) @@ -145,12 +150,6 @@ for %%i in ( ) popd -rmdir /s /q %BUILDING_DIR%\Scripts -rmdir /s /q %BUILDING_DIR%\Scripts -:: for /f %%a in ('dir %BUILDING_DIR%\Lib\site-packages\*.egg-info /b /s /a:d') do ( -:: rmdir /s /q %%a -::) - :: Use universal files and remove Py3 only files pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt for /f %%a in ('dir /b /s *_py3.py') do ( @@ -167,8 +166,8 @@ popd pushd %BUILDING_DIR%\Lib\site-packages for /f %%f in ('dir /b /s *.pyc') do ( set PARENT_DIR=%%~df%%~pf.. - echo !PARENT_DIR! | findstr /C:"!BUILDING_DIR!\Lib\site-packages\pip" 1>nul - if errorlevel 1 ( + echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ 1>nul + if !errorlevel! neq 0 ( set FILENAME=%%~nf set BASE_FILENAME=!FILENAME:~0,-11! set pyc=!BASE_FILENAME!.pyc diff --git a/build_scripts/windows/scripts/build2.cmd b/build_scripts/windows/scripts/build2.cmd new file mode 100644 index 00000000000..97542d40aba --- /dev/null +++ b/build_scripts/windows/scripts/build2.cmd @@ -0,0 +1,19 @@ +@echo off +SetLocal EnableDelayedExpansion + +set ARTIFACTS_DIR=%~dp0..\artifacts + +set BUILDING_DIR=%ARTIFACTS_DIR%\cli +:: Remove .py and only deploy .pyc files +pushd %BUILDING_DIR%\Lib\site-packages +for /f %%f in ('dir /b /s *.pyc') do ( + set PARENT_DIR=%%~df%%~pf.. + echo !PARENT_DIR! + echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ + if !errorlevel! neq 0 ( + echo should trim + ) ELSE ( + echo --SKIP !PARENT_DIR! under pip + ) +) +popd diff --git a/build_scripts/windows/scripts/patch_models_v2.py b/build_scripts/windows/scripts/patch_models_v2.py index 837be604146..7ac6caa9bdb 100644 --- a/build_scripts/windows/scripts/patch_models_v2.py +++ b/build_scripts/windows/scripts/patch_models_v2.py @@ -1,3 +1,7 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- from enum import Enum import importlib import inspect From 666db120776c1fb782f2f56ac67608a939f6d3f5 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Mon, 22 Oct 2018 11:27:07 -0700 Subject: [PATCH 06/11] revert a few unnecessary change --- build_scripts/windows/scripts/build.cmd | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 1c661fb7004..0b82be130ba 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -98,7 +98,7 @@ robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL %BUILDING_DIR%\python.exe -m pip install wheel echo Building CLI packages... set CLI_SRC=%REPO_ROOT%\src -for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg) do ( +for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg %CLI_SRC%\azure-cli-telemetry) do ( pushd %%a %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %TEMP_SCRATCH_FOLDER% popd @@ -119,8 +119,8 @@ for %%i in (%TEMP_SCRATCH_FOLDER%\*.whl) do ( set ALL_MODULES=!ALL_MODULES! %%i ) echo All modules: %ALL_MODULES% -%BUILDING_DIR%\python.exe -m pip install --no-cache-dir %ALL_MODULES% -%BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg +%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --no-cache-dir %ALL_MODULES% +%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg pushd %BUILDING_DIR% %BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py @@ -137,11 +137,11 @@ copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING pushd %BUILDING_DIR% for %%i in ( Doc - include + include Scripts Tcl - Tools - libs + Tools + libs ) do ( if exist %%i ( echo Deleting %%i... @@ -167,7 +167,7 @@ pushd %BUILDING_DIR%\Lib\site-packages for /f %%f in ('dir /b /s *.pyc') do ( set PARENT_DIR=%%~df%%~pf.. echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ 1>nul - if !errorlevel! neq 0 ( + if !errorlevel! neq 0 ( set FILENAME=%%~nf set BASE_FILENAME=!FILENAME:~0,-11! set pyc=!BASE_FILENAME!.pyc From e8d588c56e551c36890dc8a5c8d62e51462773da Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Mon, 22 Oct 2018 13:48:33 -0700 Subject: [PATCH 07/11] lint fix --- build_scripts/windows/scripts/patch_models_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_scripts/windows/scripts/patch_models_v2.py b/build_scripts/windows/scripts/patch_models_v2.py index 7ac6caa9bdb..ce09f604385 100644 --- a/build_scripts/windows/scripts/patch_models_v2.py +++ b/build_scripts/windows/scripts/patch_models_v2.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- + from enum import Enum import importlib import inspect From 61861ae5de8fcaa2d87efc40c4ced0ba300d9582 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Tue, 23 Oct 2018 11:30:02 -0700 Subject: [PATCH 08/11] remove an irrelevant command file --- build_scripts/windows/scripts/build2.cmd | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 build_scripts/windows/scripts/build2.cmd diff --git a/build_scripts/windows/scripts/build2.cmd b/build_scripts/windows/scripts/build2.cmd deleted file mode 100644 index 97542d40aba..00000000000 --- a/build_scripts/windows/scripts/build2.cmd +++ /dev/null @@ -1,19 +0,0 @@ -@echo off -SetLocal EnableDelayedExpansion - -set ARTIFACTS_DIR=%~dp0..\artifacts - -set BUILDING_DIR=%ARTIFACTS_DIR%\cli -:: Remove .py and only deploy .pyc files -pushd %BUILDING_DIR%\Lib\site-packages -for /f %%f in ('dir /b /s *.pyc') do ( - set PARENT_DIR=%%~df%%~pf.. - echo !PARENT_DIR! - echo !PARENT_DIR! | findstr /C:\Lib\site-packages\pip\ - if !errorlevel! neq 0 ( - echo should trim - ) ELSE ( - echo --SKIP !PARENT_DIR! under pip - ) -) -popd From bace5df3827eb27aaf25c169ebedd44ea76e5045 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Tue, 23 Oct 2018 16:59:35 -0700 Subject: [PATCH 09/11] backup python to storage accounts --- build_scripts/windows/scripts/build.cmd | 47 ++++++++++--------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 0b82be130ba..d347b5ae147 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -12,6 +12,7 @@ if "%CLI_VERSION%"=="" ( set PYTHON_VERSION=3.6.6 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip" :: Set up the output directory and temp. directories echo Cleaning previous build artifacts... @@ -24,37 +25,10 @@ mkdir %ARTIFACTS_DIR% set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli set WIX_DIR=%ARTIFACTS_DIR%\wix +set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32 set REPO_ROOT=%~dp0..\..\.. -:: look for python 3.x so we can build into the installer -:: To prepare: install 32 bits Python 3.6.6+ at local and supply the folder path. To minimize the size, -:: turn off the option of "test" and "tcl tools" -if not "%1"=="" ( - set PYTHON_DIR=%1 - set PYTHON_EXE=%1\python.exe - goto PYTHON_FOUND -) - -FOR /f %%i IN ('where python') DO ( - set PY_FILE_DRIVE=%%~di - set PY_FILE_PATH=%%~pi - set PY_FILE_NAME=%%~ni - set PYTHON_EXE=!PY_FILE_DRIVE!!PY_FILE_PATH!!PY_FILE_NAME!.exe - set PYTHON_DIR=!PY_FILE_DRIVE!!PY_FILE_PATH! - FOR /F "delims=" %%j IN ('!PYTHON_EXE! --version') DO ( - set PYTHON_VER=%%j - echo.!PYTHON_VER!|findstr /C:%PYTHON_VERSION% >nul 2>&1 - if not errorlevel 1 ( - goto PYTHON_FOUND - ) - ) -) -echo python %PYTHON_VERSION% is needed to create installer. -exit /b 1 -:PYTHON_FOUND -echo Python Executables: %PYTHON_DIR%, %PYTHON_EXE% - ::reset working folders if exist %BUILDING_DIR% rmdir /s /q %BUILDING_DIR% ::rmdir always returns 0, so check folder's existence @@ -91,6 +65,23 @@ if not exist %WIX_DIR% ( popd ) +::ensure Python is available +if exist %PYTHON_DIR% ( + echo Using existing Python at %PYTHON_DIR% +) +if not exist %PYTHON_DIR% ( + mkdir %PYTHON_DIR% + pushd %PYTHON_DIR% + echo Downloading Python. + curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python366-32.zip + if %errorlevel% neq 0 goto ERROR + del Python366-32.zip + echo Python downloaded and extracted successfully. + popd +) +set PYTHON_EXE=%PYTHON_DIR%\python.exe + :: Use the Python version on the machine that creates the MSI robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL From 67371f3d30596a2759f4babf65f570d6901a84c5 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Tue, 23 Oct 2018 19:23:42 -0700 Subject: [PATCH 10/11] remove irrelevant comment --- build_scripts/windows/scripts/build.cmd | 1 - 1 file changed, 1 deletion(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index d347b5ae147..253a0e1a762 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -82,7 +82,6 @@ if not exist %PYTHON_DIR% ( ) set PYTHON_EXE=%PYTHON_DIR%\python.exe -:: Use the Python version on the machine that creates the MSI robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL :: Build & install all the packages with bdist_wheel From 2e6e89243fc76d9980e82437f7ba8bed14623ab5 Mon Sep 17 00:00:00 2001 From: yugangw-msft Date: Tue, 23 Oct 2018 20:51:07 -0700 Subject: [PATCH 11/11] simplify --- build_scripts/windows/scripts/build.cmd | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index 253a0e1a762..7234b1afad6 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -123,23 +123,6 @@ if %errorlevel% neq 0 goto ERROR copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% -: Delete some files we don't need -pushd %BUILDING_DIR% -for %%i in ( - Doc - include - Scripts - Tcl - Tools - libs -) do ( - if exist %%i ( - echo Deleting %%i... - rmdir /s /q %%i - ) -) -popd - :: Use universal files and remove Py3 only files pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt for /f %%a in ('dir /b /s *_py3.py') do (