From ee79fb9e64ba0175cb49c965afd86895fe56a485 Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Sun, 15 May 2022 23:25:31 +0530 Subject: [PATCH 1/3] Adding time based collection and support for installed-ir-path --- src/datamigration/HISTORY.rst | 5 +++ .../azext_datamigration/manual/_help.py | 6 ++++ .../azext_datamigration/manual/_params.py | 2 ++ .../azext_datamigration/manual/custom.py | 33 ++++++++++++++++--- .../azext_datamigration/manual/helper.py | 28 +++++++++++++--- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index 4ffc8853932..389e754f3b4 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +0.3.1 +++++++ +* [NEW PARAMETER] `az datamigration register-integration-runtime`: Added parameter `--installed-ir-path` to read the installed location of Microsoft Integration Runtime (SHIR) and use it for registering the Database Migration Service if command is unable to find the installed SHIR path. +* [NEW PARAMETER] `az datamigration performance-data-collection`: Added parameter `--time` to specify the amount of time(in seconds) performance data collection is to be done. After the timeout the process is terminated automatically. + 0.3.0 ++++++ * [BREAKING CHANGE] `az datamigration sql-managed-instance/sql-vm create`: Remove `--provisioing-error` and `--migration-operation-id` as they are unnecessary parameters. diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index a908fc50c5c..2fc784d1042 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -34,6 +34,9 @@ - name: Collect performance data of a given SQL Server using assessment config file. text: |- az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json" + - name: Collect performance data of a given SQL Server by specifying a time limit. If the time limit specified is before the complition of a iteration cycle, the process will end without saving the last cycle performance data. + text: |- + az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 --time 60 """ helps['datamigration get-sku-recommendation'] = """ @@ -58,6 +61,9 @@ - name: Install Integration Runtime and register a Sql Migration Service on it. text: |- az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --ir-path "C:\\Users\\user\\Downloads\\IntegrationRuntime.msi" + - name: Read the Integration Runtime from given installation location. + text: |- + az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --installed-ir-path "D:\\My Softwares\\Microsoft Integration Runtime\\5.0" """ helps['datamigration sql-managed-instance create'] = """ diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 44bf7c04634..561cb8392e0 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -35,6 +35,7 @@ def load_arguments(self, _): c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds.') c.argument('number_of_iteration', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. Minimum: 2.') c.argument('config_file_path', type=str, help='Path of the ConfigFile') + c.argument('time', type=int, help='Time after which the command execution automatically stops, in seconds. If this parameter is not specified manual intervention will be required to stop the command execution.') with self.argument_context('datamigration get-sku-recommendation') as c: c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') @@ -54,6 +55,7 @@ def load_arguments(self, _): with self.argument_context('datamigration register-integration-runtime') as c: c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service') c.argument('ir_path', type=str, help='Path of Integration Runtime MSI') + c.argument('installed_ir_path', type=str, help='Version folder path in the Integration Runtime installed location. This can be provided when IR is installed but the command is failing to read it. Format: "\\Microsoft Integration Runtime\\"') with self.argument_context('datamigration sql-db create') as c: c.argument('resource_group_name', resource_group_name_type) diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 07442e36a4d..729d3e622b8 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -12,6 +12,7 @@ # pylint: disable=line-too-long import os +import signal import subprocess from azure.cli.core.azclierror import MutuallyExclusiveArgumentError from azure.cli.core.azclierror import RequiredArgumentMissingError @@ -62,7 +63,8 @@ def datamigration_performance_data_collection(connection_string=None, perf_query_interval=30, static_query_interval=3600, number_of_iteration=20, - config_file_path=None): + config_file_path=None, + time=None): try: @@ -83,11 +85,31 @@ def datamigration_performance_data_collection(connection_string=None, for param in parameterList: if parameterList[param] is not None: cmd += f' {param} "{parameterList[param]}"' - subprocess.call(cmd, shell=False) + + if time is None: + subprocess.call(cmd, shell=False) + else: + sp = subprocess.Popen(cmd, shell=False) + try: + outs, errs = sp.communicate(timeout=time) + except subprocess.TimeoutExpired: + sp.send_signal(signal.SIGTERM) + outs, errs = sp.communicate() + elif config_file_path is not None: helper.validate_config_file_path(config_file_path, "perfdatacollection") cmd = f'{exePath} --configFile "{config_file_path}"' - subprocess.call(cmd, shell=False) + + if time is None: + subprocess.call(cmd, shell=False) + else: + sp = subprocess.Popen(cmd, shell=False) + try: + outs, errs = sp.communicate(timeout=time) + except subprocess.TimeoutExpired: + sp.send_signal(signal.SIGTERM) + outs, errs = sp.communicate() + else: raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: sql_connection_string, config_file_path') @@ -162,7 +184,8 @@ def datamigration_get_sku_recommendation(output_folder=None, # Register Sql Migration Service on IR command Implementation. # ----------------------------------------------------------------------------------------------------------------- def datamigration_register_ir(auth_key, - ir_path=None): + ir_path=None, + installed_ir_path=None): helper.validate_os_env() @@ -172,4 +195,4 @@ def datamigration_register_ir(auth_key, if ir_path is not None: helper.install_gateway(ir_path) - helper.register_ir(auth_key) + helper.register_ir(auth_key, installed_ir_path) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index ba953b9343b..e00362551d3 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -41,7 +41,7 @@ def validate_config_file_path(path, action): if not os.path.exists(path): raise InvalidArgumentValueError(f'Invalid config file path: {path}. Please provide a valid config file path.') - # JSON file + # JSON file read with open(path, "r", encoding=None) as f: configJson = json.loads(f.read()) try: @@ -206,10 +206,13 @@ def install_gateway(path): # ----------------------------------------------------------------------------------------------------------------- # Helper function to register Sql Migration Service on IR # ----------------------------------------------------------------------------------------------------------------- -def register_ir(key): +def register_ir(key, installed_ir_path=None): print(f"Start to register IR with key: {key}") - cmdFilePath = get_cmd_file_path() + if installed_ir_path is None: + cmdFilePath = get_cmd_file_path() + else: + cmdFilePath = get_cmd_file_path_from_input(installed_ir_path) directoryPath = os.path.dirname(cmdFilePath) parentDirPath = os.path.dirname(directoryPath) @@ -244,7 +247,7 @@ def get_cmd_file_path(): diaCmdPath = get_cmd_file_path_static() return diaCmdPath except FileNotFoundError as e: - raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command") from e + raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command or use --installed-ir-path parameter to provide the installed IR location") from e except IndexError as e: raise FileOperationError("IR is not properly installed. Please re-install it and re-run this command") from e @@ -274,3 +277,20 @@ def get_cmd_file_path_static(): raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") return diaCmdPath + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to get DiaCmdPath using the installed IR path user has given +# ----------------------------------------------------------------------------------------------------------------- +def get_cmd_file_path_from_input(installed_ir_path): + + if not os.path.exists(installed_ir_path): + raise FileNotFoundError(f"The system cannot find the path specified: {installed_ir_path}") + + # Create diaCmd default path and check if it is valid or not. + diaCmdPath = os.path.join(installed_ir_path, "Shared", "diacmd.exe") + + if not os.path.exists(diaCmdPath): + raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") + + return diaCmdPath From c2644abe8f90e5e982d9324281a0fc519e3266e3 Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Mon, 16 May 2022 02:01:03 +0530 Subject: [PATCH 2/3] Adding comments to custom commands --- .../azext_datamigration/manual/action.py | 2 + .../azext_datamigration/manual/custom.py | 37 +++++++++++++++++++ .../azext_datamigration/manual/helper.py | 20 ++++++++-- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/action.py b/src/datamigration/azext_datamigration/manual/action.py index 7d4ce4a232c..b9d2a1718f2 100644 --- a/src/datamigration/azext_datamigration/manual/action.py +++ b/src/datamigration/azext_datamigration/manual/action.py @@ -22,6 +22,8 @@ from knack.util import CLIError +# Adding new input type for target connection details +# As having type as AddSourceSqlConnection overwrites one of the parameters class AddTargetSqlConnection(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): action = self.get_action(values, option_string) diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 729d3e622b8..f7575bd94fd 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -30,20 +30,31 @@ def datamigration_assessment(connection_string=None, try: + # Setup the console app defaultOutputFolder, exePath = helper.console_app_setup() + # Specifying both parameters is an error if connection_string is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Connection string. if connection_string is not None: + + # Formating for multiple connection string connection_string = " ".join(f"\"{i}\"" for i in connection_string) + + # Joining parameters in a string cmd = f'{exePath} Assess --sqlConnectionStrings {connection_string} ' if output_folder is None else f'{exePath} Assess --sqlConnectionStrings {connection_string} --outputFolder "{output_folder}" ' cmd += '--overwrite False' if overwrite is False else '' subprocess.call(cmd, shell=False) + + # When config file. elif config_file_path is not None: helper.validate_config_file_path(config_file_path, "assess") cmd = f'{exePath} --configFile "{config_file_path}"' subprocess.call(cmd, shell=False) + + # if no parameter is provided. else: raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: connection_string, config_file_path') @@ -68,24 +79,33 @@ def datamigration_performance_data_collection(connection_string=None, try: + # Setup the console app defaultOutputFolder, exePath = helper.console_app_setup() if connection_string is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Connection string. if connection_string is not None: + + # Formating for multiple connection string connection_string = " ".join(f"\"{i}\"" for i in connection_string) + + # parameter set for Perfornace data collection parameterList = { "--outputFolder": output_folder, "--perfQueryIntervalInSec": perf_query_interval, "--staticQueryIntervalInSec": static_query_interval, "--numberOfIterations": number_of_iteration } + + # joining paramaters together in a string cmd = f'{exePath} PerfDataCollection --sqlConnectionStrings {connection_string}' for param in parameterList: if parameterList[param] is not None: cmd += f' {param} "{parameterList[param]}"' + # If time parameter is specified, catch TimeoutExpired exception and terminate the process if time is None: subprocess.call(cmd, shell=False) else: @@ -96,10 +116,12 @@ def datamigration_performance_data_collection(connection_string=None, sp.send_signal(signal.SIGTERM) outs, errs = sp.communicate() + # When Config file. elif config_file_path is not None: helper.validate_config_file_path(config_file_path, "perfdatacollection") cmd = f'{exePath} --configFile "{config_file_path}"' + # If time parameter is specified, catch TimeoutExpired exception and terminate the process if time is None: subprocess.call(cmd, shell=False) else: @@ -139,16 +161,23 @@ def datamigration_get_sku_recommendation(output_folder=None, config_file_path=None): try: + + # Setup Console app defaultOutputFolder, exePath = helper.console_app_setup() if output_folder is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both output_folder and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Config file - Handling this case first to allow no parameter to be specified (runs non config file scenario) if config_file_path is not None: helper.validate_config_file_path(config_file_path, "getskurecommendation") cmd = f'{exePath} --configFile "{config_file_path}"' subprocess.call(cmd, shell=False) + + # When non-config file else: + + # parameter set for Sku recommendation parameterList = { "--outputFolder": output_folder, "--targetPlatform": target_platform, @@ -164,9 +193,13 @@ def datamigration_get_sku_recommendation(output_folder=None, "--databaseDenyList": database_deny_list } cmd = f'{exePath} GetSkuRecommendation' + + # formating the parameter list into a string for param in parameterList: if parameterList[param] is not None and not param.__contains__("List"): cmd += f' {param} "{parameterList[param]}"' + + # in case the parameter input is list format it accordingly elif param.__contains__("List") and parameterList[param] is not None: parameterList[param] = " ".join(f"\"{i}\"" for i in parameterList[param]) cmd += f' {param} {parameterList[param]}' @@ -189,10 +222,14 @@ def datamigration_register_ir(auth_key, helper.validate_os_env() + # This command can only be run as admin and in windows if not helper.is_user_admin(): raise UnclassifiedUserFault("Failed: You do not have Administrator rights to run this command. Please re-run this command as an Administrator!") helper.validate_input(auth_key) + + # Run installation if ir_path is provided if ir_path is not None: helper.install_gateway(ir_path) + # register or re-register Dms on ir helper.register_ir(auth_key, installed_ir_path) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index e00362551d3..ba9c33dbd54 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -41,7 +41,7 @@ def validate_config_file_path(path, action): if not os.path.exists(path): raise InvalidArgumentValueError(f'Invalid config file path: {path}. Please provide a valid config file path.') - # JSON file read + # JSON file read and validation of value in action with open(path, "r", encoding=None) as f: configJson = json.loads(f.read()) try: @@ -159,6 +159,7 @@ def check_whether_gateway_installed(name): # Get the path of Installed softwares accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") + # Check if any software has name as given name for i in range(0, winreg.QueryInfoKey(accessKey)[0]): installedSoftware = winreg.EnumKey(accessKey, i) installedSoftwareKey = winreg.OpenKey(accessKey, installedSoftware) @@ -169,7 +170,7 @@ def check_whether_gateway_installed(name): except FileNotFoundError: pass - # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) + # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) to tackle x32 bit python try: diaCmdPath = get_cmd_file_path_static() if os.path.exists(diaCmdPath): @@ -185,17 +186,21 @@ def check_whether_gateway_installed(name): # ----------------------------------------------------------------------------------------------------------------- def install_gateway(path): + # check if gateway is installaed. If yes don't do the installation again if check_whether_gateway_installed("Microsoft Integration Runtime"): print("Microsoft Integration Runtime is already installed") return + # validate path of IR MSI validate_ir_extension(path) + # Check for IR path existance if not os.path.exists(path): raise InvalidArgumentValueError(f"Invalid Integration Runtime MSI path : {path}. Please provide a valid Integration Runtime MSI path") print("Start Integration Runtime installation") + # Installed MSI installCmd = f'msiexec.exe /i "{path}" /quiet /passive' subprocess.call(installCmd, shell=False) time.sleep(30) @@ -209,18 +214,23 @@ def install_gateway(path): def register_ir(key, installed_ir_path=None): print(f"Start to register IR with key: {key}") + # get SHIR installation location - using registry or user provided if installed_ir_path is None: cmdFilePath = get_cmd_file_path() else: cmdFilePath = get_cmd_file_path_from_input(installed_ir_path) + # extract the dmgcmd.exe and RegisterIntegrationRuntime.ps1 Script path. directoryPath = os.path.dirname(cmdFilePath) parentDirPath = os.path.dirname(directoryPath) dmgCmdPath = os.path.join(directoryPath, "dmgcmd.exe") regIRScriptPath = os.path.join(parentDirPath, "PowerShellScript", "RegisterIntegrationRuntime.ps1") + # Open Intranet Port (Necessary for Re-Register. Service has to be running for Re-Register to work.) portCmd = f'{dmgCmdPath} -EnableRemoteAccess 8060' + + # Register/ Re-register IR irCmd = f'powershell -command "& \'{regIRScriptPath}\' -gatewayKey {key}"' subprocess.call(portCmd, shell=False) @@ -228,7 +238,7 @@ def register_ir(key, installed_ir_path=None): # ----------------------------------------------------------------------------------------------------------------- -# Helper function to get SHIR script path +# Helper function to get SHIR script path from windows registry # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path(): @@ -242,6 +252,8 @@ def get_cmd_file_path(): accessValue = winreg.QueryValueEx(accessKey, r"DiacmdPath")[0] return accessValue + + # Handling the case for x32 Python as x64 software like SHIR in registry are not found by x32 Python. except FileNotFoundError: try: diaCmdPath = get_cmd_file_path_static() @@ -253,7 +265,7 @@ def get_cmd_file_path(): # ----------------------------------------------------------------------------------------------------------------- -# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files +# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files. This function is for handling the case with x32 Python # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path_static(): From e0f8fb41769d362784547c8cc6df553d3c5398f8 Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Mon, 16 May 2022 18:20:53 +0530 Subject: [PATCH 3/3] Updating help and version --- src/datamigration/README.md | 2 +- src/datamigration/azext_datamigration/manual/_help.py | 8 +++++++- src/datamigration/setup.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/datamigration/README.md b/src/datamigration/README.md index 16ba265f4df..c2ebcd6efc3 100644 --- a/src/datamigration/README.md +++ b/src/datamigration/README.md @@ -26,7 +26,7 @@ az datamigration performance-data-collection --connection-string "Data Source=La ##### Get-sku-recommendation ##### ``` -az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite +az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks AdventureWorks2 --display-result --overwrite ``` #### datamigration sql-managed-instance #### diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index 2fc784d1042..9af90b51197 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -22,6 +22,9 @@ - name: Run SQL Assessment on given SQL Server using assessment config file. text: |- az datamigration get-assessment --config-file-path "C:\\Users\\user\\document\\config.json" + - name: Run SQL Assessment on multiple SQL Servers in one call using connection string. + text: |- + az datamigration get-assessment --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\AssessmentOutput" --overwrite """ helps['datamigration performance-data-collection'] = """ @@ -31,6 +34,9 @@ - name: Collect performance data of a given SQL Server using connection string. text: |- az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 + - name: Collect performance data of multiple SQL Servers in one call using connection string. + text: |- + az datamigration performance-data-collection --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 - name: Collect performance data of a given SQL Server using assessment config file. text: |- az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json" @@ -45,7 +51,7 @@ examples: - name: Get SKU recommendation for given SQL Server using command line. text: |- - az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite + az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks1 AdventureWorks2 --display-result --overwrite - name: Get SKU recommendation for given SQL Server using assessment config file. text: |- az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json" diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index 5a090250d28..315bfa127af 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '0.3.0' +VERSION = '0.3.1' try: from azext_datamigration.manual.version import VERSION except ImportError: