From abdff816363941ab34fae46bca4a4592b5115dd8 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:02:43 +0800 Subject: [PATCH 01/14] [update] using user_input to replace raw_inuput or input --- pkgsdb.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkgsdb.py b/pkgsdb.py index b556b881..a6db58d1 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -27,8 +27,7 @@ import sqlite3 import os import hashlib -import sys - +from cmds.cmd_package.cmd_package_utils import user_input from vars import Import SHOW_SQL = False @@ -171,10 +170,7 @@ def remove_unchanged_file(pathname, dbfilename, dbsqlname): print('If you choose to keep the changed file,you should copy the file to another folder. ' '\nbecaues it may be covered by the next update.') - if sys.version_info < (3, 0): - rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') - else: - rc = input('Press the Y Key to delete the file or just press Enter to keep the file.') + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" conn.commit() From da7fe3c0d3a840108820a45c2239c49db01c7573 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:04:08 +0800 Subject: [PATCH 02/14] [update] reduce some unnecessary log --- cmds/cmd_package/cmd_package_update.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 07ac4f1f..6f810329 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -112,7 +112,8 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - print('Error message:%s\t' % e) + # print('Error message:%s\t' % e) + # So much error message should be ignore print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -375,7 +376,7 @@ def update_latest_packages(sys_value): result = False except Exception as e: - print("Error message : %s" % e) + # print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") if not right_path_flag: From 28f72189272a96cd2394cdf696b1194fe92817ef Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:08:12 +0800 Subject: [PATCH 03/14] [optimize] code clear up --- cmds/cmd_menuconfig.py | 2 +- cmds/cmd_package/cmd_package_update.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 541278ec..d1eb2dea 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -35,7 +35,7 @@ def is_pkg_special_config(config_str): """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER""" - if type(config_str) == type('a'): + if isinstance(config_str, str): if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')): return True return False diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 6f810329..a2752f20 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -34,7 +34,8 @@ import requests from package import Package, Bridge_SConscript from vars import Import, Export -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, find_macro_in_config +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ + find_macro_in_config def determine_support_chinese(env_root): @@ -98,6 +99,7 @@ def determine_url_valid(url_from_srv): 'Accept': '*/*', 'User-Agent': 'curl/7.54.0'} + # noinspection PyBroadException try: for i in range(0, 3): r = requests.get(url_from_srv, stream=True, headers=headers) @@ -346,6 +348,7 @@ def update_latest_packages(sys_value): repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) + # noinspection PyBroadException try: # If mirror acceleration is enabled, get the update address from the mirror server. if (not os.path.isfile(env_config_file)) or \ From f8555deeb530e7be42130933512a5c12ec27b481 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 14:07:08 +0800 Subject: [PATCH 04/14] [optimize] code clear up and add logging system --- env.py | 62 +++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/env.py b/env.py index c122eb9d..9592964d 100644 --- a/env.py +++ b/env.py @@ -27,6 +27,8 @@ import os import sys import argparse +import logging +import time import platform from cmds import * @@ -39,8 +41,7 @@ def init_argparse(): parser = argparse.ArgumentParser(description=__doc__) subs = parser.add_subparsers() - parser.add_argument('-v', '--version', - action='version', version=__version__) + parser.add_argument('-v', '--version', action='version', version=__version__) cmd_system.add_parser(subs) cmd_menuconfig.add_parser(subs) @@ -49,9 +50,21 @@ def init_argparse(): return parser -def main(): - bsp_root = os.getcwd() - script_root = os.path.split(os.path.realpath(__file__))[0] +def init_logger(env_root): + localtime = time.asctime(time.localtime(time.time())) + log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-")) + os.makedirs(log_path) + log_name = os.path.join(log_path, "running_log.txt") + + log_format = "%(asctime)s %(name)s %(levelname)s %(pathname)s %(lineno)d %(message)s " + date_format = '%Y-%m-%d %H:%M:%S %a ' + logging.basicConfig(level=logging.DEBUG, + format=log_format, + datefmt=date_format, + filename=log_name) + + +def get_env_root(): env_root = os.getenv("ENV_ROOT") if env_root is None: if platform.system() != 'Windows': @@ -59,15 +72,18 @@ def main(): else: env_root = os.path.join(os.getenv('USERPROFILE'), '.env') - sys.path = sys.path + [os.path.join(script_root)] + return env_root - pkgs_root = os.getenv("PKGS_ROOT") - if pkgs_root is None: - pkgs_root = os.path.join(env_root, 'packages') - Export('env_root') - Export('bsp_root') - Export('pkgs_root') +def get_package_root(env_root): + package_root = os.getenv("PKGS_ROOT") + if package_root is None: + package_root = os.path.join(env_root, 'packages') + return package_root + + +def get_bsp_root(): + bsp_root = os.getcwd() # noinspection PyBroadException try: @@ -84,7 +100,26 @@ def main(): if platform.system() == "Windows": os.system('chcp 437 > nul') - return False + exit(1) + + return bsp_root + + +def export_environment_variable(): + script_root = os.path.split(os.path.realpath(__file__))[0] + sys.path = sys.path + [os.path.join(script_root)] + bsp_root = get_bsp_root() + env_root = get_env_root() + pkgs_root = get_package_root(env_root) + + Export('env_root') + Export('bsp_root') + Export('pkgs_root') + + +def main(): + export_environment_variable() + init_logger(get_env_root()) parser = init_argparse() args = parser.parse_args() @@ -93,4 +128,3 @@ def main(): if __name__ == '__main__': main() - From f6a7f5cea06da7685363c3a8cbfb9723d5a6ab36 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 14:10:09 +0800 Subject: [PATCH 05/14] [optimize] add need_using_mirror_download function to simplify code --- cmds/cmd_package/cmd_package_update.py | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index a2752f20..9d497faf 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -128,6 +128,16 @@ def is_user_mange_package(bsp_package_path, pkg): return False +def need_using_mirror_download(config_file): + """default using mirror url to download packages""" + + if not os.path.isfile(config_file): + return True + elif os.path.isfile(config_file) and find_macro_in_config(config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE'): + return True + + +# noinspection PyUnboundLocalVariable def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" @@ -167,9 +177,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): upstream_change_flag = False try: - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # Check whether the package package url is valid @@ -208,18 +216,13 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): submodule_path = os.path.join(repo_path, '.gitmodules') if os.path.isfile(submodule_path): print("Start to update submodule") - if (not os.path.isfile(env_config_file)) \ - or (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file cmd = 'git submodule update --init --recursive' execute_command(cmd, cwd=repo_path) - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and - find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - + if need_using_mirror_download(env_config_file): if len(replace_list): for item in replace_list: submod_dir_path = os.path.join(repo_path, item[2]) @@ -227,10 +230,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): cmd = 'git remote set-url origin ' + item[0] execute_command(cmd, cwd=submod_dir_path) - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - + if need_using_mirror_download(env_config_file): if os.path.isfile(submodule_path): cmd = 'git checkout .gitmodules' execute_command(cmd, cwd=repo_path) From 31d9106c7c49aeab4396a1537799c939d0bacb6c Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 15:11:17 +0800 Subject: [PATCH 06/14] [update] add some basic logging and optimize code --- cmds/cmd_package/cmd_package_update.py | 94 ++++++++++++++++---------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 9d497faf..500d3088 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -32,6 +32,7 @@ import time import archive import requests +import logging from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ @@ -181,10 +182,10 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # Check whether the package package url is valid - if get_package_url is not None and determine_url_valid(get_package_url): + if get_package_url and determine_url_valid(get_package_url): package_url = get_package_url - if get_ver_sha is not None: + if get_ver_sha: ver_sha = get_ver_sha upstream_change_flag = True @@ -318,6 +319,8 @@ def update_latest_packages(sys_value): message provided by git. """ + logging.info("Begin to update latest version packages") + result = True package_filename = sys_value[3] @@ -428,6 +431,7 @@ def get_git_root_path(repo_path): def pre_package_update(): """ Make preparations before updating the software package. """ + logging.info("Begin prepare package update") bsp_root = Import('bsp_root') env_root = Import('env_root') @@ -559,6 +563,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file def rm_package(dir_remove): + logging.info("remove dir: {0}".format(dir_remove)) + if platform.system() != "Windows": shutil.rmtree(dir_remove) else: @@ -605,15 +611,17 @@ def handle_download_error_packages(sys_value, force_update): Check to see if the packages stored in the Json file list actually exist, and then download the packages if they don't exist. """ + + logging.info("begin to handel download error packages") package_filename = sys_value[3] bsp_packages_path = sys_value[5] with open(package_filename, 'r') as f: - read_back_pkgs_json = json.load(f) + package_json = json.load(f) error_packages_list = [] - for pkg in read_back_pkgs_json: + for pkg in package_json: remove_path = get_package_remove_path(pkg, bsp_packages_path) if os.path.exists(remove_path): continue @@ -622,12 +630,13 @@ def handle_download_error_packages(sys_value, force_update): error_packages_list.append(pkg) # Handle the failed download packages - get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update) + get_flag = error_packages_handle(error_packages_list, package_json, package_filename, force_update) return get_flag def delete_useless_packages(sys_value): + logging.info("Begin to delete useless packages") package_delete_error_list = sys_value[2] bsp_packages_path = sys_value[5] @@ -646,7 +655,48 @@ def delete_useless_packages(sys_value): return True +def is_git_package(pkg, bsp_packages_path): + remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) + remove_path_git = os.path.join(remove_path_with_version, '.git') + return os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git) + + +def delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list): + git_folder_to_remove = remove_path_with_version + + print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) + if force_update: + logging.info("package force update, Begin to remove package {0}".format(git_folder_to_remove)) + if not rm_package(git_folder_to_remove): + print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) + print("Please delete this folder manually.") + else: + print("The folder is managed by git. Do you want to delete this folder?\n") + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + try: + if not rm_package(git_folder_to_remove): + package_delete_fail_list.append(pkg) + print("Error: Please delete the folder manually.") + except Exception as e: + print('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + + +def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname): + if os.path.isdir(remove_path_with_version): + print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) + try: + pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) + except Exception as e: + package_delete_fail_list.append(pkg) + print('Error message:\n%s %s. %s \n\t' % ( + "Delete folder failed, please delete the folder manually", + remove_path_with_version.encode("utf-8"), e)) + + def remove_packages(sys_value, force_update): + logging.info("Begin to remove packages") old_package = sys_value[0] new_package = sys_value[1] package_error_list_filename = sys_value[4] @@ -658,38 +708,12 @@ def remove_packages(sys_value, force_update): for pkg in case_delete: remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) - remove_path_git = os.path.join(remove_path_with_version, '.git') # delete .git directory - if os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git): - git_folder_to_remove = remove_path_with_version - - print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) - if force_update: - if not rm_package(git_folder_to_remove): - print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) - print("Please delete this folder manually.") - else: - print("The folder is managed by git. Do you want to delete this folder?\n") - rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - if rc == 'y' or rc == 'Y': - try: - if not rm_package(git_folder_to_remove): - package_delete_fail_list.append(pkg) - print("Error: Please delete the folder manually.") - except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + if is_git_package(pkg, bsp_packages_path): + delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list) else: - if os.path.isdir(remove_path_with_version): - print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) - try: - pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) - except Exception as e: - package_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", - remove_path_with_version.encode("utf-8"), e)) + delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname) # write error messages with open(package_error_list_filename, 'w') as f: @@ -707,6 +731,8 @@ def install_packages(sys_value, force_update): and then download again when the update command is executed. """ + logging.info("Begin to install packages") + old_package = sys_value[0] new_package = sys_value[1] package_filename = sys_value[3] From 176966b8a0b04937537a8e71accd793b94132ab5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:00:40 +0800 Subject: [PATCH 07/14] [add] is_git_url function --- cmds/cmd_package/cmd_package_update.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 500d3088..8bd890e6 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -138,6 +138,10 @@ def need_using_mirror_download(config_file): return True +def is_git_url(package_url): + return package_url.endswith('.git') + + # noinspection PyUnboundLocalVariable def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" @@ -165,14 +169,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): package_url = package.get_url(pkg['ver']) pkgs_name_in_json = package.get_name() - # print("==================================================>") - # print("packages name :", pkgs_name_in_json.encode("utf-8")) - # print("ver :", pkg['ver']) - # print("url :", package_url.encode("utf-8")) - # print("url_from_json : ", url_from_json.encode("utf-8")) - # print("==================================================>") - - if package_url[-4:] == '.git': + logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) + if is_git_url(package_url): ver_sha = package.get_versha(pkg['ver']) upstream_change_flag = False @@ -193,7 +191,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - if package_url.endswith('.git'): + if is_git_url(package_url): try: repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] From ad28b198fdec174d5f38e0da7670d1daa0b39950 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:11:23 +0800 Subject: [PATCH 08/14] [optimize] code clear up --- cmds/cmd_package/cmd_package_update.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 8bd890e6..c10c140b 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -173,7 +173,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if is_git_url(package_url): ver_sha = package.get_versha(pkg['ver']) - upstream_change_flag = False + upstream_changed = False try: if need_using_mirror_download(env_config_file): @@ -186,7 +186,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if get_ver_sha: ver_sha = get_ver_sha - upstream_change_flag = True + upstream_changed = True except Exception as e: print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") @@ -207,7 +207,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("\nFailed to download software package with git. Please check the network connection.") return False - if upstream_change_flag: + # change upstream to origin url + if upstream_changed: cmd = 'git remote set-url origin ' + url_from_json execute_command(cmd, cwd=repo_path) From ab9c945866c5bc31d4e6cde928e5bee2c3f80328 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:18:43 +0800 Subject: [PATCH 09/14] [update] change default log level --- env.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/env.py b/env.py index 9592964d..06eb18a5 100644 --- a/env.py +++ b/env.py @@ -56,12 +56,13 @@ def init_logger(env_root): os.makedirs(log_path) log_name = os.path.join(log_path, "running_log.txt") - log_format = "%(asctime)s %(name)s %(levelname)s %(pathname)s %(lineno)d %(message)s " + log_format = "%(pathname)s %(lineno)d %(message)s " date_format = '%Y-%m-%d %H:%M:%S %a ' - logging.basicConfig(level=logging.DEBUG, + logging.basicConfig(level=logging.WARNING, format=log_format, datefmt=date_format, - filename=log_name) + # filename=log_name + ) def get_env_root(): From 2dd4b05b2dee587639094cdcd5c885d816665fba Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:25:42 +0800 Subject: [PATCH 10/14] [optimize] install_pkg function --- cmds/cmd_package/cmd_package_update.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index c10c140b..5a863380 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -143,7 +143,7 @@ def is_git_url(package_url): # noinspection PyUnboundLocalVariable -def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): +def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): """Install the required packages.""" ret = True @@ -151,33 +151,32 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: - if is_user_mange_package(bsp_package_path, pkg): + if is_user_mange_package(bsp_package_path, package_info): return ret # get the .config file from env - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') package = Package() - pkg_path = pkg['path'] + pkg_path = package_info['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') package.parse(pkg_path) - url_from_json = package.get_url(pkg['ver']) - package_url = package.get_url(pkg['ver']) + url_from_json = package.get_url(package_info['ver']) + package_url = package.get_url(package_info['ver']) pkgs_name_in_json = package.get_name() logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) if is_git_url(package_url): - ver_sha = package.get_versha(pkg['ver']) + ver_sha = package.get_versha(package_info['ver']) upstream_changed = False try: if need_using_mirror_download(env_config_file): - get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) + get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, package_info['ver']) # Check whether the package package url is valid if get_package_url and determine_url_valid(get_package_url): @@ -194,10 +193,11 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if is_git_url(package_url): try: repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) - repo_path = repo_path + '-' + pkg['ver'] + repo_path = repo_path + '-' + package_info['ver'] repo_path_full = '"' + repo_path + '"' clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + logging.info(clone_cmd) execute_command(clone_cmd, cwd=bsp_package_path) git_check_cmd = 'git checkout -q ' + ver_sha @@ -236,12 +236,12 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): execute_command(cmd, cwd=repo_path) else: # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path, package_url): + if not package.download(package_info['ver'], local_pkgs_path, package_url): return False - pkg_dir = package.get_filename(pkg['ver']) + pkg_dir = package.get_filename(package_info['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) if not archive.package_integrity_test(package_path): print("package : %s is invalid" % package_path.encode("utf-8")) @@ -250,7 +250,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): # unpack package if not os.path.exists(pkg_dir): try: - if not package.unpack(package_path, bsp_package_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): ret = False except Exception as e: os.remove(package_path) @@ -742,14 +742,14 @@ def install_packages(sys_value, force_update): case_download = sub_list(new_package, old_package) packages_download_fail_list = [] - for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): + for package in case_download: + if install_pkg(env_root, pkgs_root, bsp_root, package, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( - pkg['name'], pkg['ver'])) + package['name'], package['ver'])) else: # if package download fails, record it in the packages_download_fail_list - packages_download_fail_list.append(pkg) - print(pkg, 'download failed.') + packages_download_fail_list.append(package) + print(package, 'download failed.') return False # Get the currently updated configuration. From 94b1f79be316ebed65db0bde23263f1dd5e01783 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:48:59 +0800 Subject: [PATCH 11/14] [optimize] extract git_package_install and non_git_package_install process --- cmds/cmd_package/cmd_package_update.py | 186 ++++++++++++++----------- 1 file changed, 101 insertions(+), 85 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 5a863380..9d85185e 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -91,7 +91,7 @@ def modify_submod_file_to_mirror(submodule_path): return replace_list except Exception as e: - print('Error message:%s\t' % e) + logging.warning('Error message:%s\t' % e) def determine_url_valid(url_from_srv): @@ -115,9 +115,8 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - # print('Error message:%s\t' % e) # So much error message should be ignore - print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) + logging.error('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) def is_user_mange_package(bsp_package_path, pkg): @@ -142,17 +141,94 @@ def is_git_url(package_url): return package_url.endswith('.git') +def install_git_package(bsp_package_path, package_name, package_info, package_url, ver_sha, upstream_changed, + url_origin, env_config_file): + try: + repo_path = os.path.join(bsp_package_path, package_name) + repo_path = repo_path + '-' + package_info['ver'] + repo_path_full = '"' + repo_path + '"' + + clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + logging.info(clone_cmd) + execute_command(clone_cmd, cwd=bsp_package_path) + + git_check_cmd = 'git checkout -q ' + ver_sha + execute_command(git_check_cmd, cwd=repo_path) + except Exception as e: + print('Error message:%s' % e) + print("\nFailed to download software package with git. Please check the network connection.") + return False + + # change upstream to origin url + if upstream_changed: + cmd = 'git remote set-url origin ' + url_origin + execute_command(cmd, cwd=repo_path) + + # If there is a .gitmodules file in the package, prepare to update submodule. + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): + print("Start to update submodule") + if need_using_mirror_download(env_config_file): + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file + + cmd = 'git submodule update --init --recursive' + execute_command(cmd, cwd=repo_path) + + if need_using_mirror_download(env_config_file): + if len(replace_list): + for item in replace_list: + submod_dir_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submod_dir_path): + cmd = 'git remote set-url origin ' + item[0] + execute_command(cmd, cwd=submod_dir_path) + + if need_using_mirror_download(env_config_file): + if os.path.isfile(submodule_path): + cmd = 'git checkout .gitmodules' + execute_command(cmd, cwd=repo_path) + + return True + + +def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json): + # Download a package of compressed package type. + if not package.download(package_info['ver'], local_pkgs_path, package_url): + return False + + pkg_dir = package.get_filename(package_info['ver']) + pkg_dir = os.path.splitext(pkg_dir)[0] + package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) + + if not archive.package_integrity_test(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) + return False + + # unpack package + if not os.path.exists(pkg_dir): + try: + if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): + result = False + except Exception as e: + os.remove(package_path) + result = False + print('Error message: %s\t' % e) + else: + print("The file does not exist.") + + return True + + # noinspection PyUnboundLocalVariable -def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): +def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): """Install the required packages.""" - ret = True + result = True local_pkgs_path = os.path.join(env_root, 'local_pkgs') bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: if is_user_mange_package(bsp_package_path, package_info): - return ret + return result # get the .config file from env env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') @@ -174,6 +250,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): upstream_changed = False + # noinspection PyBroadException try: if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, package_info['ver']) @@ -187,78 +264,18 @@ def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): upstream_changed = True except Exception as e: - print('Error message:%s\t' % e) - print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + logging.warning("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") if is_git_url(package_url): - try: - repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) - repo_path = repo_path + '-' + package_info['ver'] - repo_path_full = '"' + repo_path + '"' - - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - logging.info(clone_cmd) - execute_command(clone_cmd, cwd=bsp_package_path) - - git_check_cmd = 'git checkout -q ' + ver_sha - execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: - print('Error message:%s' % e) - print("\nFailed to download software package with git. Please check the network connection.") - return False - - # change upstream to origin url - if upstream_changed: - cmd = 'git remote set-url origin ' + url_from_json - execute_command(cmd, cwd=repo_path) - - # If there is a .gitmodules file in the package, prepare to update submodule. - submodule_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submodule_path): - print("Start to update submodule") - if need_using_mirror_download(env_config_file): - replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file - - cmd = 'git submodule update --init --recursive' - execute_command(cmd, cwd=repo_path) - - if need_using_mirror_download(env_config_file): - if len(replace_list): - for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): - cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) - - if need_using_mirror_download(env_config_file): - if os.path.isfile(submodule_path): - cmd = 'git checkout .gitmodules' - execute_command(cmd, cwd=repo_path) + if not install_git_package(bsp_package_path, pkgs_name_in_json, package_info, package_url, ver_sha, + upstream_changed, + url_from_json, env_config_file): + result = False else: - # Download a package of compressed package type. - if not package.download(package_info['ver'], local_pkgs_path, package_url): - return False - - pkg_dir = package.get_filename(package_info['ver']) - pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) - - if not archive.package_integrity_test(package_path): - print("package : %s is invalid" % package_path.encode("utf-8")) - return False - - # unpack package - if not os.path.exists(pkg_dir): - try: - if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): - ret = False - except Exception as e: - os.remove(package_path) - ret = False - print('Error message: %s\t' % e) - else: - print("The file does not exist.") - return ret + if not install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, + pkgs_name_in_json): + result = False + return result def sub_list(aList, bList): @@ -293,7 +310,7 @@ def update_submodule(repo_path): execute_command(cmd, cwd=repo_path) print("Submodule update successful") except Exception as e: - print('Error message:%s' % e) + logging.warning('Error message:%s' % e) def get_pkg_folder_by_orign_path(orign_path, version): @@ -304,7 +321,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + logging.warning('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -381,8 +398,7 @@ def update_latest_packages(sys_value): result = False except Exception as e: - # print("Error message : %s" % e) - print("Failed to connect to the mirror server, using non-mirror server to update.") + logging.warning("Failed to connect to the mirror server, using non-mirror server to update.") if not right_path_flag: continue @@ -420,7 +436,7 @@ def get_git_root_path(repo_path): os.chdir(before) return get_git_root except Exception as e: - print("Error message : %s" % e) + logging.warning("Error message : %s" % e) return None else: print("Missing path %s" % repo_path) @@ -541,7 +557,7 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file # re-download the packages in error_packages_list for pkg in error_packages_list: - if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): + if install_package(env_root, pkgs_root, bsp_root, pkg, force_update): print("\n==============================> %s %s update done \n" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: @@ -678,8 +694,8 @@ def delete_git_package(pkg, remove_path_with_version, force_update, package_dele package_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + logging.warning('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname): @@ -689,7 +705,7 @@ def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) except Exception as e: package_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( + logging.warning('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", remove_path_with_version.encode("utf-8"), e)) @@ -743,7 +759,7 @@ def install_packages(sys_value, force_update): packages_download_fail_list = [] for package in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, package, force_update): + if install_package(env_root, pkgs_root, bsp_root, package, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( package['name'], package['ver'])) else: From ce04de292bced35cffc2ceb107d2ca58e5f1462d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:50:46 +0800 Subject: [PATCH 12/14] [update] change pathname to filename --- env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env.py b/env.py index 06eb18a5..ca9daf83 100644 --- a/env.py +++ b/env.py @@ -56,7 +56,7 @@ def init_logger(env_root): os.makedirs(log_path) log_name = os.path.join(log_path, "running_log.txt") - log_format = "%(pathname)s %(lineno)d %(message)s " + log_format = "%(filename)s %(lineno)d %(message)s " date_format = '%Y-%m-%d %H:%M:%S %a ' logging.basicConfig(level=logging.WARNING, format=log_format, From b5a549312d93afc7d6b4bd4d0d4832e350f007e3 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 18:02:46 +0800 Subject: [PATCH 13/14] [update] rename Package class --- cmds/cmd_package/cmd_package_list.py | 4 ++-- cmds/cmd_package/cmd_package_update.py | 14 ++++++-------- package.py | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmds/cmd_package/cmd_package_list.py b/cmds/cmd_package/cmd_package_list.py index 5e7c17b6..1b0cbcc8 100644 --- a/cmds/cmd_package/cmd_package_list.py +++ b/cmds/cmd_package/cmd_package_list.py @@ -26,7 +26,7 @@ import os import platform import kconfig -from package import Package +from package import PackageOperation from vars import Import @@ -55,7 +55,7 @@ def list_packages(): packages = kconfig.parse(config_file) for pkg in packages: - package = Package() + package = PackageOperation() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 9d85185e..64385e13 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -21,6 +21,7 @@ # Change Logs: # Date Author Notes # 2020-04-08 SummerGift Optimize program structure +# 2020-04-13 SummerGift refactoring # import os @@ -33,7 +34,7 @@ import archive import requests import logging -from package import Package, Bridge_SConscript +from package import PackageOperation, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ find_macro_in_config @@ -233,7 +234,7 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): # get the .config file from env env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') - package = Package() + package = PackageOperation() pkg_path = package_info['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] @@ -345,15 +346,14 @@ def update_latest_packages(sys_value): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') with open(package_filename, 'r') as f: read_back_pkgs_json = json.load(f) for pkg in read_back_pkgs_json: right_path_flag = True - package = Package() + package = PackageOperation() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] @@ -370,9 +370,7 @@ def update_latest_packages(sys_value): # noinspection PyBroadException try: # If mirror acceleration is enabled, get the update address from the mirror server. - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") # Change repo's upstream address. diff --git a/package.py b/package.py index 824a9b4e..dd99625a 100644 --- a/package.py +++ b/package.py @@ -130,7 +130,7 @@ ''' -class Package: +class PackageOperation: pkg = None def parse(self, filename): From 9e09486d5988b84ca84d7ae145accfe15f7b8935 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 18:49:58 +0800 Subject: [PATCH 14/14] [update] fix overwrite package problem --- archive.py | 26 +++++++++++++++++++++----- cmds/cmd_package/cmd_package_update.py | 3 ++- env.py | 1 + 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/archive.py b/archive.py index a5c992d8..c6c3d86e 100644 --- a/archive.py +++ b/archive.py @@ -30,12 +30,19 @@ import pkgsdb import platform import shutil +import logging -def unpack(archive_fn, path, pkg, package_name): - pkg_ver = pkg['ver'] +def unpack(archive_fn, path, package_info, package_name): + pkg_ver = package_info['ver'] flag = True + package_temp_path = os.path.join(path, "package_temp") + os.makedirs(package_temp_path) + + logging.info("archive_fn", archive_fn) + logging.info("path", path) + if platform.system() == "Windows": is_windows = True else: @@ -52,6 +59,7 @@ def unpack(archive_fn, path, pkg, package_name): else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) + pkgsdb.save_to_database(a, archive_fn) arch.close() @@ -72,8 +80,8 @@ def unpack(archive_fn, path, pkg, package_name): if ".zip" in archive_fn: arch = zipfile.ZipFile(archive_fn, "r") for item in arch.namelist(): - arch.extract(item, path) - if not os.path.isdir(os.path.join(path, item)): + arch.extract(item, package_temp_path) + if not os.path.isdir(os.path.join(package_temp_path, item)): if is_windows: right_path = item.replace('/', '\\') else: @@ -86,6 +94,7 @@ def unpack(archive_fn, path, pkg, package_name): flag = False right_name_to_db = right_path.replace(dir_name, change_dirname, 1) + right_path = os.path.join("package_temp", right_path) pkgsdb.save_to_database(right_name_to_db, archive_fn, right_path) arch.close() @@ -99,7 +108,14 @@ def unpack(archive_fn, path, pkg, package_name): else: shutil.rmtree(os.path.join(path, change_dirname)) - os.rename(os.path.join(path, dir_name), os.path.join(path, change_dirname)) + rename_path = os.path.join(package_temp_path, change_dirname) + os.rename(os.path.join(package_temp_path, dir_name), rename_path) + + # copy to bsp packages path. + shutil.move(rename_path, os.path.join(path, change_dirname)) + + # remove temp dir + shutil.rmtree(package_temp_path) def package_integrity_test(path): diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 64385e13..d9b3ea9b 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -192,6 +192,7 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json): + result = True # Download a package of compressed package type. if not package.download(package_info['ver'], local_pkgs_path, package_url): return False @@ -216,7 +217,7 @@ def install_not_git_package(package, package_info, local_pkgs_path, package_url, else: print("The file does not exist.") - return True + return result # noinspection PyUnboundLocalVariable diff --git a/env.py b/env.py index ca9daf83..96b54db1 100644 --- a/env.py +++ b/env.py @@ -22,6 +22,7 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2019-1-16 SummerGift Add chinese detection +# 2020-4-13 SummerGift refactoring # import os