From d5cf840718f9bbbc5bc50abc42ff482b0f0007fe Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 16:14:24 +0800 Subject: [PATCH 01/48] [optimize] code clear up --- archive.py | 8 +++--- cmds/cmd_menuconfig.py | 23 +++++++++-------- env.py | 5 ++-- init_env.py | 11 ++++++--- kconfig.py | 10 +++++--- package.py | 9 ++++--- pkgsdb.py | 56 +++++++++++++++++++++--------------------- 7 files changed, 64 insertions(+), 58 deletions(-) diff --git a/archive.py b/archive.py index 24117f41..b0ea9f1b 100644 --- a/archive.py +++ b/archive.py @@ -83,11 +83,11 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): dir_name = os.path.split(right_path)[0] change_dirname = pkgs_name_in_json + '-' + pkg_ver flag = False - + right_name_to_db = right_path.replace(dir_name, change_dirname, 1) pkgsdb.savetodb(right_name_to_db, archive_fn, right_path) arch.close() - + # Change the folder name change_dirname = pkgs_name_in_json + '-' + pkg_ver @@ -97,8 +97,8 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): os.system(cmd) else: shutil.rmtree(os.path.join(path, change_dirname)) - - os.rename(os.path.join(path, dir_name),os.path.join(path, change_dirname)) + + os.rename(os.path.join(path, dir_name), os.path.join(path, change_dirname)) def packtest(path): diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index a8a75d60..7cb61081 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -32,7 +32,7 @@ def is_pkg_special_config(config_str): - ''' judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER''' + """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER""" if type(config_str) == type('a'): if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')): @@ -43,7 +43,7 @@ def is_pkg_special_config(config_str): def mk_rtconfig(filename): try: config = open(filename, 'r') - except: + except Exception as e: print('open config:%s failed' % filename) return @@ -89,7 +89,7 @@ def mk_rtconfig(filename): if setting[1] == 'y': rtconfig.write('#define %s\n' % setting[0]) else: - rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$",line)[0])) + rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$", line)[0])) if os.path.isfile('rtconfig_project.h'): rtconfig.write('#include "rtconfig_project.h"\n') @@ -102,7 +102,7 @@ def mk_rtconfig(filename): def find_macro_in_config(filename, macro_name): try: config = open(filename, "r") - except: + except Exception as e: print('open .config failed') return @@ -122,7 +122,7 @@ def find_macro_in_config(filename, macro_name): empty_line = 1 continue - #comment_line = line[1:] + # comment_line = line[1:] if line.startswith('# CONFIG_'): line = ' ' + line[9:] else: @@ -146,7 +146,6 @@ def find_macro_in_config(filename, macro_name): def cmd(args): - env_root = Import('env_root') os_version = platform.platform(True)[10:13] kconfig_win7_path = os.path.join( @@ -159,9 +158,9 @@ def cmd(args): print("\n\033[1;31;40m 命令应当在某一特定 BSP 目录下执行，例如：\"rt-thread/bsp/stm32/stm32f091-st-nucleo\"\033[0m") print("\033[1;31;40m请确保当前目录为 BSP 根目录，并且该目录中有 Kconfig 文件。\033[0m\n") - print (" command should be used in a bsp root path with a Kconfig file.") - print ("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") - print ("You should check if there is a Kconfig file in your bsp root first.") + print(" command should be used in a bsp root path with a Kconfig file.") + print("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") + print("You should check if there is a Kconfig file in your bsp root first.") if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -242,13 +241,13 @@ def cmd(args): if find_macro_in_config(fn, 'SYS_CREATE_MDK_IAR_PROJECT'): if find_macro_in_config(fn, 'SYS_CREATE_MDK4'): os.system('scons --target=mdk4 -s') - print("Create mdk4 project done") + print("Create mdk4 project done") elif find_macro_in_config(fn, 'SYS_CREATE_MDK5'): os.system('scons --target=mdk5 -s') - print("Create mdk5 project done") + print("Create mdk5 project done") elif find_macro_in_config(fn, 'SYS_CREATE_IAR'): os.system('scons --target=iar -s') - print("Create iar project done") + print("Create iar project done") def add_parser(sub): diff --git a/env.py b/env.py index 18a4efa7..3dbb3e9a 100644 --- a/env.py +++ b/env.py @@ -53,7 +53,7 @@ def main(): bsp_root = os.getcwd() script_root = os.path.split(os.path.realpath(__file__))[0] env_root = os.getenv("ENV_ROOT") - if env_root == None: + if env_root is None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') else: @@ -62,13 +62,14 @@ def main(): sys.path = sys.path + [os.path.join(script_root)] pkgs_root = os.getenv("PKGS_ROOT") - if pkgs_root == None: + if pkgs_root is None: pkgs_root = os.path.join(env_root, 'packages') Export('env_root') Export('bsp_root') Export('pkgs_root') + # noinspection PyBroadException try: bsp_root.encode('utf-8').decode("ascii") except Exception as e: diff --git a/init_env.py b/init_env.py index 500fb0d9..0c024212 100644 --- a/init_env.py +++ b/init_env.py @@ -25,24 +25,27 @@ from multiprocessing import Process import os -import sys + def run_proc(name, env_root): - exec_file = os.path.join(env_root, "tools\scripts\env.py") + exec_file = os.path.join(env_root, r"tools\scripts\env.py") log_std = os.path.join(env_root, "env_log_std") log_err = os.path.join(env_root, "env_log_err") + # noinspection PyBroadException try: - os.system("python %s package --upgrade 1>%s 2>%s"%(exec_file, log_std, log_err)) + os.system("python %s package --upgrade 1>%s 2>%s" % (exec_file, log_std, log_err)) except Exception as e: print("Auto upgrade failed, please check your network.") pass + def main(): env_root = env_root = os.getenv("ENV_ROOT") p = Process(target=run_proc, args=('upgrade', env_root)) p.start() p.join() -if __name__=='__main__': + +if __name__ == '__main__': main() diff --git a/kconfig.py b/kconfig.py index 0e904203..7c7022b3 100644 --- a/kconfig.py +++ b/kconfig.py @@ -1,4 +1,4 @@ - # -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- # # File : kconfig.py # This file is part of RT-Thread RTOS @@ -50,10 +50,12 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): ret = [] + + # noinspection PyBroadException try: - config = open(filename, "r") - except: - print('open .config failed') + config = open(filename, "r") + except Exception as e: + print('open .config failed') return ret for line in config: diff --git a/package.py b/package.py index 99e83383..c3674eb4 100644 --- a/package.py +++ b/package.py @@ -128,6 +128,7 @@ Return('group') ''' + class Package: pkg = None @@ -185,7 +186,7 @@ def download(self, ver, path, url_from_srv): os.remove(path) else: if archive.packtest(path): - #print "The file is rigit." + # print "The file is rigit." return True else: os.remove(path) @@ -200,7 +201,7 @@ def download(self, ver, path, url_from_srv): print('Start to download package : %s ' % filename.encode("utf-8")) while True: - #print("retryCount : %d"%retryCount) + # print("retryCount : %d"%retryCount) try: r = requests.get(url_from_srv, stream=True, headers=headers) @@ -233,8 +234,8 @@ def download(self, ver, path, url_from_srv): ret = False break except Exception as e: - print(url_from_srv) - print('error message:%s\t' %e) + print(url_from_srv) + print('error message:%s\t' % e) retryCount = retryCount + 1 if retryCount > 5: print('%s download fail!\n' % path.encode("utf-8")) diff --git a/pkgsdb.py b/pkgsdb.py index 488008cb..715d54cb 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -29,6 +29,7 @@ import sys from vars import Import + SHOW_SQL = False @@ -49,7 +50,7 @@ def GetFileMd5(filename): def get_conn(path): conn = sqlite3.connect(path) if os.path.exists(path) and os.path.isfile(path): - #print('on disk:[{}]'.format(path)) + # print('on disk:[{}]'.format(path)) return conn else: conn = None @@ -76,7 +77,7 @@ def create_table(conn, sql): print('执行sql:[{}]'.format(sql)) cu.execute(sql) conn.commit() - #print('create data table successful!') + # print('create data table successful!') close_all(conn) else: print('the [{}] is empty or equal None!'.format(sql)) @@ -113,7 +114,7 @@ def isdataexist(pathname): return ret -#将数据添加到数据库，如果数据库中已经存在则不重复添加 +# 将数据添加到数据库，如果数据库中已经存在则不重复添加 def savetodb(pathname, pkgspathname, before_change_name): dbpathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') @@ -123,14 +124,14 @@ def savetodb(pathname, pkgspathname, before_change_name): save_sql = '''insert into packagefile values (?, ?, ?)''' package = os.path.basename(pkgspathname) md5pathname = os.path.join(bsppkgs, before_change_name) - #print("pathname to save : %s"%pathname) - #print("md5pathname : %s"%md5pathname) - + # print("pathname to save : %s"%pathname) + # print("md5pathname : %s"%md5pathname) + if not os.path.isfile(md5pathname): print("md5pathname is Invalid") - + md5 = GetFileMd5(md5pathname) - #print("md5 to save : %s"%md5) + # print("md5 to save : %s"%md5) data = [(pathname, package, md5)] save(conn, save_sql, data) @@ -140,33 +141,32 @@ def dbdump(dbpathname): c = get_cursor(conn) cursor = c.execute("SELECT pathname, package, md5 from packagefile") for row in cursor: - print("pathname = ", row[0]) - print("package = ", row[1]) - print("md5 = ", row[2], "\n") + print("pathname = ", row[0]) + print("package = ", row[1]) + print("md5 = ", row[2], "\n") conn.close() -#delete the unchanged file - def remove_unchangedfile(pathname, dbpathname, dbsqlname): + """delete unchanged files""" flag = True conn = get_conn(dbpathname) c = get_cursor(conn) - #print('pathname : %s'%pathname) - #print('dbsqlname : %s'%dbsqlname) + # print('pathname : %s'%pathname) + # print('dbsqlname : %s'%dbsqlname) filemd5 = GetFileMd5(pathname) - #print("filemd5 : %s"%filemd5) + # print("filemd5 : %s"%filemd5) dbmd5 = 0 sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' - #print sql + # print sql cursor = c.execute(sql) for row in cursor: dbmd5 = row[0] # fetch md5 from databas - #print("dbmd5 : %s"%dbmd5) + # print("dbmd5 : %s"%dbmd5) if dbmd5 == filemd5: # delete file info from database @@ -175,10 +175,11 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): conn.commit() os.remove(pathname) else: - print ("%s has been changed." % pathname) - print ('Are you sure you want to permanently delete the file: %s?' % - os.path.basename(pathname)) - print ('If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') + print("%s has been changed." % pathname) + print('Are you sure you want to permanently delete the file: %s?' % + os.path.basename(pathname)) + print( + 'If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') if sys.version_info < (3, 0): rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') @@ -196,8 +197,8 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): return flag -#删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 -#如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false +# 删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 +# 如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false def deletepackdir(dirpath, dbpathname): flag = getdirdisplay(dirpath, dbpathname) @@ -209,11 +210,11 @@ def deletepackdir(dirpath, dbpathname): for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(dirpath) - #print "the dir should be delete" + # print "the dir should be delete" return flag -#遍历filepath下所有文件，包括子目录 +# 遍历filepath下所有文件，包括子目录 def displaydir(filepath, basepath, length, dbpathname): flag = True if os.path.isdir(filepath): @@ -225,7 +226,7 @@ def displaydir(filepath, basepath, length, dbpathname): else: pathname = os.path.join(filepath, fi_d) dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] - #print("dbsqlname : %s"%dbsqlname) + # print("dbsqlname : %s"%dbsqlname) if not remove_unchangedfile(pathname, dbpathname, dbsqlname): flag = False return flag @@ -235,6 +236,5 @@ def getdirdisplay(filepath, dbpathname): display = filepath length = len(display) basepath = os.path.basename(filepath) - #print "basepath:",basepath flag = displaydir(filepath, basepath, length, dbpathname) return flag From 08910db124706d32ccda5d2adc6c19368d845d8f Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 16:52:42 +0800 Subject: [PATCH 02/48] [optimize] Code improvement --- package.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/package.py b/package.py index c3674eb4..027bf529 100644 --- a/package.py +++ b/package.py @@ -22,6 +22,7 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2018-12-28 Ernest Chen Add package information and enjoy package maker +# 2020-4-7 SummerGift Code improvement # import os @@ -191,7 +192,7 @@ def download(self, ver, path, url_from_srv): else: os.remove(path) - retryCount = 0 + retry_count = 0 headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', @@ -201,7 +202,7 @@ def download(self, ver, path, url_from_srv): print('Start to download package : %s ' % filename.encode("utf-8")) while True: - # print("retryCount : %d"%retryCount) + # print("retry_count : %d"%retry_count) try: r = requests.get(url_from_srv, stream=True, headers=headers) @@ -216,7 +217,7 @@ def download(self, ver, path, url_from_srv): sys.stdout.write("\rDownloding %d KB" % flush_count) sys.stdout.flush() - retryCount = retryCount + 1 + retry_count = retry_count + 1 if archive.packtest(path): # make sure the file is right ret = True @@ -226,7 +227,7 @@ def download(self, ver, path, url_from_srv): else: if os.path.isfile(path): os.remove(path) - if retryCount > 5: + if retry_count > 5: print( "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) if os.path.isfile(path): @@ -236,21 +237,22 @@ def download(self, ver, path, url_from_srv): except Exception as e: print(url_from_srv) print('error message:%s\t' % e) - retryCount = retryCount + 1 - if retryCount > 5: + retry_count = retry_count + 1 + if retry_count > 5: print('%s download fail!\n' % path.encode("utf-8")) if os.path.isfile(path): os.remove(path) return False return ret - def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): + @staticmethod + def unpack(package_path, path, pkg, package_name_in_json): try: # ignore the return value - archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) + archive.unpack(package_path, path, pkg, package_name_in_json) return True except Exception as e: print('unpack error message :%s' % e) - print('unpack %s failed' % os.path.basename(fullpkg_path)) - os.remove(fullpkg_path) + print('unpack %s failed' % os.path.basename(package_path)) + os.remove(package_path) return False From e32ed122b796545b1d6fa7aec4617159b9399661 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 17:08:59 +0800 Subject: [PATCH 03/48] [optimize] code improvement --- cmds/cmd_package.py | 267 ++++++++++++++++++++++++-------------------- 1 file changed, 144 insertions(+), 123 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 2689b022..182ffa36 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -54,7 +54,6 @@ "* $ sudo apt-get install python-requests\n" "****************************************\n") -"""package command""" def execute_command(cmdstring, cwd=None, shell=True): """Execute the system command at the specified address.""" @@ -72,6 +71,7 @@ def execute_command(cmdstring, cwd=None, shell=True): return stdout_str + def git_pull_repo(repo_path, repo_url=''): if platform.system() == "Windows": cmd = r'git config --local core.autocrlf true' @@ -79,6 +79,7 @@ def git_pull_repo(repo_path, repo_url=''): cmd = r'git pull ' + repo_url execute_command(cmd, cwd=repo_path) + def determine_support_chinese(env_root): get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') if os.path.isfile(get_flag_file_path): @@ -86,6 +87,7 @@ def determine_support_chinese(env_root): else: return False + def user_input(msg, default_value): """Gets the user's keyboard input.""" @@ -103,7 +105,8 @@ def user_input(msg, default_value): return value -def union_input(msg = None): + +def union_input(msg=None): """Gets the union keyboard input.""" if sys.version_info < (3, 0): @@ -226,8 +229,8 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None -def determine_url_valid(url_from_srv): +def determine_url_valid(url_from_srv): headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', @@ -289,14 +292,16 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = False try: - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # determine whether the package package url is valid - if get_package_url != None and determine_url_valid(get_package_url): + if get_package_url is not None and determine_url_valid(get_package_url): package_url = get_package_url - if get_ver_sha != None: + if get_ver_sha is not None: ver_sha = get_ver_sha upstream_change_flag = True @@ -315,7 +320,6 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): git_check_cmd = 'git checkout -q ' + ver_sha execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: print("\nFailed to download software package with git. Please check the network connection.") return False @@ -325,20 +329,26 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): execute_command(cmd, cwd=repo_path) # If there is a .gitmodules file in the package, prepare to update submodule. - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): print("Start to update submodule") # print("开始更新软件包子模块") - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) \ + or (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + # print("开启了镜像加速，开始修改 .gitmodules 文件") - replace_list = modify_submod_file_to_mirror(submod_path) # Modify .gitmodules file + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file # print("开始执行更新动作") cmd = 'git submodule update --init --recursive' execute_command(cmd, cwd=repo_path) - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and + find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if len(replace_list): for item in replace_list: submod_dir_path = os.path.join(repo_path, item[2]) @@ -346,11 +356,13 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): cmd = 'git remote set-url origin ' + item[0] execute_command(cmd, cwd=submod_dir_path) - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - if os.path.isfile(submod_path): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if os.path.isfile(submodule_path): cmd = 'git checkout .gitmodules' execute_command(cmd, cwd=repo_path) - else: # Download a package of compressed package type. if not package.download(pkg['ver'], local_pkgs_path, package_url): @@ -358,20 +370,20 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): pkg_dir = package.get_filename(pkg['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] - pkg_fullpath = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - if not archive.packtest(pkg_fullpath): - print("package : %s is invalid"%pkg_fullpath.encode("utf-8")) + if not archive.packtest(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) return False - + # unpack package if not os.path.exists(pkg_dir): try: - if not package.unpack(pkg_fullpath, bsp_pkgs_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): ret = False except Exception as e: - os.remove(pkg_fullpath) + os.remove(package_path) ret = False print('error message: %s\t' % e) else: @@ -394,11 +406,11 @@ def package_list(): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - print ('\033[1;31;40mNo system configuration file : .config.\033[0m') - print ('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + print('\033[1;31;40mNo system configuration file : .config.\033[0m') + print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -417,12 +429,12 @@ def package_list(): package.parse(pkg_path) pkgs_name_in_json = package.get_name() - print ("package name : %s, ver : %s "%(pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) if not pkgs: - print ("Packages list is empty.") - print ('You can use < menuconfig > command to select online packages.') - print ('Then use < pkgs --update > command to install them.') + print("Packages list is empty.") + print('You can use < menuconfig > command to select online packages.') + print('Then use < pkgs --update > command to install them.') return @@ -471,7 +483,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e)) + print('error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -512,14 +524,16 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): try: # If mirror acceleration is enabled, get the update address from # the mirror server. - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") # Change repo's upstream address. mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) - if mirror_url[0] != None: + if mirror_url[0] is not None: cmd = 'git remote set-url origin ' + mirror_url[0] git_cmd_exec(cmd, repo_path) @@ -536,13 +550,13 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): # recover origin url to the path which get from packages.json file if package.get_url(pkg['ver']): cmd = 'git remote set-url origin ' + \ - package.get_url(pkg['ver']) + package.get_url(pkg['ver']) git_cmd_exec(cmd, repo_path) else: print("Can't find the package : %s's url in file : %s" % (payload_pkgs_name_in_json, pkg_path)) - print("==============================> %s update done\n" %(pkgs_name_in_json)) + print("==============================> %s update done\n" % (pkgs_name_in_json)) def pre_package_update(): @@ -555,11 +569,11 @@ def pre_package_update(): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - print ('No system configuration file : .config.') - print ('You should use < menuconfig > command to config bsp first.') + print('No system configuration file : .config.') + print('You should use < menuconfig > command to config bsp first.') if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -583,7 +597,7 @@ def pre_package_update(): dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') Export('dbsqlite_pathname') dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - + # Avoid creating tables more than one time if not os.path.isfile(dbsqlite_pathname): conn = pkgsdb.get_conn(dbsqlite_pathname) @@ -634,7 +648,7 @@ def pre_package_update(): # create SConscript file if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - with open(os.path.join(bsp_packages_path, 'SConscript'),'w') as f: + with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] @@ -652,11 +666,11 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): if len(error_packages_list): print("\n==============================> Packages list to download : \n") for pkg in error_packages_list: - print("Package name : %s, Ver : %s"%(pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) print("\nThe packages in the list above are accidentally deleted, env will redownload them.") print("Warning: Packages should be deleted in command.\n") - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + for pkg in error_packages_list: # Redownloaded the packages in error_packages_list if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) @@ -667,8 +681,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): if len(error_packages_redownload_error_list): print("%s" % error_packages_redownload_error_list) - print ("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % + (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) write_back_pkgs_json = sub_list( read_back_pkgs_json, error_packages_redownload_error_list) read_back_pkgs_json = write_back_pkgs_json @@ -684,7 +698,7 @@ def rm_package(dir): if platform.system() != "Windows": shutil.rmtree(dir) else: - dir = '"' + dir + '"' + dir = '"' + dir + '"' cmd = 'rd /s /q ' + dir os.system(cmd) @@ -692,15 +706,15 @@ def rm_package(dir): if platform.system() != "Windows": shutil.rmtree(dir) else: - dir = '"' + dir + '"' + dir = '"' + dir + '"' cmd = 'rmdir /s /q ' + dir os.system(cmd) if os.path.isdir(dir): - print ("Folder path: %s" % dir.encode("utf-8")) + print("Folder path: %s" % dir.encode("utf-8")) return False else: - print ("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) return True @@ -741,7 +755,7 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): continue else: error_packages_list.append(pkg) - + # Handle the failed download packages get_flag = error_packages_handle( error_packages_list, read_back_pkgs_json, pkgs_fn) @@ -756,7 +770,7 @@ def write_storage_file(pkgs_fn, newpkgs): next update. """ - with open(pkgs_fn,'w') as f: + with open(pkgs_fn, 'w') as f: f.write(str(json.dumps(newpkgs, indent=1))) @@ -782,7 +796,7 @@ def package_update(isDeleteOld=False): # According to the env version, whether Chinese output is supported or not if determine_support_chinese(env_root): if platform.system() == "Windows": - os.system('chcp 65001 > nul') + os.system('chcp 65001 > nul') oldpkgs = sys_value[0] newpkgs = sys_value[1] @@ -801,11 +815,12 @@ def package_update(isDeleteOld=False): error_package, bsp_packages_path) if os.path.isdir(removepath_ver): - print("\nError: %s package delete failed, begin to remove it."% + print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) if rm_package(removepath_ver) == False: - print("Error: Delete package %s failed! Please delete the folder manually.\n"%error_package['name'].encode("utf-8")) + print("Error: Delete package %s failed! Please delete the folder manually.\n" % + error_package['name'].encode("utf-8")) return # 1.in old ,not in new : Software packages that need to be removed. @@ -821,13 +836,13 @@ def package_update(isDeleteOld=False): if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): gitdir = removepath_ver - print ("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) + print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) if isDeleteOld: if rm_package(gitdir) == False: print("Floder delete fail: %s" % gitdir.encode("utf-8")) print("Please delete this folder manually.") else: - print ("The folder is managed by git. Do you want to delete this folder?\n") + print("The folder is managed by git. Do you want to delete this folder?\n") if sys.version_info < (3, 0): rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') else: @@ -859,7 +874,7 @@ def package_update(isDeleteOld=False): return else: # write error messages - with open(pkgs_error_list_fn,'w') as f: + with open(pkgs_error_list_fn, 'w') as f: f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) # 2.in new not in old : Software packages to be installed. @@ -878,7 +893,7 @@ def package_update(isDeleteOld=False): # If the PKG download fails, record it in the # pkgs_download_fail_list. pkgs_download_fail_list.append(pkg) - print(pkg, 'download failed.') + print(pkg, 'download failed.') flag = False # Get the currently updated configuration. @@ -886,7 +901,7 @@ def package_update(isDeleteOld=False): # Give hints based on the success of the download. if len(pkgs_download_fail_list): - print("\nPackage download failed list:" ) + print("\nPackage download failed list:") for item in pkgs_download_fail_list: print(item) @@ -909,104 +924,107 @@ def package_update(isDeleteOld=False): flag = False if flag: - print ("Operation completed successfully.") + print("Operation completed successfully.") else: - print ("Operation failed.") - + print("Operation failed.") + + def package_wizard(): """Packages creation wizard. The user enters the package name, version number, category, and automatically generates the package index file. """ # Welcome - print ('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') - - #Simple introduction about the wizard - print ('note :') - print (' \033[5;35;40m[ ]\033[0m means default setting or optional information.') - print (' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') - - #first step - print ('\033[5;33;40m\n1.Please input a new package name :\033[0m') + print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') + + # Simple introduction about the wizard + print('note :') + print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') + print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') + + # first step + print('\033[5;33;40m\n1.Please input a new package name :\033[0m') name = union_input() regular_obj = re.compile('\W') while name == '' or name.isspace() == True or regular_obj.search(name.strip()): if name == '' or name.isspace(): - print ('\033[1;31;40mError: you must input a package name. Try again.\033[0m') + print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') name = union_input() - else: - print ('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') + else: + print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') name = union_input() default_description = 'Please add description of ' + name + ' in English.' - #description = user_input('menuconfig option name,default:\n',default_description) + # description = user_input('menuconfig option name,default:\n',default_description) description = default_description - description_zh = "请添加软件包 " + name +" 的中文描述。" - - #second step + description_zh = "请添加软件包 " + name + " 的中文描述。" + + # second step ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') ver_standard = ver.replace('.', '') - #keyword = user_input('keyword,default:\n', name) + # keyword = user_input('keyword,default:\n', name) keyword = name - #third step + # third step packageclass = ('iot', 'language', 'misc', 'multimedia', 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') - print ('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print ("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') + print( + "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") classnu = union_input() - while classnu == '' or classnu.isdigit()== False or int(classnu) < 1 or int(classnu) >9: - if classnu == '' : - print ('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') - else : - print ('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') + while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: + if classnu == '': + print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') + else: + print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') classnu = union_input() - - pkgsclass = packageclass[int(classnu) - 1] - #fourth step - print ("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") + pkgsclass = packageclass[int(classnu) - 1] + + # fourth step + print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") authorname = union_input() while authorname == '': - print ("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") + print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") authorname = union_input() - #fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + # fifth step + authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') while authoremail == '': - print ('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() - - #sixth step - print ('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') - print ("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") + print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') + authoremail = union_input() + + # sixth step + print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') + print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') license_class = union_input() - while license_class == '' : - print ('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() + while license_class == '': + print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') + license_class = union_input() - if license_class.isdigit()== True and int(license_class) >= 1 and int(license_class) <= 4: + if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: license = license_index[int(license_class) - 1] - else : - license = license_class - - #seventh step - print ('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') - print ("\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - + else: + license = license_class + + # seventh step + print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') + print( + "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + repository = union_input() while repository == '': - print ('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() + print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') + repository = union_input() pkg_path = name if not os.path.exists(pkg_path): os.mkdir(pkg_path) else: - print ("\033[1;31;40mError: the package directory is exits!\033[0m") + print("\033[1;31;40mError: the package directory is exits!\033[0m") s = Template(Kconfig_file) uppername = str.upper(name) @@ -1017,7 +1035,9 @@ def package_wizard(): f.close() s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass,authorname=authorname,authoremail=authoremail, description=description, description_zh=description_zh,version=ver, keyword=keyword,license=license, repository=repository, pkgs_using_name=uppername) + package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + description=description, description_zh=description_zh, version=ver, keyword=keyword, + license=license, repository=repository, pkgs_using_name=uppername) f = open(os.path.join(pkg_path, 'package.json'), 'w') f.write(package) f.close() @@ -1025,9 +1045,10 @@ def package_wizard(): print ('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') print ('Please \033[5;34;40mupdate\033[0m other information of this package based on Kconfig and package.json in directory '+name+'.') + def upgrade_packages_index(): """Update the package repository index.""" - + env_root = Import('env_root') pkgs_root = Import('pkgs_root') env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') @@ -1091,26 +1112,26 @@ def upgrade_env_script(): def package_upgrade(): """Update the package repository directory and env function scripts.""" - + upgrade_packages_index() upgrade_env_script() def package_print_env(): - print ("Here are some environmental variables.") - print ( + print("Here are some environmental variables.") + print( "If you meet some problems,please check them. Make sure the configuration is correct.") - print ("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) - print ("RTT_CC:%s" % (os.getenv("RTT_CC"))) - print ("SCONS:%s" % (os.getenv("SCONS"))) - print ("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) + print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) + print("RTT_CC:%s" % (os.getenv("RTT_CC"))) + print("SCONS:%s" % (os.getenv("SCONS"))) + print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) env_root = os.getenv('ENV_ROOT') if env_root == None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') - print ("ENV_ROOT:%s" % (env_root)) + print("ENV_ROOT:%s" % (env_root)) def cmd(args): From 3be7a9d8c762ea4aa086307aded9e59c0a814b75 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 17:21:52 +0800 Subject: [PATCH 04/48] [optimize] clear up --- cmds/cmd_package.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 182ffa36..24912b8e 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -818,7 +818,7 @@ def package_update(isDeleteOld=False): print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) - if rm_package(removepath_ver) == False: + if not rm_package(removepath_ver): print("Error: Delete package %s failed! Please delete the folder manually.\n" % error_package['name'].encode("utf-8")) return @@ -1042,8 +1042,9 @@ def package_wizard(): f.write(package) f.close() - print ('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') - print ('Please \033[5;34;40mupdate\033[0m other information of this package based on Kconfig and package.json in directory '+name+'.') + print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') + print('Please \033[5;34;40mupdate\033[0m other information of this package ' + 'based on Kconfig and package.json in directory ' + name + '.') def upgrade_packages_index(): @@ -1051,25 +1052,28 @@ def upgrade_packages_index(): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') - if get_package_url != None: + + if get_package_url is not None: git_repo = get_package_url else: print("Failed to get url from mirror server. Using default url.") git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' else: git_repo = 'https://github.com/RT-Thread/packages.git' - + packages_root = pkgs_root pkgs_path = os.path.join(packages_root, 'packages') if not os.path.isdir(pkgs_path): cmd = 'git clone ' + git_repo + ' ' + pkgs_path os.system(cmd) - print ("upgrade from :%s" % (git_repo.encode("utf-8"))) + print("upgrade from :%s" % (git_repo.encode("utf-8"))) else: print("Begin to upgrade env packages.") git_pull_repo(pkgs_path, git_repo) @@ -1093,11 +1097,13 @@ def upgrade_env_script(): print("Begin to upgrade env scripts.") env_root = Import('env_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') - if get_package_url != None: + + if get_package_url is not None: env_scripts_repo = get_package_url else: print("Failed to get url from mirror server. Using default url.") @@ -1119,15 +1125,14 @@ def package_upgrade(): def package_print_env(): print("Here are some environmental variables.") - print( - "If you meet some problems,please check them. Make sure the configuration is correct.") + print("If you meet some problems,please check them. Make sure the configuration is correct.") print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) print("RTT_CC:%s" % (os.getenv("RTT_CC"))) print("SCONS:%s" % (os.getenv("SCONS"))) print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) env_root = os.getenv('ENV_ROOT') - if env_root == None: + if env_root is None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') @@ -1135,7 +1140,7 @@ def package_print_env(): def cmd(args): - """Env's pkgs command execution options.""" + """Env's packages command execution options.""" if args.package_update_y: package_update(True) @@ -1154,12 +1159,12 @@ def cmd(args): def add_parser(sub): - """The pkgs command parser for env.""" + """The packages command parser for env.""" parser = sub.add_parser('package', help=__doc__, description=__doc__) parser.add_argument('--force-update', - help='force update and clean packages, install or remove the packages by your settings in menuconfig', + help='force update and clean packages, install or remove packages by settings in menuconfig', action='store_true', default=False, dest='package_update_y') From 8e4bc4d0f8aad25bd0edd490ad194902f08352cd Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 18:05:07 +0800 Subject: [PATCH 05/48] [update] cmds/cmd_package.py --- cmds/cmd_package.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 24912b8e..c8f550b2 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -110,12 +110,12 @@ def union_input(msg=None): """Gets the union keyboard input.""" if sys.version_info < (3, 0): - if msg != None: + if msg is not None: value = raw_input(msg) else: value = raw_input() else: - if msg != None: + if msg is not None: value = input(msg) else: value = input() @@ -263,7 +263,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): bsp_pkgs_path = os.path.join(bsp_root, 'packages') # get the .config file from env - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') package = Package() @@ -297,7 +297,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - # determine whether the package package url is valid + # Check whether the package package url is valid if get_package_url is not None and determine_url_valid(get_package_url): package_url = get_package_url @@ -500,7 +500,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') with open(pkgs_fn, 'r') as f: @@ -632,8 +632,7 @@ def pre_package_update(): # print("oldpkgs", oldpkgs) # regenerate file : packages/pkgs_error.json - pkgs_error_list_fn = os.path.join( - bsp_packages_path, 'pkgs_error.json') + pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') if not os.path.exists(pkgs_error_list_fn): os.chdir(bsp_packages_path) @@ -914,7 +913,7 @@ def package_update(isDeleteOld=False): get_flag = handle_download_error_packages( pkgs_fn, bsp_packages_path) - if get_flag != None: + if get_flag is not None: flag = get_flag # Update the software packages, which the version is 'latest' From 28fdab19581dc5bad4562ec500e80e608ad14011 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:32:31 +0800 Subject: [PATCH 06/48] [optimize] Optimize program structure --- cmds/__init__.py | 12 +++++++ cmds/cmd_package.py | 67 +++------------------------------------- cmds/cmd_package_list.py | 49 +++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 63 deletions(-) create mode 100644 cmds/cmd_package_list.py diff --git a/cmds/__init__.py b/cmds/__init__.py index 1006f6a6..def30d7e 100644 --- a/cmds/__init__.py +++ b/cmds/__init__.py @@ -24,3 +24,15 @@ # __all__ = ['cmd_package', 'cmd_system', 'cmd_menuconfig'] + +try: + import requests +except ImportError: + print("****************************************\n" + "* Import requests module error.\n" + "* Please install requests module first.\n" + "* pip install step:\n" + "* $ pip install requests\n" + "* command install step:\n" + "* $ sudo apt-get install python-requests\n" + "****************************************\n") \ No newline at end of file diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index c8f550b2..01201a0a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -33,7 +33,6 @@ import platform import subprocess import time -import logging import archive import sys import re @@ -41,18 +40,7 @@ from vars import Import, Export from string import Template from .cmd_menuconfig import find_macro_in_config - -try: - import requests -except ImportError: - print("****************************************\n" - "* Import requests module error.\n" - "* Please install requests module first.\n" - "* pip install step:\n" - "* $ pip install requests\n" - "* command install step:\n" - "* $ sudo apt-get install python-requests\n" - "****************************************\n") +from .cmd_package_list import list_packages def execute_command(cmdstring, cwd=None, shell=True): @@ -391,53 +379,6 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): return ret -def package_list(): - """Print the packages list in env. - - Read the.config file in the BSP directory, - and list the version number of the selected package. - """ - - fn = '.config' - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - if not os.path.isfile(fn): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print('\033[1;31;40mNo system configuration file : .config.\033[0m') - print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') - - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return - - pkgs = kconfig.parse(fn) - - for pkg in pkgs: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - pkgs_name_in_json = package.get_name() - print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) - - if not pkgs: - print("Packages list is empty.") - print('You can use < menuconfig > command to select online packages.') - print('Then use < pkgs --update > command to install them.') - return - - def sub_list(aList, bList): """Return the items in aList but not in bList.""" @@ -1147,8 +1088,8 @@ def cmd(args): package_update() elif args.package_create: package_wizard() - elif args.package_list: - package_list() + elif args.list_packages: + list_packages() elif args.package_upgrade: package_upgrade() elif args.package_print_env: @@ -1178,7 +1119,7 @@ def add_parser(sub): help='list target packages', action='store_true', default=False, - dest='package_list') + dest='list_packages') parser.add_argument('--wizard', help='create a new package with wizard', diff --git a/cmds/cmd_package_list.py b/cmds/cmd_package_list.py new file mode 100644 index 00000000..a48f2761 --- /dev/null +++ b/cmds/cmd_package_list.py @@ -0,0 +1,49 @@ +import os +import platform +import kconfig +from package import Package +from vars import Import + + +def list_packages(): + """Print the packages list in env. + + Read the.config file in the BSP directory, + and list the version number of the selected package. + """ + + fn = '.config' + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + if not os.path.isfile(fn): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\033[1;31;40mCan't find system configuration file : .config.\033[0m") + print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return + + pkgs = kconfig.parse(fn) + + for pkg in pkgs: + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + pkgs_name_in_json = package.get_name() + print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + + if not pkgs: + print("Packages list is empty.") + print('You can use < menuconfig > command to select online packages.') + print('Then use < pkgs --update > command to install them.') + return From 8cb4f595ba446a931ed5424b40b2d7a1c85ded6d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:41:41 +0800 Subject: [PATCH 07/48] [update] cmds/cmd_package_list.py --- cmds/cmd_package_list.py | 44 +++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/cmds/cmd_package_list.py b/cmds/cmd_package_list.py index a48f2761..5e7c17b6 100644 --- a/cmds/cmd_package_list.py +++ b/cmds/cmd_package_list.py @@ -1,3 +1,28 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + import os import platform import kconfig @@ -12,25 +37,24 @@ def list_packages(): and list the version number of the selected package. """ - fn = '.config' - env_root = Import('env_root') + config_file = '.config' pkgs_root = Import('pkgs_root') - if not os.path.isfile(fn): + if not os.path.isfile(config_file): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print("\033[1;31;40mCan't find system configuration file : .config.\033[0m") - print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + print("\033[1;31;40mWarning: Can't find .config.\033[0m") + print('\033[1;31;40mYou should use command to config bsp first.\033[0m') if platform.system() == "Windows": os.system('chcp 437 > nul') return - pkgs = kconfig.parse(fn) + packages = kconfig.parse(config_file) - for pkg in pkgs: + for pkg in packages: package = Package() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': @@ -39,10 +63,10 @@ def list_packages(): pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') package.parse(pkg_path) - pkgs_name_in_json = package.get_name() - print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + package_name_in_json = package.get_name().encode("utf-8") + print("package name : %s, ver : %s " % (package_name_in_json, pkg['ver'].encode("utf-8"))) - if not pkgs: + if not packages: print("Packages list is empty.") print('You can use < menuconfig > command to select online packages.') print('Then use < pkgs --update > command to install them.') From 73ca2acbcbe9fcc238d44e43f1ffbda348ff66b8 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:48:07 +0800 Subject: [PATCH 08/48] [add] cmd_package_wizard.py --- cmds/cmd_package.py | 155 +------------------------------ cmds/cmd_package_wizard.py | 183 +++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 154 deletions(-) create mode 100644 cmds/cmd_package_wizard.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 01201a0a..246ada9a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -35,12 +35,12 @@ import time import archive import sys -import re from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file from vars import Import, Export from string import Template from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages +from .cmd_package_wizard import package_wizard def execute_command(cmdstring, cwd=None, shell=True): @@ -76,41 +76,6 @@ def determine_support_chinese(env_root): return False -def user_input(msg, default_value): - """Gets the user's keyboard input.""" - - if default_value != '': - msg = '%s[%s]' % (msg, default_value) - - print(msg) - if sys.version_info < (3, 0): - value = raw_input() - else: - value = input() - - if value == '': - value = default_value - - return value - - -def union_input(msg=None): - """Gets the union keyboard input.""" - - if sys.version_info < (3, 0): - if msg is not None: - value = raw_input(msg) - else: - value = raw_input() - else: - if msg is not None: - value = input(msg) - else: - value = input() - - return value - - def get_mirror_giturl(submod_name): """Gets the submodule's url on mirror server. @@ -869,124 +834,6 @@ def package_update(isDeleteOld=False): print("Operation failed.") -def package_wizard(): - """Packages creation wizard. - - The user enters the package name, version number, category, and automatically generates the package index file. - """ - # Welcome - print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') - - # Simple introduction about the wizard - print('note :') - print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') - print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') - - # first step - print('\033[5;33;40m\n1.Please input a new package name :\033[0m') - - name = union_input() - regular_obj = re.compile('\W') - while name == '' or name.isspace() == True or regular_obj.search(name.strip()): - if name == '' or name.isspace(): - print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = union_input() - else: - print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') - name = union_input() - - default_description = 'Please add description of ' + name + ' in English.' - # description = user_input('menuconfig option name,default:\n',default_description) - description = default_description - description_zh = "请添加软件包 " + name + " 的中文描述。" - - # second step - ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') - ver_standard = ver.replace('.', '') - # keyword = user_input('keyword,default:\n', name) - keyword = name - - # third step - packageclass = ('iot', 'language', 'misc', 'multimedia', - 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') - print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print( - "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = union_input() - while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: - if classnu == '': - print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') - else: - print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = union_input() - - pkgsclass = packageclass[int(classnu) - 1] - - # fourth step - print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") - - authorname = union_input() - while authorname == '': - print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") - authorname = union_input() - - # fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') - while authoremail == '': - print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() - - # sixth step - print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') - print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") - license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = union_input() - while license_class == '': - print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() - - if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: - license = license_index[int(license_class) - 1] - else: - license = license_class - - # seventh step - print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') - print( - "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - - repository = union_input() - while repository == '': - print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() - - pkg_path = name - if not os.path.exists(pkg_path): - os.mkdir(pkg_path) - else: - print("\033[1;31;40mError: the package directory is exits!\033[0m") - - s = Template(Kconfig_file) - uppername = str.upper(name) - kconfig = s.substitute(name=uppername, description=description, version=ver, - pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) - f = open(os.path.join(pkg_path, 'Kconfig'), 'w') - f.write(kconfig) - f.close() - - s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, - description=description, description_zh=description_zh, version=ver, keyword=keyword, - license=license, repository=repository, pkgs_using_name=uppername) - f = open(os.path.join(pkg_path, 'package.json'), 'w') - f.write(package) - f.close() - - print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') - print('Please \033[5;34;40mupdate\033[0m other information of this package ' - 'based on Kconfig and package.json in directory ' + name + '.') - - def upgrade_packages_index(): """Update the package repository index.""" diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py new file mode 100644 index 00000000..0a3e6a8e --- /dev/null +++ b/cmds/cmd_package_wizard.py @@ -0,0 +1,183 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import re +import sys +from package import Kconfig_file, Package_json_file +from string import Template + + +def user_input(msg, default_value): + """Gets the user's keyboard input.""" + + if default_value != '': + msg = '%s[%s]' % (msg, default_value) + + print(msg) + if sys.version_info < (3, 0): + value = raw_input() + else: + value = input() + + if value == '': + value = default_value + + return value + + +def union_input(msg=None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg is not None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg is not None: + value = input(msg) + else: + value = input() + + return value + + +def package_wizard(): + """Packages creation wizard. + + The user enters the package name, version number, category, and automatically generates the package index file. + """ + # Welcome + print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') + + # Simple introduction about the wizard + print('note :') + print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') + print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') + + # first step + print('\033[5;33;40m\n1.Please input a new package name :\033[0m') + + name = union_input() + regular_obj = re.compile('\W') + while name == '' or name.isspace() == True or regular_obj.search(name.strip()): + if name == '' or name.isspace(): + print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') + name = union_input() + else: + print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') + name = union_input() + + default_description = 'Please add description of ' + name + ' in English.' + # description = user_input('menuconfig option name,default:\n',default_description) + description = default_description + description_zh = "请添加软件包 " + name + " 的中文描述。" + + # second step + ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') + ver_standard = ver.replace('.', '') + # keyword = user_input('keyword,default:\n', name) + keyword = name + + # third step + packageclass = ('iot', 'language', 'misc', 'multimedia', + 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') + print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') + print( + "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + classnu = union_input() + while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: + if classnu == '': + print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') + else: + print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') + classnu = union_input() + + pkgsclass = packageclass[int(classnu) - 1] + + # fourth step + print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") + + authorname = union_input() + while authorname == '': + print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") + authorname = union_input() + + # fifth step + authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + while authoremail == '': + print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') + authoremail = union_input() + + # sixth step + print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') + print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") + license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') + license_class = union_input() + while license_class == '': + print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') + license_class = union_input() + + if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: + license = license_index[int(license_class) - 1] + else: + license = license_class + + # seventh step + print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') + print( + "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + + repository = union_input() + while repository == '': + print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') + repository = union_input() + + pkg_path = name + if not os.path.exists(pkg_path): + os.mkdir(pkg_path) + else: + print("\033[1;31;40mError: the package directory is exits!\033[0m") + + s = Template(Kconfig_file) + uppername = str.upper(name) + kconfig = s.substitute(name=uppername, description=description, version=ver, + pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) + f = open(os.path.join(pkg_path, 'Kconfig'), 'w') + f.write(kconfig) + f.close() + + s = Template(Package_json_file) + package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + description=description, description_zh=description_zh, version=ver, keyword=keyword, + license=license, repository=repository, pkgs_using_name=uppername) + f = open(os.path.join(pkg_path, 'package.json'), 'w') + f.write(package) + f.close() + + print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') + print('Please \033[5;34;40mupdate\033[0m other information of this package ' + 'based on Kconfig and package.json in directory ' + name + '.') From f3cfd2b7b4b23fe79f43922b9ede0f07cd6b9d33 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:55:14 +0800 Subject: [PATCH 09/48] [add] cmds/cmd_package_printenv.py --- cmds/cmd_package.py | 18 +-------------- cmds/cmd_package_printenv.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 cmds/cmd_package_printenv.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 246ada9a..22afc2e4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -37,10 +37,10 @@ import sys from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file from vars import Import, Export -from string import Template from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard +from .cmd_package_printenv import package_print_env def execute_command(cmdstring, cwd=None, shell=True): @@ -910,22 +910,6 @@ def package_upgrade(): upgrade_env_script() -def package_print_env(): - print("Here are some environmental variables.") - print("If you meet some problems,please check them. Make sure the configuration is correct.") - print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) - print("RTT_CC:%s" % (os.getenv("RTT_CC"))) - print("SCONS:%s" % (os.getenv("SCONS"))) - print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) - - env_root = os.getenv('ENV_ROOT') - if env_root is None: - if platform.system() != 'Windows': - env_root = os.path.join(os.getenv('HOME'), '.env') - - print("ENV_ROOT:%s" % (env_root)) - - def cmd(args): """Env's packages command execution options.""" diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package_printenv.py new file mode 100644 index 00000000..f3e1a529 --- /dev/null +++ b/cmds/cmd_package_printenv.py @@ -0,0 +1,43 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import platform + + +def package_print_env(): + print("Here are some environmental variables.") + print("If you meet some problems,please check them. Make sure the configuration is correct.") + print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) + print("RTT_CC:%s" % (os.getenv("RTT_CC"))) + print("SCONS:%s" % (os.getenv("SCONS"))) + print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) + + env_root = os.getenv('ENV_ROOT') + if env_root is None: + if platform.system() != 'Windows': + env_root = os.path.join(os.getenv('HOME'), '.env') + + print("ENV_ROOT:%s" % (env_root)) From 662ed0b0d9cfa6f0fe33d783149ef7a5863df7f2 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:06:34 +0800 Subject: [PATCH 10/48] [optimize] add cmd_package_upgrade.py and cmd_package_utils.py --- cmds/cmd_package.py | 160 +----------------------------------- cmds/cmd_package_upgrade.py | 105 +++++++++++++++++++++++ cmds/cmd_package_utils.py | 113 +++++++++++++++++++++++++ cmds/cmd_package_wizard.py | 2 +- 4 files changed, 220 insertions(+), 160 deletions(-) create mode 100644 cmds/cmd_package_upgrade.py create mode 100644 cmds/cmd_package_utils.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 22afc2e4..1d6dd09a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -41,31 +41,7 @@ from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env - - -def execute_command(cmdstring, cwd=None, shell=True): - """Execute the system command at the specified address.""" - - if shell: - cmdstring_list = cmdstring - - sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, shell=shell, bufsize=4096) - - stdout_str = '' - while sub.poll() is None: - stdout_str += str(sub.stdout.read()) - time.sleep(0.1) - - return stdout_str - - -def git_pull_repo(repo_path, repo_url=''): - if platform.system() == "Windows": - cmd = r'git config --local core.autocrlf true' - execute_command(cmd, cwd=repo_path) - cmd = r'git pull ' + repo_url - execute_command(cmd, cwd=repo_path) +from .cmd_package_upgrade import package_upgrade def determine_support_chinese(env_root): @@ -125,64 +101,6 @@ def modify_submod_file_to_mirror(submod_path): print('error message:%s\t' % e) -def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): - """Get the download address from the mirror server based on the package name.""" - - try: - if type(pkgs_name_in_json) != type("str"): - if sys.version_info < (3, 0): - pkgs_name_in_json = str(pkgs_name_in_json) - else: - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - - payload = { - "userName": "RT-Thread", - "packages": [ - { - "name": "NULL", - } - ] - } - payload["packages"][0]['name'] = pkgs_name_in_json - - # print(payload) - - try: - r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) - - if r.status_code == requests.codes.ok: - package_info = json.loads(r.text) - - # Can't find package,change git package SHA if it's a git - # package - if len(package_info['packages']) == 0: - print("Package was NOT found on mirror server. Using a non-mirrored address to download.") - return None, None - else: - for item in package_info['packages'][0]['packages_info']['site']: - if item['version'] == pkgs_ver: - # Change download url - download_url = item['URL'] - if download_url[-4:] == '.git': - # Change git package SHA - repo_sha = item['VER_SHA'] - return download_url, repo_sha - return download_url, None - - print("\nTips : \nThe system needs to be upgraded.") - print("Please use the command to upgrade packages index.\n") - return None, None - - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - - def determine_url_valid(url_from_srv): headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', @@ -834,82 +752,6 @@ def package_update(isDeleteOld=False): print("Operation failed.") -def upgrade_packages_index(): - """Update the package repository index.""" - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') - - if get_package_url is not None: - git_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' - else: - git_repo = 'https://github.com/RT-Thread/packages.git' - - packages_root = pkgs_root - pkgs_path = os.path.join(packages_root, 'packages') - - if not os.path.isdir(pkgs_path): - cmd = 'git clone ' + git_repo + ' ' + pkgs_path - os.system(cmd) - print("upgrade from :%s" % (git_repo.encode("utf-8"))) - else: - print("Begin to upgrade env packages.") - git_pull_repo(pkgs_path, git_repo) - print("==============================> Env packages upgrade done \n") - - for filename in os.listdir(packages_root): - package_path = os.path.join(packages_root, filename) - if os.path.isdir(package_path): - - if package_path == pkgs_path: - continue - - if os.path.isdir(os.path.join(package_path, '.git')): - print("Begin to upgrade %s." % filename) - git_pull_repo(package_path) - print("==============================> Env %s update done \n" % filename) - - -def upgrade_env_script(): - """Update env function scripts.""" - - print("Begin to upgrade env scripts.") - env_root = Import('env_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') - - if get_package_url is not None: - env_scripts_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' - else: - env_scripts_repo = 'https://github.com/RT-Thread/env.git' - - env_scripts_root = os.path.join(env_root, 'tools', 'scripts') - git_pull_repo(env_scripts_root, env_scripts_repo) - print("==============================> Env scripts upgrade done \n") - - -def package_upgrade(): - """Update the package repository directory and env function scripts.""" - - upgrade_packages_index() - upgrade_env_script() - - def cmd(args): """Env's packages command execution options.""" diff --git a/cmds/cmd_package_upgrade.py b/cmds/cmd_package_upgrade.py new file mode 100644 index 00000000..982cfe5d --- /dev/null +++ b/cmds/cmd_package_upgrade.py @@ -0,0 +1,105 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +from vars import Import +from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server +from .cmd_menuconfig import find_macro_in_config + + +def upgrade_packages_index(): + """Update the package repository index.""" + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') + + if get_package_url is not None: + git_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' + else: + git_repo = 'https://github.com/RT-Thread/packages.git' + + packages_root = pkgs_root + pkgs_path = os.path.join(packages_root, 'packages') + + if not os.path.isdir(pkgs_path): + cmd = 'git clone ' + git_repo + ' ' + pkgs_path + os.system(cmd) + print("upgrade from :%s" % (git_repo.encode("utf-8"))) + else: + print("Begin to upgrade env packages.") + git_pull_repo(pkgs_path, git_repo) + print("==============================> Env packages upgrade done \n") + + for filename in os.listdir(packages_root): + package_path = os.path.join(packages_root, filename) + if os.path.isdir(package_path): + + if package_path == pkgs_path: + continue + + if os.path.isdir(os.path.join(package_path, '.git')): + print("Begin to upgrade %s." % filename) + git_pull_repo(package_path) + print("==============================> Env %s update done \n" % filename) + + +def upgrade_env_script(): + """Update env function scripts.""" + + print("Begin to upgrade env scripts.") + env_root = Import('env_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') + + if get_package_url is not None: + env_scripts_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' + else: + env_scripts_repo = 'https://github.com/RT-Thread/env.git' + + env_scripts_root = os.path.join(env_root, 'tools', 'scripts') + git_pull_repo(env_scripts_root, env_scripts_repo) + print("==============================> Env scripts upgrade done \n") + + +def package_upgrade(): + """Update the package repository directory and env function scripts.""" + + upgrade_packages_index() + upgrade_env_script() diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py new file mode 100644 index 00000000..306215ca --- /dev/null +++ b/cmds/cmd_package_utils.py @@ -0,0 +1,113 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import platform +import subprocess +import time +import json +import requests + + +def execute_command(cmdstring, cwd=None, shell=True): + """Execute the system command at the specified address.""" + + if shell: + cmdstring_list = cmdstring + + sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, shell=shell, bufsize=4096) + + stdout_str = '' + while sub.poll() is None: + stdout_str += str(sub.stdout.read()) + time.sleep(0.1) + + return stdout_str + + +def git_pull_repo(repo_path, repo_url=''): + if platform.system() == "Windows": + cmd = r'git config --local core.autocrlf true' + execute_command(cmd, cwd=repo_path) + cmd = r'git pull ' + repo_url + execute_command(cmd, cwd=repo_path) + + +def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): + """Get the download address from the mirror server based on the package name.""" + + try: + if type(pkgs_name_in_json) != type("str"): + if sys.version_info < (3, 0): + pkgs_name_in_json = str(pkgs_name_in_json) + else: + pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] + except Exception as e: + print('error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None + + payload = { + "userName": "RT-Thread", + "packages": [ + { + "name": "NULL", + } + ] + } + payload["packages"][0]['name'] = pkgs_name_in_json + + # print(payload) + + try: + r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) + + if r.status_code == requests.codes.ok: + package_info = json.loads(r.text) + + # Can't find package,change git package SHA if it's a git + # package + if len(package_info['packages']) == 0: + print("Package was NOT found on mirror server. Using a non-mirrored address to download.") + return None, None + else: + for item in package_info['packages'][0]['packages_info']['site']: + if item['version'] == pkgs_ver: + # Change download url + download_url = item['URL'] + if download_url[-4:] == '.git': + # Change git package SHA + repo_sha = item['VER_SHA'] + return download_url, repo_sha + return download_url, None + + print("\nTips : \nThe system needs to be upgraded.") + print("Please use the command to upgrade packages index.\n") + return None, None + + except Exception as e: + print('error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py index 0a3e6a8e..10808d83 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package_wizard.py @@ -2,7 +2,7 @@ # # File : cmd_package.py # This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From e4ed8412165b5f00e4a63099fd5cca2bda117002 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:31:05 +0800 Subject: [PATCH 11/48] [update] cmd_package.py --- cmds/cmd_package.py | 6 ++++-- cmds/cmd_package_utils.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 1d6dd09a..eb493fe4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -23,6 +23,7 @@ # 2018-05-28 SummerGift Add copyright information # 2018-12-28 Ernest Chen Add package information and enjoy package maker # 2019-01-07 SummerGift The prompt supports utf-8 encoding +# 2020-04-08 SummerGift Optimize program structure # import os @@ -31,17 +32,18 @@ import pkgsdb import shutil import platform -import subprocess import time import archive import sys -from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file +import requests +from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo def determine_support_chinese(env_root): diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 306215ca..5a8944a3 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -27,6 +27,7 @@ import subprocess import time import json +import sys import requests From cfda04f5d74d77474344cf13b96728dffeff50d5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:41:14 +0800 Subject: [PATCH 12/48] [optimize] error message --- cmds/cmd_package.py | 12 ++++++------ cmds/cmd_package_utils.py | 25 ++++++++++--------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index eb493fe4..1b807767 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -100,7 +100,7 @@ def modify_submod_file_to_mirror(submod_path): return replace_list except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) def determine_url_valid(url_from_srv): @@ -123,7 +123,7 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -179,7 +179,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = True except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") if package_url[-4:] == '.git': @@ -258,7 +258,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): except Exception as e: os.remove(package_path) ret = False - print('error message: %s\t' % e) + print('Error message: %s\t' % e) else: print("The file does not exist.") return ret @@ -309,7 +309,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -364,7 +364,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): git_cmd_exec(cmd, repo_path) except Exception as e: - print("error message : %s" % e) + print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") # Update the package repository from upstream. diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 5a8944a3..80b50e32 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -31,13 +31,10 @@ import requests -def execute_command(cmdstring, cwd=None, shell=True): +def execute_command(cmd_string, cwd=None, shell=True): """Execute the system command at the specified address.""" - if shell: - cmdstring_list = cmdstring - - sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, + sub = subprocess.Popen(cmd_string, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=shell, bufsize=4096) stdout_str = '' @@ -56,17 +53,17 @@ def git_pull_repo(repo_path, repo_url=''): execute_command(cmd, cwd=repo_path) -def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): +def get_url_from_mirror_server(package_name, package_version): """Get the download address from the mirror server based on the package name.""" try: - if type(pkgs_name_in_json) != type("str"): + if isinstance(package_name, str): if sys.version_info < (3, 0): - pkgs_name_in_json = str(pkgs_name_in_json) + package_name = str(package_name) else: - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] + package_name = str(package_name)[2:-1] except Exception as e: - print('error message:%s' % e) + print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None @@ -78,9 +75,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } ] } - payload["packages"][0]['name'] = pkgs_name_in_json - - # print(payload) + payload["packages"][0]['name'] = package_name try: r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) @@ -95,7 +90,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None else: for item in package_info['packages'][0]['packages_info']['site']: - if item['version'] == pkgs_ver: + if item['version'] == package_version: # Change download url download_url = item['URL'] if download_url[-4:] == '.git': @@ -109,6 +104,6 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None except Exception as e: - print('error message:%s' % e) + print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None From 4a7a9ffd5170e0522fc73193c916247e1ec39ec5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 11:34:27 +0800 Subject: [PATCH 13/48] [optimize] extract user_input to cmd_package_utils.py --- cmds/cmd_package.py | 12 ++-- cmds/cmd_package_utils.py | 17 ++++++ cmds/cmd_package_wizard.py | 114 ++++++++++++++----------------------- 3 files changed, 64 insertions(+), 79 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 1b807767..54a074b4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -43,7 +43,7 @@ from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input def determine_support_chinese(env_root): @@ -663,19 +663,15 @@ def package_update(isDeleteOld=False): print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) if isDeleteOld: - if rm_package(gitdir) == False: + if not rm_package(gitdir): print("Floder delete fail: %s" % gitdir.encode("utf-8")) print("Please delete this folder manually.") else: print("The folder is managed by git. Do you want to delete this folder?\n") - if sys.version_info < (3, 0): - rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - else: - rc = input('Press the Y Key to delete the folder or just press Enter to keep it : ') - + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': try: - if rm_package(gitdir) == False: + if not rm_package(gitdir): pkgs_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 80b50e32..5b56d0bf 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -107,3 +107,20 @@ def get_url_from_mirror_server(package_name, package_version): print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None + + +def user_input(msg=None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg is not None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg is not None: + value = input(msg) + else: + value = input() + + return value diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py index 10808d83..949a3ef6 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package_wizard.py @@ -28,41 +28,7 @@ import sys from package import Kconfig_file, Package_json_file from string import Template - - -def user_input(msg, default_value): - """Gets the user's keyboard input.""" - - if default_value != '': - msg = '%s[%s]' % (msg, default_value) - - print(msg) - if sys.version_info < (3, 0): - value = raw_input() - else: - value = input() - - if value == '': - value = default_value - - return value - - -def union_input(msg=None): - """Gets the union keyboard input.""" - - if sys.version_info < (3, 0): - if msg is not None: - value = raw_input(msg) - else: - value = raw_input() - else: - if msg is not None: - value = input(msg) - else: - value = input() - - return value +from .cmd_package_utils import user_input def package_wizard(): @@ -70,6 +36,7 @@ def package_wizard(): The user enters the package name, version number, category, and automatically generates the package index file. """ + # Welcome print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') @@ -81,80 +48,85 @@ def package_wizard(): # first step print('\033[5;33;40m\n1.Please input a new package name :\033[0m') - name = union_input() + name = user_input() regular_obj = re.compile('\W') while name == '' or name.isspace() == True or regular_obj.search(name.strip()): if name == '' or name.isspace(): print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = union_input() + name = user_input() else: print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') - name = union_input() + name = user_input() default_description = 'Please add description of ' + name + ' in English.' - # description = user_input('menuconfig option name,default:\n',default_description) description = default_description description_zh = "请添加软件包 " + name + " 的中文描述。" # second step - ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') + print("\033[5;33;40m\n2.Please input this package version, default : '1.0.0' \033[0m") + ver = user_input() + if ver == '': + print("using default version 1.0.0") + ver = '1.0.0' + ver_standard = ver.replace('.', '') - # keyword = user_input('keyword,default:\n', name) keyword = name # third step - packageclass = ('iot', 'language', 'misc', 'multimedia', - 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') + package_class_list = ('iot', 'language', 'misc', 'multimedia', + 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print( - "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = union_input() - while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: - if classnu == '': + print("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|" + "[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + + class_number = user_input() + while class_number == '' or class_number.isdigit() is False or int(class_number) < 1 or int(class_number) > 9: + if class_number == '': print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') else: print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = union_input() + class_number = user_input() - pkgsclass = packageclass[int(classnu) - 1] + package_class = package_class_list[int(class_number) - 1] # fourth step print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") - authorname = union_input() - while authorname == '': + author_name = user_input() + while author_name == '': print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") - authorname = union_input() + author_name = user_input() # fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') - while authoremail == '': + author_email = user_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + while author_email == '': print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() + author_email = user_input() - # sixth step + # sixth step print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = union_input() + license_class = user_input() while license_class == '': print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() + license_class = user_input() - if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: - license = license_index[int(license_class) - 1] + if license_class.isdigit() and 1 <= int(license_class) <= 4: + license_choice = license_index[int(license_class) - 1] else: - license = license_class + license_choice = license_class - # seventh step + # seventh step print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') print( - "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + "\033[1;32;40mFor example, hello package's repository url " + "is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - repository = union_input() + repository = user_input() while repository == '': print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() + repository = user_input() pkg_path = name if not os.path.exists(pkg_path): @@ -163,17 +135,17 @@ def package_wizard(): print("\033[1;31;40mError: the package directory is exits!\033[0m") s = Template(Kconfig_file) - uppername = str.upper(name) - kconfig = s.substitute(name=uppername, description=description, version=ver, - pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) + upper_name = str.upper(name) + kconfig = s.substitute(name=upper_name, description=description, version=ver, + pkgs_class=package_class, lowercase_name=name, version_standard=ver_standard) f = open(os.path.join(pkg_path, 'Kconfig'), 'w') f.write(kconfig) f.close() s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + package = s.substitute(name=name, pkgsclass=package_class, authorname=author_name, authoremail=author_email, description=description, description_zh=description_zh, version=ver, keyword=keyword, - license=license, repository=repository, pkgs_using_name=uppername) + license=license_choice, repository=repository, pkgs_using_name=upper_name) f = open(os.path.join(pkg_path, 'package.json'), 'w') f.write(package) f.close() From 9b26ef42ae6854ac7c645c39eebc3186cbac0357 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 11:50:24 +0800 Subject: [PATCH 14/48] [optimize] cmds/cmd_package.py --- cmds/cmd_package.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 54a074b4..b3e8a6f7 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -34,7 +34,6 @@ import platform import time import archive -import sys import requests from package import Package, Bridge_SConscript from vars import Import, Export @@ -687,26 +686,22 @@ def package_update(isDeleteOld=False): print('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) + # write error messages + with open(pkgs_error_list_fn, 'w') as f: + f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + if len(pkgs_delete_fail_list): - # write error messages - pkgs_file = file(pkgs_error_list_fn, 'w') - pkgs_file.write(json.dumps(pkgs_delete_fail_list, indent=1)) - pkgs_file.close() return - else: - # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) # 2.in new not in old : Software packages to be installed. # If the package download fails, record it, and then download again when # the update command is executed. - casedownload = sub_list(newpkgs, oldpkgs) + case_download = sub_list(newpkgs, oldpkgs) # print 'in new not in old:', casedownload pkgs_download_fail_list = [] - for pkg in casedownload: + for pkg in case_download: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) From 01a62b46ed5f35bf6533daf99c57e63a01052977 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 14:03:14 +0800 Subject: [PATCH 15/48] [optimize] add cmd_package_update.py and code clear up --- cmds/cmd_package.py | 733 +--------------------------------- cmds/cmd_package_printenv.py | 4 + cmds/cmd_package_update.py | 737 +++++++++++++++++++++++++++++++++++ 3 files changed, 750 insertions(+), 724 deletions(-) create mode 100644 cmds/cmd_package_update.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index b3e8a6f7..1ec8fe9b 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -2,7 +2,7 @@ # # File : cmd_package.py # This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,729 +26,17 @@ # 2020-04-08 SummerGift Optimize program structure # -import os -import json -import kconfig -import pkgsdb -import shutil -import platform -import time -import archive -import requests -from package import Package, Bridge_SConscript -from vars import Import, Export -from .cmd_menuconfig import find_macro_in_config +from .cmd_package_printenv import package_print_env, package_print_help from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard -from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input +from .cmd_package_update import package_update -def determine_support_chinese(env_root): - get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') - if os.path.isfile(get_flag_file_path): - return True - else: - return False - - -def get_mirror_giturl(submod_name): - """Gets the submodule's url on mirror server. - - Retrurn the download address of the submodule on the mirror server from the submod_name. - """ - - mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' - return mirror_url - - -def modify_submod_file_to_mirror(submod_path): - """Modify the.gitmodules file based on the submodule to be updated""" - - replace_list = [] - try: - with open(submod_path, 'r') as f: - for line in f: - line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') - if line.startswith('url'): - submod_git_url = line.split('=')[1] - submodule_name = submod_git_url.split('/')[-1].replace('.git', '') - replace_url = get_mirror_giturl(submodule_name) - # print(replace_url) - query_submodule_name = 'submod_' + submodule_name - # print(query_submodule_name) - get_package_url, get_ver_sha = get_url_from_mirror_server( - query_submodule_name, 'latest') - - if get_package_url != None and determine_url_valid(get_package_url): - replace_list.append( - (submod_git_url, replace_url, submodule_name)) - - with open(submod_path, 'r+') as f: - submod_file_count = f.read() - - write_content = submod_file_count - - for item in replace_list: - write_content = write_content.replace(item[0], item[1]) - - with open(submod_path, 'w') as f: - f.write(str(write_content)) - - return replace_list - - except Exception as e: - print('Error message:%s\t' % e) - - -def determine_url_valid(url_from_srv): - headers = {'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip, deflate', - 'Accept': '*/*', - 'User-Agent': 'curl/7.54.0'} - - try: - for i in range(0, 3): - r = requests.get(url_from_srv, stream=True, headers=headers) - if r.status_code == requests.codes.not_found: - if i == 2: - print("Warning : %s is invalid." % url_from_srv) - return False - time.sleep(1) - else: - break - - return True - - except Exception as e: - print('Error message:%s\t' % e) - print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) - - -def install_pkg(env_root, pkgs_root, bsp_root, pkg): - """Install the required packages.""" - - # default true - ret = True - local_pkgs_path = os.path.join(env_root, 'local_pkgs') - bsp_pkgs_path = os.path.join(bsp_root, 'packages') - - # get the .config file from env - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - url_from_json = package.get_url(pkg['ver']) - package_url = package.get_url(pkg['ver']) - pkgs_name_in_json = package.get_name() - - if package_url[-4:] == '.git': - ver_sha = package.get_versha(pkg['ver']) - - # print("==================================================>") - # print("packages name :"%pkgs_name_in_json.encode("utf-8")) - # print("ver :"%pkg['ver']) - # print("url :"%package_url.encode("utf-8")) - # print("url_from_json : "%url_from_json.encode("utf-8")) - # print("==================================================>") - - get_package_url = None - get_ver_sha = None - upstream_change_flag = False - - try: - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - - # Check whether the package package url is valid - if get_package_url is not None and determine_url_valid(get_package_url): - package_url = get_package_url - - if get_ver_sha is not None: - ver_sha = get_ver_sha - - upstream_change_flag = True - except Exception as e: - print('Error message:%s\t' % e) - print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - - if package_url[-4:] == '.git': - try: - repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) - repo_path = repo_path + '-' + pkg['ver'] - repo_path_full = '"' + repo_path + '"' - - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(clone_cmd, cwd=bsp_pkgs_path) - - git_check_cmd = 'git checkout -q ' + ver_sha - execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: - print("\nFailed to download software package with git. Please check the network connection.") - return False - - if upstream_change_flag: - cmd = 'git remote set-url origin ' + url_from_json - execute_command(cmd, cwd=repo_path) - - # If there is a .gitmodules file in the package, prepare to update submodule. - submodule_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submodule_path): - print("Start to update submodule") - # print("开始更新软件包子模块") - - if (not os.path.isfile(env_config_file)) \ - or (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - # print("开启了镜像加速，开始修改 .gitmodules 文件") - replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file - - # print("开始执行更新动作") - cmd = 'git submodule update --init --recursive' - execute_command(cmd, cwd=repo_path) - - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and - find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - if len(replace_list): - for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): - cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) - - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - if os.path.isfile(submodule_path): - cmd = 'git checkout .gitmodules' - execute_command(cmd, cwd=repo_path) - else: - # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path, package_url): - return False - - pkg_dir = package.get_filename(pkg['ver']) - pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - - if not archive.packtest(package_path): - print("package : %s is invalid" % package_path.encode("utf-8")) - return False - - # unpack package - if not os.path.exists(pkg_dir): - - try: - if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): - ret = False - except Exception as e: - os.remove(package_path) - ret = False - print('Error message: %s\t' % e) - else: - print("The file does not exist.") - return ret - - -def sub_list(aList, bList): - """Return the items in aList but not in bList.""" - - tmp = [] - for a in aList: - if a not in bList: - tmp.append(a) - return tmp - - -def and_list(aList, bList): - """Return the items in aList and in bList.""" - - tmp = [] - for a in aList: - if a in bList: - tmp.append(a) - return tmp - - -def update_submodule(repo_path): - """Update the submodules in the repository.""" - - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): - print("Please wait a few seconds in order to update the submodule.") - cmd = 'git submodule init -q' - execute_command(cmd, cwd=repo_path) - cmd = 'git submodule update' - execute_command(cmd, cwd=repo_path) - print("Submodule update successful") - - -def get_pkg_folder_by_orign_path(orign_path, version): - # TODO fix for old version project, will remove after new major version - # release - if os.path.exists(orign_path + '-' + version): - return orign_path + '-' + version - return orign_path - - -def git_cmd_exec(cmd, cwd): - try: - execute_command(cmd, cwd=cwd) - except Exception as e: - print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) - print("You can solve this problem by manually removing old packages and re-downloading them using env.") - - -def update_latest_packages(pkgs_fn, bsp_packages_path): - """ update the packages that are latest version. - - If the selected package is the latest version, - check to see if it is the latest version after the update command, - if not, then update the latest version from the remote repository. - If the download has a conflict, you are currently using the prompt - message provided by git. - """ - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - for pkg in read_back_pkgs_json: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - pkgs_name_in_json = package.get_name() - - # Find out the packages which version is 'latest' - if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": - repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) - repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) - - try: - # If mirror acceleration is enabled, get the update address from - # the mirror server. - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") - - # Change repo's upstream address. - mirror_url = get_url_from_mirror_server( - payload_pkgs_name_in_json, pkg['ver']) - - if mirror_url[0] is not None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) - - except Exception as e: - print("Error message : %s" % e) - print("Failed to connect to the mirror server, using non-mirror server to update.") - - # Update the package repository from upstream. - git_pull_repo(repo_path) - - # If the package has submodules, update the submodules. - update_submodule(repo_path) - - # recover origin url to the path which get from packages.json file - if package.get_url(pkg['ver']): - cmd = 'git remote set-url origin ' + \ - package.get_url(pkg['ver']) - git_cmd_exec(cmd, repo_path) - else: - print("Can't find the package : %s's url in file : %s" % - (payload_pkgs_name_in_json, pkg_path)) - - print("==============================> %s update done\n" % (pkgs_name_in_json)) - - -def pre_package_update(): - """ Make preparations before updating the software package. """ - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - - if not os.path.exists('.config'): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print('No system configuration file : .config.') - print('You should use < menuconfig > command to config bsp first.') +def run_env_cmd(args): + """Run packages command.""" - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return False - - bsp_packages_path = os.path.join(bsp_root, 'packages') - if not os.path.exists(bsp_packages_path): - os.mkdir("packages") - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # prepare target packages file - dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') - Export('dbsqlite_pathname') - dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - - # Avoid creating tables more than one time - if not os.path.isfile(dbsqlite_pathname): - conn = pkgsdb.get_conn(dbsqlite_pathname) - sql = '''CREATE TABLE packagefile - (pathname TEXT ,package TEXT ,md5 TEXT );''' - pkgsdb.create_table(conn, sql) - - fn = '.config' - pkgs = kconfig.parse(fn) - - # print("newpkgs", pkgs) - - newpkgs = pkgs - - if not os.path.exists(bsp_packages_path): - os.mkdir(bsp_packages_path) - - pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') - - # regenerate file : packages/pkgs.json - if not os.path.exists(pkgs_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs.json - with open(pkgs_fn, 'r') as f: - oldpkgs = json.load(f) - - # print("oldpkgs", oldpkgs) - - # regenerate file : packages/pkgs_error.json - pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') - - if not os.path.exists(pkgs_error_list_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs_error.json - with open(pkgs_error_list_fn, 'r') as f: - pkgs_error = json.load(f) - - # create SConscript file - if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: - f.write(str(Bridge_SConscript)) - - return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] - - -def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - flag = None - - error_packages_redownload_error_list = [] - - if len(error_packages_list): - print("\n==============================> Packages list to download : \n") - for pkg in error_packages_list: - print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - print("\nThe packages in the list above are accidentally deleted, env will redownload them.") - print("Warning: Packages should be deleted in command.\n") - - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is redownloaded successfully. \n" % ( - pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - else: - error_packages_redownload_error_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - if len(error_packages_redownload_error_list): - print("%s" % error_packages_redownload_error_list) - print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_pkgs_json = sub_list( - read_back_pkgs_json, error_packages_redownload_error_list) - read_back_pkgs_json = write_back_pkgs_json - # print("write_back_pkgs_json:%s"%write_back_pkgs_json) - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) - pkgs_file.close() - - return flag - - -def rm_package(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rd /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rmdir /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - print("Folder path: %s" % dir.encode("utf-8")) - return False - else: - print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) - return True - - -def get_package_remove_path(pkg, bsp_packages_path): - dirpath = pkg['path'] - ver = pkg['ver'] - if dirpath[0] == '/' or dirpath[0] == '\\': - dirpath = dirpath[1:] - - if platform.system() == "Windows": - dirpath = os.path.basename(dirpath.replace('/', '\\')) - else: - dirpath = os.path.basename(dirpath) - - removepath = os.path.join(bsp_packages_path, dirpath) - - # Handles the deletion of git repository folders with version Numbers - removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) - return removepath_ver - - -def handle_download_error_packages(pkgs_fn, bsp_packages_path): - """ handle download error packages. - - Check to see if the packages stored in the Json file list actually exist, - and then download the packages if they don't exist. - """ - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - error_packages_list = [] - - for pkg in read_back_pkgs_json: - removepath = get_package_remove_path(pkg, bsp_packages_path) - - if os.path.exists(removepath): - continue - else: - error_packages_list.append(pkg) - - # Handle the failed download packages - get_flag = error_packages_handle( - error_packages_list, read_back_pkgs_json, pkgs_fn) - - return get_flag - - -def write_storage_file(pkgs_fn, newpkgs): - """Writes the updated configuration to pkgs.json file. - - Packages that are not downloaded correctly will be redownloaded at the - next update. - """ - - with open(pkgs_fn, 'w') as f: - f.write(str(json.dumps(newpkgs, indent=1))) - - -def package_update(isDeleteOld=False): - """Update env's packages. - - Compare the old and new software package list and update the package. - Remove unwanted packages and download the newly selected package.- - Check if the files in the deleted packages have been changed, and if so, - remind the user saved the modified file. - """ - - sys_value = pre_package_update() - - if not sys_value: - return - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - flag = True - - # According to the env version, whether Chinese output is supported or not - if determine_support_chinese(env_root): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - oldpkgs = sys_value[0] - newpkgs = sys_value[1] - pkgs_delete_error_list = sys_value[2] - pkgs_fn = sys_value[3] - pkgs_error_list_fn = sys_value[4] - bsp_packages_path = sys_value[5] - dbsqlite_pathname = sys_value[6] - - # print(oldpkgs) - # print(newpkgs) - - if len(pkgs_delete_error_list): - for error_package in pkgs_delete_error_list: - removepath_ver = get_package_remove_path( - error_package, bsp_packages_path) - - if os.path.isdir(removepath_ver): - print("\nError: %s package delete failed, begin to remove it." % - error_package['name'].encode("utf-8")) - - if not rm_package(removepath_ver): - print("Error: Delete package %s failed! Please delete the folder manually.\n" % - error_package['name'].encode("utf-8")) - return - - # 1.in old ,not in new : Software packages that need to be removed. - casedelete = sub_list(oldpkgs, newpkgs) - pkgs_delete_fail_list = [] - - for pkg in casedelete: - - removepath_ver = get_package_remove_path(pkg, bsp_packages_path) - removepath_git = os.path.join(removepath_ver, '.git') - - # Delete. Git directory. - if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): - gitdir = removepath_ver - - print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) - if isDeleteOld: - if not rm_package(gitdir): - print("Floder delete fail: %s" % gitdir.encode("utf-8")) - print("Please delete this folder manually.") - else: - print("The folder is managed by git. Do you want to delete this folder?\n") - rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - if rc == 'y' or rc == 'Y': - try: - if not rm_package(gitdir): - pkgs_delete_fail_list.append(pkg) - print("Error: Please delete the folder manually.") - except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", gitdir.encode("utf-8"), e)) - else: - if os.path.isdir(removepath_ver): - print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) - try: - pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) - except Exception as e: - pkgs_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) - - # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) - - if len(pkgs_delete_fail_list): - return - - # 2.in new not in old : Software packages to be installed. - # If the package download fails, record it, and then download again when - # the update command is executed. - - case_download = sub_list(newpkgs, oldpkgs) - # print 'in new not in old:', casedownload - pkgs_download_fail_list = [] - - for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is downloaded successfully. \n" % ( - pkg['name'], pkg['ver'])) - else: - # If the PKG download fails, record it in the - # pkgs_download_fail_list. - pkgs_download_fail_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - # Get the currently updated configuration. - newpkgs = sub_list(newpkgs, pkgs_download_fail_list) - - # Give hints based on the success of the download. - if len(pkgs_download_fail_list): - print("\nPackage download failed list:") - for item in pkgs_download_fail_list: - print(item) - - print("You need to reuse the command to download again.") - - # update pkgs.json and SConscript - write_storage_file(pkgs_fn, newpkgs) - - # handle download error packages. - get_flag = handle_download_error_packages( - pkgs_fn, bsp_packages_path) - - if get_flag is not None: - flag = get_flag - - # Update the software packages, which the version is 'latest' - try: - update_latest_packages(pkgs_fn, bsp_packages_path) - except KeyboardInterrupt: - flag = False - - if flag: - print("Operation completed successfully.") - else: - print("Operation failed.") - - -def cmd(args): - """Env's packages command execution options.""" - - if args.package_update_y: + if args.package_update_force: package_update(True) elif args.package_update: package_update() @@ -761,7 +49,7 @@ def cmd(args): elif args.package_print_env: package_print_env() else: - os.system('pkgs -h') + package_print_help() def add_parser(sub): @@ -773,7 +61,7 @@ def add_parser(sub): help='force update and clean packages, install or remove packages by settings in menuconfig', action='store_true', default=False, - dest='package_update_y') + dest='package_update_force') parser.add_argument('--update', help='update packages, install or remove the packages by your settings in menuconfig', @@ -805,7 +93,4 @@ def add_parser(sub): default=False, dest='package_print_env') - # parser.add_argument('--upgrade', dest='reposource', required=False, - # help='add source & update packages repo ') - - parser.set_defaults(func=cmd) + parser.set_defaults(func=run_env_cmd) diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package_printenv.py index f3e1a529..377ead46 100644 --- a/cmds/cmd_package_printenv.py +++ b/cmds/cmd_package_printenv.py @@ -41,3 +41,7 @@ def package_print_env(): env_root = os.path.join(os.getenv('HOME'), '.env') print("ENV_ROOT:%s" % (env_root)) + + +def package_print_help(): + os.system('pkgs -h') diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py new file mode 100644 index 00000000..fb7b3274 --- /dev/null +++ b/cmds/cmd_package_update.py @@ -0,0 +1,737 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import json +import kconfig +import pkgsdb +import shutil +import platform +import time +import archive +import requests +from package import Package, Bridge_SConscript +from vars import Import, Export +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input + + +def determine_support_chinese(env_root): + get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') + if os.path.isfile(get_flag_file_path): + return True + else: + return False + + +def get_mirror_giturl(submod_name): + """Gets the submodule's url on mirror server. + + Retrurn the download address of the submodule on the mirror server from the submod_name. + """ + + mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' + return mirror_url + + +def modify_submod_file_to_mirror(submod_path): + """Modify the.gitmodules file based on the submodule to be updated""" + + replace_list = [] + try: + with open(submod_path, 'r') as f: + for line in f: + line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') + if line.startswith('url'): + submod_git_url = line.split('=')[1] + submodule_name = submod_git_url.split('/')[-1].replace('.git', '') + replace_url = get_mirror_giturl(submodule_name) + # print(replace_url) + query_submodule_name = 'submod_' + submodule_name + # print(query_submodule_name) + get_package_url, get_ver_sha = get_url_from_mirror_server( + query_submodule_name, 'latest') + + if get_package_url != None and determine_url_valid(get_package_url): + replace_list.append( + (submod_git_url, replace_url, submodule_name)) + + with open(submod_path, 'r+') as f: + submod_file_count = f.read() + + write_content = submod_file_count + + for item in replace_list: + write_content = write_content.replace(item[0], item[1]) + + with open(submod_path, 'w') as f: + f.write(str(write_content)) + + return replace_list + + except Exception as e: + print('Error message:%s\t' % e) + + +def determine_url_valid(url_from_srv): + headers = {'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + 'Accept': '*/*', + 'User-Agent': 'curl/7.54.0'} + + try: + for i in range(0, 3): + r = requests.get(url_from_srv, stream=True, headers=headers) + if r.status_code == requests.codes.not_found: + if i == 2: + print("Warning : %s is invalid." % url_from_srv) + return False + time.sleep(1) + else: + break + + return True + + except Exception as e: + print('Error message:%s\t' % e) + print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) + + +def install_pkg(env_root, pkgs_root, bsp_root, pkg): + """Install the required packages.""" + + # default true + ret = True + local_pkgs_path = os.path.join(env_root, 'local_pkgs') + bsp_pkgs_path = os.path.join(bsp_root, 'packages') + + # get the .config file from env + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + url_from_json = package.get_url(pkg['ver']) + package_url = package.get_url(pkg['ver']) + pkgs_name_in_json = package.get_name() + + if package_url[-4:] == '.git': + ver_sha = package.get_versha(pkg['ver']) + + # print("==================================================>") + # print("packages name :"%pkgs_name_in_json.encode("utf-8")) + # print("ver :"%pkg['ver']) + # print("url :"%package_url.encode("utf-8")) + # print("url_from_json : "%url_from_json.encode("utf-8")) + # print("==================================================>") + + get_package_url = None + get_ver_sha = None + upstream_change_flag = False + + try: + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) + + # Check whether the package package url is valid + if get_package_url is not None and determine_url_valid(get_package_url): + package_url = get_package_url + + if get_ver_sha is not None: + ver_sha = get_ver_sha + + upstream_change_flag = True + except Exception as e: + print('Error message:%s\t' % e) + print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + + if package_url[-4:] == '.git': + try: + repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) + repo_path = repo_path + '-' + pkg['ver'] + repo_path_full = '"' + repo_path + '"' + + clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + execute_command(clone_cmd, cwd=bsp_pkgs_path) + + git_check_cmd = 'git checkout -q ' + ver_sha + execute_command(git_check_cmd, cwd=repo_path) + except Exception as e: + print("\nFailed to download software package with git. Please check the network connection.") + return False + + if upstream_change_flag: + cmd = 'git remote set-url origin ' + url_from_json + execute_command(cmd, cwd=repo_path) + + # If there is a .gitmodules file in the package, prepare to update submodule. + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): + print("Start to update submodule") + # print("开始更新软件包子模块") + + if (not os.path.isfile(env_config_file)) \ + or (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + # print("开启了镜像加速，开始修改 .gitmodules 文件") + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file + + # print("开始执行更新动作") + cmd = 'git submodule update --init --recursive' + execute_command(cmd, cwd=repo_path) + + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and + find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if len(replace_list): + for item in replace_list: + submod_dir_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submod_dir_path): + cmd = 'git remote set-url origin ' + item[0] + execute_command(cmd, cwd=submod_dir_path) + + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if os.path.isfile(submodule_path): + cmd = 'git checkout .gitmodules' + execute_command(cmd, cwd=repo_path) + else: + # Download a package of compressed package type. + if not package.download(pkg['ver'], local_pkgs_path, package_url): + return False + + pkg_dir = package.get_filename(pkg['ver']) + pkg_dir = os.path.splitext(pkg_dir)[0] + package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + + if not archive.packtest(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) + return False + + # unpack package + if not os.path.exists(pkg_dir): + + try: + if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): + ret = False + except Exception as e: + os.remove(package_path) + ret = False + print('Error message: %s\t' % e) + else: + print("The file does not exist.") + return ret + + +def sub_list(aList, bList): + """Return the items in aList but not in bList.""" + + tmp = [] + for a in aList: + if a not in bList: + tmp.append(a) + return tmp + + +def and_list(aList, bList): + """Return the items in aList and in bList.""" + + tmp = [] + for a in aList: + if a in bList: + tmp.append(a) + return tmp + + +def update_submodule(repo_path): + """Update the submodules in the repository.""" + + submod_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submod_path): + print("Please wait a few seconds in order to update the submodule.") + cmd = 'git submodule init -q' + execute_command(cmd, cwd=repo_path) + cmd = 'git submodule update' + execute_command(cmd, cwd=repo_path) + print("Submodule update successful") + + +def get_pkg_folder_by_orign_path(orign_path, version): + # TODO fix for old version project, will remove after new major version + # release + if os.path.exists(orign_path + '-' + version): + return orign_path + '-' + version + return orign_path + + +def git_cmd_exec(cmd, cwd): + try: + execute_command(cmd, cwd=cwd) + except Exception as e: + print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + print("You can solve this problem by manually removing old packages and re-downloading them using env.") + + +def update_latest_packages(pkgs_fn, bsp_packages_path): + """ update the packages that are latest version. + + If the selected package is the latest version, + check to see if it is the latest version after the update command, + if not, then update the latest version from the remote repository. + If the download has a conflict, you are currently using the prompt + message provided by git. + """ + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + + with open(pkgs_fn, 'r') as f: + read_back_pkgs_json = json.load(f) + + for pkg in read_back_pkgs_json: + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + pkgs_name_in_json = package.get_name() + + # Find out the packages which version is 'latest' + if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": + repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) + repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) + + try: + # If mirror acceleration is enabled, get the update address from + # the mirror server. + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") + + # Change repo's upstream address. + mirror_url = get_url_from_mirror_server( + payload_pkgs_name_in_json, pkg['ver']) + + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + + except Exception as e: + print("Error message : %s" % e) + print("Failed to connect to the mirror server, using non-mirror server to update.") + + # Update the package repository from upstream. + git_pull_repo(repo_path) + + # If the package has submodules, update the submodules. + update_submodule(repo_path) + + # recover origin url to the path which get from packages.json file + if package.get_url(pkg['ver']): + cmd = 'git remote set-url origin ' + \ + package.get_url(pkg['ver']) + git_cmd_exec(cmd, repo_path) + else: + print("Can't find the package : %s's url in file : %s" % + (payload_pkgs_name_in_json, pkg_path)) + + print("==============================> %s update done\n" % (pkgs_name_in_json)) + + +def pre_package_update(): + """ Make preparations before updating the software package. """ + + bsp_root = Import('bsp_root') + env_root = Import('env_root') + + if not os.path.exists('.config'): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + + print('No system configuration file : .config.') + print('You should use < menuconfig > command to config bsp first.') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return False + + bsp_packages_path = os.path.join(bsp_root, 'packages') + if not os.path.exists(bsp_packages_path): + os.mkdir("packages") + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # prepare target packages file + dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') + Export('dbsqlite_pathname') + dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') + + # Avoid creating tables more than one time + if not os.path.isfile(dbsqlite_pathname): + conn = pkgsdb.get_conn(dbsqlite_pathname) + sql = '''CREATE TABLE packagefile + (pathname TEXT ,package TEXT ,md5 TEXT );''' + pkgsdb.create_table(conn, sql) + + fn = '.config' + pkgs = kconfig.parse(fn) + + # print("newpkgs", pkgs) + + newpkgs = pkgs + + if not os.path.exists(bsp_packages_path): + os.mkdir(bsp_packages_path) + + pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') + + # regenerate file : packages/pkgs.json + if not os.path.exists(pkgs_fn): + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # Reading data back from pkgs.json + with open(pkgs_fn, 'r') as f: + oldpkgs = json.load(f) + + # print("oldpkgs", oldpkgs) + + # regenerate file : packages/pkgs_error.json + pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') + + if not os.path.exists(pkgs_error_list_fn): + os.chdir(bsp_packages_path) + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # Reading data back from pkgs_error.json + with open(pkgs_error_list_fn, 'r') as f: + pkgs_error = json.load(f) + + # create SConscript file + if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): + with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: + f.write(str(Bridge_SConscript)) + + return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + + +def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): + bsp_root = Import('bsp_root') + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + flag = None + + error_packages_redownload_error_list = [] + + if len(error_packages_list): + print("\n==============================> Packages list to download : \n") + for pkg in error_packages_list: + print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("\nThe packages in the list above are accidentally deleted, env will redownload them.") + print("Warning: Packages should be deleted in command.\n") + + for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + if install_pkg(env_root, pkgs_root, bsp_root, pkg): + print("==============================> %s %s is redownloaded successfully. \n" % ( + pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + else: + error_packages_redownload_error_list.append(pkg) + print(pkg, 'download failed.') + flag = False + + if len(error_packages_redownload_error_list): + print("%s" % error_packages_redownload_error_list) + print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % + (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + write_back_pkgs_json = sub_list( + read_back_pkgs_json, error_packages_redownload_error_list) + read_back_pkgs_json = write_back_pkgs_json + # print("write_back_pkgs_json:%s"%write_back_pkgs_json) + pkgs_file = file(pkgs_fn, 'w') + pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) + pkgs_file.close() + + return flag + + +def rm_package(dir): + if platform.system() != "Windows": + shutil.rmtree(dir) + else: + dir = '"' + dir + '"' + cmd = 'rd /s /q ' + dir + os.system(cmd) + + if os.path.isdir(dir): + if platform.system() != "Windows": + shutil.rmtree(dir) + else: + dir = '"' + dir + '"' + cmd = 'rmdir /s /q ' + dir + os.system(cmd) + + if os.path.isdir(dir): + print("Folder path: %s" % dir.encode("utf-8")) + return False + else: + print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + return True + + +def get_package_remove_path(pkg, bsp_packages_path): + dirpath = pkg['path'] + ver = pkg['ver'] + if dirpath[0] == '/' or dirpath[0] == '\\': + dirpath = dirpath[1:] + + if platform.system() == "Windows": + dirpath = os.path.basename(dirpath.replace('/', '\\')) + else: + dirpath = os.path.basename(dirpath) + + removepath = os.path.join(bsp_packages_path, dirpath) + + # Handles the deletion of git repository folders with version Numbers + removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) + return removepath_ver + + +def handle_download_error_packages(pkgs_fn, bsp_packages_path): + """ handle download error packages. + + Check to see if the packages stored in the Json file list actually exist, + and then download the packages if they don't exist. + """ + + with open(pkgs_fn, 'r') as f: + read_back_pkgs_json = json.load(f) + + error_packages_list = [] + + for pkg in read_back_pkgs_json: + removepath = get_package_remove_path(pkg, bsp_packages_path) + + if os.path.exists(removepath): + continue + else: + error_packages_list.append(pkg) + + # Handle the failed download packages + get_flag = error_packages_handle( + error_packages_list, read_back_pkgs_json, pkgs_fn) + + return get_flag + + +def write_storage_file(pkgs_fn, newpkgs): + """Writes the updated configuration to pkgs.json file. + + Packages that are not downloaded correctly will be redownloaded at the + next update. + """ + + with open(pkgs_fn, 'w') as f: + f.write(str(json.dumps(newpkgs, indent=1))) + + +def package_update(isDeleteOld=False): + """Update env's packages. + + Compare the old and new software package list and update the package. + Remove unwanted packages and download the newly selected package.- + Check if the files in the deleted packages have been changed, and if so, + remind the user saved the modified file. + """ + + sys_value = pre_package_update() + + if not sys_value: + return + + bsp_root = Import('bsp_root') + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + flag = True + + # According to the env version, whether Chinese output is supported or not + if determine_support_chinese(env_root): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + oldpkgs = sys_value[0] + newpkgs = sys_value[1] + pkgs_delete_error_list = sys_value[2] + pkgs_fn = sys_value[3] + pkgs_error_list_fn = sys_value[4] + bsp_packages_path = sys_value[5] + dbsqlite_pathname = sys_value[6] + + # print(oldpkgs) + # print(newpkgs) + + if len(pkgs_delete_error_list): + for error_package in pkgs_delete_error_list: + removepath_ver = get_package_remove_path( + error_package, bsp_packages_path) + + if os.path.isdir(removepath_ver): + print("\nError: %s package delete failed, begin to remove it." % + error_package['name'].encode("utf-8")) + + if not rm_package(removepath_ver): + print("Error: Delete package %s failed! Please delete the folder manually.\n" % + error_package['name'].encode("utf-8")) + return + + # 1.in old ,not in new : Software packages that need to be removed. + casedelete = sub_list(oldpkgs, newpkgs) + pkgs_delete_fail_list = [] + + for pkg in casedelete: + + removepath_ver = get_package_remove_path(pkg, bsp_packages_path) + removepath_git = os.path.join(removepath_ver, '.git') + + # Delete. Git directory. + if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): + gitdir = removepath_ver + + print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) + if isDeleteOld: + if not rm_package(gitdir): + print("Floder delete fail: %s" % gitdir.encode("utf-8")) + print("Please delete this folder manually.") + else: + print("The folder is managed by git. Do you want to delete this folder?\n") + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + try: + if not rm_package(gitdir): + pkgs_delete_fail_list.append(pkg) + print("Error: Please delete the folder manually.") + except Exception as e: + print('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", gitdir.encode("utf-8"), e)) + else: + if os.path.isdir(removepath_ver): + print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) + try: + pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) + except Exception as e: + pkgs_delete_fail_list.append(pkg) + print('Error message:\n%s %s. %s \n\t' % ( + "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) + + # write error messages + with open(pkgs_error_list_fn, 'w') as f: + f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + + if len(pkgs_delete_fail_list): + return + + # 2.in new not in old : Software packages to be installed. + # If the package download fails, record it, and then download again when + # the update command is executed. + + case_download = sub_list(newpkgs, oldpkgs) + # print 'in new not in old:', casedownload + pkgs_download_fail_list = [] + + for pkg in case_download: + if install_pkg(env_root, pkgs_root, bsp_root, pkg): + print("==============================> %s %s is downloaded successfully. \n" % ( + pkg['name'], pkg['ver'])) + else: + # If the PKG download fails, record it in the + # pkgs_download_fail_list. + pkgs_download_fail_list.append(pkg) + print(pkg, 'download failed.') + flag = False + + # Get the currently updated configuration. + newpkgs = sub_list(newpkgs, pkgs_download_fail_list) + + # Give hints based on the success of the download. + if len(pkgs_download_fail_list): + print("\nPackage download failed list:") + for item in pkgs_download_fail_list: + print(item) + + print("You need to reuse the command to download again.") + + # update pkgs.json and SConscript + write_storage_file(pkgs_fn, newpkgs) + + # handle download error packages. + get_flag = handle_download_error_packages( + pkgs_fn, bsp_packages_path) + + if get_flag is not None: + flag = get_flag + + # Update the software packages, which the version is 'latest' + try: + update_latest_packages(pkgs_fn, bsp_packages_path) + except KeyboardInterrupt: + flag = False + + if flag: + print("Operation completed successfully.") + else: + print("Operation failed.") From 6449f2f6e9acd1a6cc73598c0e6c69597d18de1d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 15:52:36 +0800 Subject: [PATCH 16/48] [optimize] cmds/cmd_package_update.py --- cmds/cmd_package_update.py | 283 ++++++++++++++++++------------------- 1 file changed, 141 insertions(+), 142 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index fb7b3274..12787605 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -35,6 +35,7 @@ from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input +from .cmd_menuconfig import find_macro_in_config def determine_support_chinese(env_root): @@ -45,39 +46,37 @@ def determine_support_chinese(env_root): return False -def get_mirror_giturl(submod_name): +def get_mirror_giturl(submodule_name): """Gets the submodule's url on mirror server. Retrurn the download address of the submodule on the mirror server from the submod_name. """ - mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' + mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submodule_name + '.git' return mirror_url -def modify_submod_file_to_mirror(submod_path): +def modify_submod_file_to_mirror(submodule_path): """Modify the.gitmodules file based on the submodule to be updated""" replace_list = [] try: - with open(submod_path, 'r') as f: + with open(submodule_path, 'r') as f: for line in f: line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') if line.startswith('url'): - submod_git_url = line.split('=')[1] - submodule_name = submod_git_url.split('/')[-1].replace('.git', '') + submodule_git_url = line.split('=')[1] + submodule_name = submodule_git_url.split('/')[-1].replace('.git', '') replace_url = get_mirror_giturl(submodule_name) - # print(replace_url) query_submodule_name = 'submod_' + submodule_name - # print(query_submodule_name) get_package_url, get_ver_sha = get_url_from_mirror_server( query_submodule_name, 'latest') - if get_package_url != None and determine_url_valid(get_package_url): + if get_package_url is not None and determine_url_valid(get_package_url): replace_list.append( - (submod_git_url, replace_url, submodule_name)) + (submodule_git_url, replace_url, submodule_name)) - with open(submod_path, 'r+') as f: + with open(submodule_path, 'r+') as f: submod_file_count = f.read() write_content = submod_file_count @@ -85,7 +84,7 @@ def modify_submod_file_to_mirror(submod_path): for item in replace_list: write_content = write_content.replace(item[0], item[1]) - with open(submod_path, 'w') as f: + with open(submodule_path, 'w') as f: f.write(str(write_content)) return replace_list @@ -397,6 +396,12 @@ def pre_package_update(): return False + # According to the env version, whether Chinese output is supported or not + if determine_support_chinese(env_root): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + # create packages folder bsp_packages_path = os.path.join(bsp_root, 'packages') if not os.path.exists(bsp_packages_path): os.mkdir("packages") @@ -415,7 +420,7 @@ def pre_package_update(): Export('dbsqlite_pathname') dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - # Avoid creating tables more than one time + # avoid creating tables more than one time if not os.path.isfile(dbsqlite_pathname): conn = pkgsdb.get_conn(dbsqlite_pathname) sql = '''CREATE TABLE packagefile @@ -424,18 +429,14 @@ def pre_package_update(): fn = '.config' pkgs = kconfig.parse(fn) - - # print("newpkgs", pkgs) - newpkgs = pkgs if not os.path.exists(bsp_packages_path): os.mkdir(bsp_packages_path) - pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') - # regenerate file : packages/pkgs.json - if not os.path.exists(pkgs_fn): + package_json_filename = os.path.join(bsp_packages_path, 'pkgs.json') + if not os.path.exists(package_json_filename): os.chdir(bsp_packages_path) fp = open("pkgs.json", 'w') fp.write("[]") @@ -443,11 +444,9 @@ def pre_package_update(): os.chdir(bsp_root) # Reading data back from pkgs.json - with open(pkgs_fn, 'r') as f: + with open(package_json_filename, 'r') as f: oldpkgs = json.load(f) - # print("oldpkgs", oldpkgs) - # regenerate file : packages/pkgs_error.json pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') @@ -458,7 +457,7 @@ def pre_package_update(): fp.close() os.chdir(bsp_root) - # Reading data back from pkgs_error.json + # read data back from pkgs_error.json with open(pkgs_error_list_fn, 'r') as f: pkgs_error = json.load(f) @@ -467,17 +466,15 @@ def pre_package_update(): with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) - return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] -def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): +def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename): bsp_root = Import('bsp_root') env_root = Import('env_root') pkgs_root = Import('pkgs_root') - - flag = None - error_packages_redownload_error_list = [] + flag = True if len(error_packages_list): print("\n==============================> Packages list to download : \n") @@ -486,7 +483,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): print("\nThe packages in the list above are accidentally deleted, env will redownload them.") print("Warning: Packages should be deleted in command.\n") - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + # re download the packages in error_packages_list + for pkg in error_packages_list: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) @@ -499,57 +497,53 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): print("%s" % error_packages_redownload_error_list) print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_pkgs_json = sub_list( - read_back_pkgs_json, error_packages_redownload_error_list) - read_back_pkgs_json = write_back_pkgs_json - # print("write_back_pkgs_json:%s"%write_back_pkgs_json) - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) - pkgs_file.close() + + write_back_package_json = sub_list(read_back_pkgs_json, error_packages_redownload_error_list) + with open(package_filename, 'w') as f: + f.write(json.dumps(write_back_package_json, indent=1)) return flag -def rm_package(dir): +def rm_package(dir_remove): if platform.system() != "Windows": - shutil.rmtree(dir) + shutil.rmtree(dir_remove) else: - dir = '"' + dir + '"' - cmd = 'rd /s /q ' + dir + dir_remove = '"' + dir_remove + '"' + cmd = 'rd /s /q ' + dir_remove os.system(cmd) - if os.path.isdir(dir): + if os.path.isdir(dir_remove): if platform.system() != "Windows": - shutil.rmtree(dir) + shutil.rmtree(dir_remove) else: - dir = '"' + dir + '"' - cmd = 'rmdir /s /q ' + dir + dir_remove = '"' + dir_remove + '"' + cmd = 'rmdir /s /q ' + dir_remove os.system(cmd) - if os.path.isdir(dir): - print("Folder path: %s" % dir.encode("utf-8")) + if os.path.isdir(dir_remove): + print("Folder path: %s" % dir_remove.encode("utf-8")) return False else: - print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + print("Path: %s \nSuccess: Folder has been removed. " % dir_remove.encode("utf-8")) return True def get_package_remove_path(pkg, bsp_packages_path): - dirpath = pkg['path'] + dir_path = pkg['path'] ver = pkg['ver'] - if dirpath[0] == '/' or dirpath[0] == '\\': - dirpath = dirpath[1:] + if dir_path[0] == '/' or dir_path[0] == '\\': + dir_path = dir_path[1:] if platform.system() == "Windows": - dirpath = os.path.basename(dirpath.replace('/', '\\')) + dir_path = os.path.basename(dir_path.replace('/', '\\')) else: - dirpath = os.path.basename(dirpath) - - removepath = os.path.join(bsp_packages_path, dirpath) + dir_path = os.path.basename(dir_path) # Handles the deletion of git repository folders with version Numbers - removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) - return removepath_ver + remove_path = os.path.join(bsp_packages_path, dir_path) + remove_path_ver = get_pkg_folder_by_orign_path(remove_path, ver) + return remove_path_ver def handle_download_error_packages(pkgs_fn, bsp_packages_path): @@ -579,57 +573,11 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): return get_flag -def write_storage_file(pkgs_fn, newpkgs): - """Writes the updated configuration to pkgs.json file. - - Packages that are not downloaded correctly will be redownloaded at the - next update. - """ - - with open(pkgs_fn, 'w') as f: - f.write(str(json.dumps(newpkgs, indent=1))) - - -def package_update(isDeleteOld=False): - """Update env's packages. - - Compare the old and new software package list and update the package. - Remove unwanted packages and download the newly selected package.- - Check if the files in the deleted packages have been changed, and if so, - remind the user saved the modified file. - """ - - sys_value = pre_package_update() - - if not sys_value: - return - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - flag = True - - # According to the env version, whether Chinese output is supported or not - if determine_support_chinese(env_root): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - oldpkgs = sys_value[0] - newpkgs = sys_value[1] - pkgs_delete_error_list = sys_value[2] - pkgs_fn = sys_value[3] - pkgs_error_list_fn = sys_value[4] - bsp_packages_path = sys_value[5] - dbsqlite_pathname = sys_value[6] - - # print(oldpkgs) - # print(newpkgs) - - if len(pkgs_delete_error_list): - for error_package in pkgs_delete_error_list: - removepath_ver = get_package_remove_path( - error_package, bsp_packages_path) - +def delete_useless_packages(package_delete_error_list, bsp_packages_path): + # try to delete useless packages, exit command if fails + if len(package_delete_error_list): + for error_package in package_delete_error_list: + removepath_ver = get_package_remove_path(error_package, bsp_packages_path) if os.path.isdir(removepath_ver): print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) @@ -637,18 +585,26 @@ def package_update(isDeleteOld=False): if not rm_package(removepath_ver): print("Error: Delete package %s failed! Please delete the folder manually.\n" % error_package['name'].encode("utf-8")) - return + return False + return True + - # 1.in old ,not in new : Software packages that need to be removed. - casedelete = sub_list(oldpkgs, newpkgs) - pkgs_delete_fail_list = [] +def remove_packages(sys_value, isDeleteOld): + old_package = sys_value[0] + new_package = sys_value[1] + package_error_list_filename = sys_value[4] + bsp_packages_path = sys_value[5] + sqlite_pathname = sys_value[6] + + case_delete = sub_list(old_package, new_package) + package_delete_fail_list = [] - for pkg in casedelete: + for pkg in case_delete: removepath_ver = get_package_remove_path(pkg, bsp_packages_path) removepath_git = os.path.join(removepath_ver, '.git') - # Delete. Git directory. + # delete .git directory if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): gitdir = removepath_ver @@ -663,7 +619,7 @@ def package_update(isDeleteOld=False): if rc == 'y' or rc == 'Y': try: if not rm_package(gitdir): - pkgs_delete_fail_list.append(pkg) + package_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: print('Error message:%s%s. error.message: %s\n\t' % @@ -672,63 +628,106 @@ def package_update(isDeleteOld=False): if os.path.isdir(removepath_ver): print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) try: - pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) + pkgsdb.deletepackdir(removepath_ver, sqlite_pathname) except Exception as e: - pkgs_delete_fail_list.append(pkg) + package_delete_fail_list.append(pkg) print('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + with open(package_error_list_filename, 'w') as f: + f.write(str(json.dumps(package_delete_fail_list, indent=1))) - if len(pkgs_delete_fail_list): - return + if len(package_delete_fail_list): + return False - # 2.in new not in old : Software packages to be installed. - # If the package download fails, record it, and then download again when - # the update command is executed. + return True + + +def install_packages(new_package, old_package, package_filename): + """ + If the package download fails, record it, + and then download again when the update command is executed. + """ - case_download = sub_list(newpkgs, oldpkgs) - # print 'in new not in old:', casedownload - pkgs_download_fail_list = [] + bsp_root = Import('bsp_root') + pkgs_root = Import('pkgs_root') + env_root = Import('env_root') + + case_download = sub_list(new_package, old_package) + packages_download_fail_list = [] for pkg in case_download: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) else: - # If the PKG download fails, record it in the - # pkgs_download_fail_list. - pkgs_download_fail_list.append(pkg) + # if package download fails, record it in the packages_download_fail_list + packages_download_fail_list.append(pkg) print(pkg, 'download failed.') - flag = False + return False # Get the currently updated configuration. - newpkgs = sub_list(newpkgs, pkgs_download_fail_list) + new_package = sub_list(new_package, packages_download_fail_list) # Give hints based on the success of the download. - if len(pkgs_download_fail_list): + if len(packages_download_fail_list): print("\nPackage download failed list:") - for item in pkgs_download_fail_list: + for item in packages_download_fail_list: print(item) print("You need to reuse the command to download again.") # update pkgs.json and SConscript - write_storage_file(pkgs_fn, newpkgs) + with open(package_filename, 'w') as f: + f.write(str(json.dumps(new_package, indent=1))) - # handle download error packages. - get_flag = handle_download_error_packages( - pkgs_fn, bsp_packages_path) + return True - if get_flag is not None: - flag = get_flag - # Update the software packages, which the version is 'latest' +def package_update(isDeleteOld=False): + """Update env's packages. + + Compare the old and new software package list and update the package. + Remove unwanted packages and download the newly selected package.- + Check if the files in the deleted packages have been changed, and if so, + remind the user saved the modified file. + """ + + sys_value = pre_package_update() + if not sys_value: + return + + flag = True + + old_package = sys_value[0] + new_package = sys_value[1] + package_delete_error_list = sys_value[2] + package_filename = sys_value[3] + package_error_list_filename = sys_value[4] + bsp_packages_path = sys_value[5] + sqlite_pathname = sys_value[6] + + if not delete_useless_packages(package_delete_error_list, bsp_packages_path): + return + + # 1. packages in old and not in new : Software packages that need to be removed + if not remove_packages(sys_value, isDeleteOld): + return + + # 2.in new not in old : Software packages to be installed. + if not install_packages(new_package, old_package, package_filename): + flag = False + + # handle download error packages. + if not handle_download_error_packages(package_filename, bsp_packages_path): + flag = False + + # update the software packages, which the version is 'latest' try: - update_latest_packages(pkgs_fn, bsp_packages_path) - except KeyboardInterrupt: + update_latest_packages(package_filename, bsp_packages_path) + except Exception as e: + print('Error message:%s\t' % e) flag = False if flag: From 41b5943079a5957da0815dcac9dca825434ac344 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 17:39:16 +0800 Subject: [PATCH 17/48] [add new feature] The env tool adds a new feature that if the user renames the package folder, such as changing the name of the Lua-v1.0.0 folder to Lua, then env will consider the package to be self-administered by the user. At this point, command will not updated this package again. If user would like to pull other versions of this package, use the command. --- cmds/cmd_package_update.py | 85 +++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 12787605..28fb9827 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -117,13 +117,25 @@ def determine_url_valid(url_from_srv): print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) -def install_pkg(env_root, pkgs_root, bsp_root, pkg): +def is_user_mange_package(bsp_package_path, pkg): + for root, dirs, files in os.walk(bsp_package_path, topdown=True): + for name in dirs: + if name.lower() == pkg["name"].lower(): + return True + break + return False + + +def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" - # default true ret = True local_pkgs_path = os.path.join(env_root, 'local_pkgs') - bsp_pkgs_path = os.path.join(bsp_root, 'packages') + bsp_package_path = os.path.join(bsp_root, 'packages') + + if not force_update: + if is_user_mange_package(bsp_package_path, pkg): + return ret # get the .config file from env env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') @@ -140,18 +152,16 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): package_url = package.get_url(pkg['ver']) pkgs_name_in_json = package.get_name() - if package_url[-4:] == '.git': - ver_sha = package.get_versha(pkg['ver']) - # print("==================================================>") - # print("packages name :"%pkgs_name_in_json.encode("utf-8")) - # print("ver :"%pkg['ver']) - # print("url :"%package_url.encode("utf-8")) - # print("url_from_json : "%url_from_json.encode("utf-8")) + # print("packages name :", pkgs_name_in_json.encode("utf-8")) + # print("ver :", pkg['ver']) + # print("url :", package_url.encode("utf-8")) + # print("url_from_json : ", url_from_json.encode("utf-8")) # print("==================================================>") - get_package_url = None - get_ver_sha = None + if package_url[-4:] == '.git': + ver_sha = package.get_versha(pkg['ver']) + upstream_change_flag = False try: @@ -172,14 +182,14 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - if package_url[-4:] == '.git': + if package_url.endswith('.git'): try: - repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) + repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] repo_path_full = '"' + repo_path + '"' clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(clone_cmd, cwd=bsp_pkgs_path) + execute_command(clone_cmd, cwd=bsp_package_path) git_check_cmd = 'git checkout -q ' + ver_sha execute_command(git_check_cmd, cwd=repo_path) @@ -241,9 +251,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): # unpack package if not os.path.exists(pkg_dir): - try: - if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_package_path, pkg, pkgs_name_in_json): ret = False except Exception as e: os.remove(package_path) @@ -288,11 +297,7 @@ def update_submodule(repo_path): def get_pkg_folder_by_orign_path(orign_path, version): - # TODO fix for old version project, will remove after new major version - # release - if os.path.exists(orign_path + '-' + version): - return orign_path + '-' + version - return orign_path + return orign_path + '-' + version def git_cmd_exec(cmd, cwd): @@ -466,10 +471,11 @@ def pre_package_update(): with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) - return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, + dbsqlite_pathname] -def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename): +def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update): bsp_root = Import('bsp_root') env_root = Import('env_root') pkgs_root = Import('pkgs_root') @@ -480,12 +486,14 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file print("\n==============================> Packages list to download : \n") for pkg in error_packages_list: print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - print("\nThe packages in the list above are accidentally deleted, env will redownload them.") - print("Warning: Packages should be deleted in command.\n") + print("\nThe packages in the list above are accidentally deleted or renamed.") + print("\nIf you manually delete the version suffix of the package folder name, ") + print("you can use the command to re-download these packages.") + print("\nIn case of accidental deletion, the ENV tool will automatically re-download these packages.") # re download the packages in error_packages_list for pkg in error_packages_list: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): + if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: @@ -546,7 +554,7 @@ def get_package_remove_path(pkg, bsp_packages_path): return remove_path_ver -def handle_download_error_packages(pkgs_fn, bsp_packages_path): +def handle_download_error_packages(pkgs_fn, bsp_packages_path, force_update): """ handle download error packages. Check to see if the packages stored in the Json file list actually exist, @@ -559,16 +567,15 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): error_packages_list = [] for pkg in read_back_pkgs_json: - removepath = get_package_remove_path(pkg, bsp_packages_path) - - if os.path.exists(removepath): + remove_path = get_package_remove_path(pkg, bsp_packages_path) + if os.path.exists(remove_path): continue else: + print("Error packages add: ", pkg) error_packages_list.append(pkg) # Handle the failed download packages - get_flag = error_packages_handle( - error_packages_list, read_back_pkgs_json, pkgs_fn) + get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn, force_update) return get_flag @@ -644,7 +651,7 @@ def remove_packages(sys_value, isDeleteOld): return True -def install_packages(new_package, old_package, package_filename): +def install_packages(new_package, old_package, package_filename, force_update): """ If the package download fails, record it, and then download again when the update command is executed. @@ -658,7 +665,7 @@ def install_packages(new_package, old_package, package_filename): packages_download_fail_list = [] for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): + if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) else: @@ -685,7 +692,7 @@ def install_packages(new_package, old_package, package_filename): return True -def package_update(isDeleteOld=False): +def package_update(force_update=False): """Update env's packages. Compare the old and new software package list and update the package. @@ -712,15 +719,15 @@ def package_update(isDeleteOld=False): return # 1. packages in old and not in new : Software packages that need to be removed - if not remove_packages(sys_value, isDeleteOld): + if not remove_packages(sys_value, force_update): return # 2.in new not in old : Software packages to be installed. - if not install_packages(new_package, old_package, package_filename): + if not install_packages(new_package, old_package, package_filename, force_update): flag = False # handle download error packages. - if not handle_download_error_packages(package_filename, bsp_packages_path): + if not handle_download_error_packages(package_filename, bsp_packages_path, force_update): flag = False # update the software packages, which the version is 'latest' From a1bd958b2b3d7d5380387265af3ddaffff5defd2 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 18:39:12 +0800 Subject: [PATCH 18/48] [fix] Solve the problem that when the user deletes the .git folder, causing the upstream path to be changed --- cmds/cmd_package_update.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 28fb9827..a4a3a651 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -328,6 +328,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): read_back_pkgs_json = json.load(f) for pkg in read_back_pkgs_json: + right_path_flag = True package = Package() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': @@ -343,8 +344,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) try: - # If mirror acceleration is enabled, get the update address from - # the mirror server. + # If mirror acceleration is enabled, get the update address from the mirror server. if (not os.path.isfile(env_config_file)) or \ (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): @@ -354,14 +354,26 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) - if mirror_url[0] is not None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) + # if git root is same as repo path, then change the upstream + get_git_root = get_git_root_path(repo_path) + if os.path.normcase(repo_path) == os.path.normcase(get_git_root): + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + else: + print("\n==============================> updating") + print("Package path: %s" % repo_path) + print("Git root: %s" % get_git_root) + print("Error: Not currently in a git root directory, cannot switch upstream.\n") + right_path_flag = False except Exception as e: print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") + if not right_path_flag: + continue + # Update the package repository from upstream. git_pull_repo(repo_path) @@ -380,6 +392,18 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): print("==============================> %s update done\n" % (pkgs_name_in_json)) +def get_git_root_path(repo_path): + before = os.getcwd() + os.chdir(repo_path) + result = os.popen('git rev-parse --show-toplevel') + result = result.read() + for line in result.splitlines()[:5]: + get_git_root = line + break + os.chdir(before) + return get_git_root + + def pre_package_update(): """ Make preparations before updating the software package. """ From 5ea388ab43de814b911aaea2b5d0f78307e5515a Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 9 Apr 2020 15:09:04 +0800 Subject: [PATCH 19/48] [optimize] package update progress --- cmds/cmd_package_update.py | 176 ++++++++++++++++++++----------------- cmds/cmd_package_utils.py | 11 ++- 2 files changed, 104 insertions(+), 83 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index a4a3a651..3f563bd9 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -205,16 +205,11 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): submodule_path = os.path.join(repo_path, '.gitmodules') if os.path.isfile(submodule_path): print("Start to update submodule") - # print("开始更新软件包子模块") - if (not os.path.isfile(env_config_file)) \ or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - # print("开启了镜像加速，开始修改 .gitmodules 文件") replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file - # print("开始执行更新动作") cmd = 'git submodule update --init --recursive' execute_command(cmd, cwd=repo_path) @@ -286,14 +281,16 @@ def and_list(aList, bList): def update_submodule(repo_path): """Update the submodules in the repository.""" - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): - print("Please wait a few seconds in order to update the submodule.") - cmd = 'git submodule init -q' - execute_command(cmd, cwd=repo_path) - cmd = 'git submodule update' - execute_command(cmd, cwd=repo_path) - print("Submodule update successful") + try: + if os.path.isfile(os.path.join(repo_path, '.gitmodules')): + print("Please wait a few seconds in order to update the submodule.") + cmd = 'git submodule init -q' + execute_command(cmd, cwd=repo_path) + cmd = 'git submodule update' + execute_command(cmd, cwd=repo_path) + print("Submodule update successful") + except Exception as e: + print('Error message:%s' % e) def get_pkg_folder_by_orign_path(orign_path, version): @@ -308,7 +305,7 @@ def git_cmd_exec(cmd, cwd): print("You can solve this problem by manually removing old packages and re-downloading them using env.") -def update_latest_packages(pkgs_fn, bsp_packages_path): +def update_latest_packages(sys_value): """ update the packages that are latest version. If the selected package is the latest version, @@ -318,13 +315,18 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): message provided by git. """ + result = True + + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] + env_root = Import('env_root') pkgs_root = Import('pkgs_root') env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - with open(pkgs_fn, 'r') as f: + with open(package_filename, 'r') as f: read_back_pkgs_json = json.load(f) for pkg in read_back_pkgs_json: @@ -356,16 +358,21 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): # if git root is same as repo path, then change the upstream get_git_root = get_git_root_path(repo_path) - if os.path.normcase(repo_path) == os.path.normcase(get_git_root): - if mirror_url[0] is not None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) + if get_git_root is not None: + if os.path.normcase(repo_path) == os.path.normcase(get_git_root): + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + else: + print("\n==============================> updating") + print("Package path: %s" % repo_path) + print("Git root: %s" % get_git_root) + print("Error: Not currently in a git root directory, cannot switch upstream.\n") + right_path_flag = False + result = False else: - print("\n==============================> updating") - print("Package path: %s" % repo_path) - print("Git root: %s" % get_git_root) - print("Error: Not currently in a git root directory, cannot switch upstream.\n") right_path_flag = False + result = False except Exception as e: print("Error message : %s" % e) @@ -389,19 +396,29 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): print("Can't find the package : %s's url in file : %s" % (payload_pkgs_name_in_json, pkg_path)) - print("==============================> %s update done\n" % (pkgs_name_in_json)) + print("==============================> %s update done\n" % pkgs_name_in_json) + + return result def get_git_root_path(repo_path): - before = os.getcwd() - os.chdir(repo_path) - result = os.popen('git rev-parse --show-toplevel') - result = result.read() - for line in result.splitlines()[:5]: - get_git_root = line - break - os.chdir(before) - return get_git_root + if os.path.isdir(repo_path): + try: + before = os.getcwd() + os.chdir(repo_path) + result = os.popen('git rev-parse --show-toplevel') + result = result.read() + for line in result.splitlines()[:5]: + get_git_root = line + break + os.chdir(before) + return get_git_root + except Exception as e: + print("Error message : %s" % e) + return None + else: + print("Missing path %s" % repo_path) + return None def pre_package_update(): @@ -512,14 +529,14 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) print("\nThe packages in the list above are accidentally deleted or renamed.") print("\nIf you manually delete the version suffix of the package folder name, ") - print("you can use the command to re-download these packages.") + print("you can use command to re-download these packages.") print("\nIn case of accidental deletion, the ENV tool will automatically re-download these packages.") - # re download the packages in error_packages_list + # re-download the packages in error_packages_list for pkg in error_packages_list: if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): - print("==============================> %s %s is redownloaded successfully. \n" % ( - pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("==============================> %s %s is redownloaded successfully. \n" + % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: error_packages_redownload_error_list.append(pkg) print(pkg, 'download failed.') @@ -527,8 +544,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file if len(error_packages_redownload_error_list): print("%s" % error_packages_redownload_error_list) - print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("Packages:%s,%s re-download error, you need to use command again to re-download them." + % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) write_back_package_json = sub_list(read_back_pkgs_json, error_packages_redownload_error_list) with open(package_filename, 'w') as f: @@ -578,14 +595,16 @@ def get_package_remove_path(pkg, bsp_packages_path): return remove_path_ver -def handle_download_error_packages(pkgs_fn, bsp_packages_path, force_update): +def handle_download_error_packages(sys_value, force_update): """ handle download error packages. Check to see if the packages stored in the Json file list actually exist, and then download the packages if they don't exist. """ + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] - with open(pkgs_fn, 'r') as f: + with open(package_filename, 'r') as f: read_back_pkgs_json = json.load(f) error_packages_list = [] @@ -599,21 +618,25 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path, force_update): error_packages_list.append(pkg) # Handle the failed download packages - get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn, force_update) + get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update) return get_flag -def delete_useless_packages(package_delete_error_list, bsp_packages_path): +def delete_useless_packages(sys_value): + package_delete_error_list = sys_value[2] + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] + # try to delete useless packages, exit command if fails if len(package_delete_error_list): for error_package in package_delete_error_list: - removepath_ver = get_package_remove_path(error_package, bsp_packages_path) - if os.path.isdir(removepath_ver): + remove_path_with_version = get_package_remove_path(error_package, bsp_packages_path) + if os.path.isdir(remove_path_with_version): print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) - if not rm_package(removepath_ver): + if not rm_package(remove_path_with_version): print("Error: Delete package %s failed! Please delete the folder manually.\n" % error_package['name'].encode("utf-8")) return False @@ -631,39 +654,39 @@ def remove_packages(sys_value, isDeleteOld): package_delete_fail_list = [] for pkg in case_delete: - - removepath_ver = get_package_remove_path(pkg, bsp_packages_path) - removepath_git = os.path.join(removepath_ver, '.git') + remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) + remove_path_git = os.path.join(remove_path_with_version, '.git') # delete .git directory - if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): - gitdir = removepath_ver + if os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git): + git_folder_to_remove = remove_path_with_version - print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) + print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) if isDeleteOld: - if not rm_package(gitdir): - print("Floder delete fail: %s" % gitdir.encode("utf-8")) + if not rm_package(git_folder_to_remove): + print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) print("Please delete this folder manually.") else: print("The folder is managed by git. Do you want to delete this folder?\n") rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': try: - if not rm_package(gitdir): + if not rm_package(git_folder_to_remove): package_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", gitdir.encode("utf-8"), e)) + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) else: - if os.path.isdir(removepath_ver): - print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) + if os.path.isdir(remove_path_with_version): + print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) try: - pkgsdb.deletepackdir(removepath_ver, sqlite_pathname) + pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) except Exception as e: package_delete_fail_list.append(pkg) print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) + "Delete folder failed, please delete the folder manually", + remove_path_with_version.encode("utf-8"), e)) # write error messages with open(package_error_list_filename, 'w') as f: @@ -675,12 +698,18 @@ def remove_packages(sys_value, isDeleteOld): return True -def install_packages(new_package, old_package, package_filename, force_update): +def install_packages(sys_value, force_update): """ If the package download fails, record it, and then download again when the update command is executed. """ + old_package = sys_value[0] + new_package = sys_value[1] + package_delete_error_list = sys_value[2] + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] + bsp_root = Import('bsp_root') pkgs_root = Import('pkgs_root') env_root = Import('env_root') @@ -731,34 +760,23 @@ def package_update(force_update=False): flag = True - old_package = sys_value[0] - new_package = sys_value[1] - package_delete_error_list = sys_value[2] - package_filename = sys_value[3] - package_error_list_filename = sys_value[4] - bsp_packages_path = sys_value[5] - sqlite_pathname = sys_value[6] - - if not delete_useless_packages(package_delete_error_list, bsp_packages_path): + if not delete_useless_packages(sys_value): return - # 1. packages in old and not in new : Software packages that need to be removed + # 1.in old and not in new : Software packages that need to be removed if not remove_packages(sys_value, force_update): return # 2.in new not in old : Software packages to be installed. - if not install_packages(new_package, old_package, package_filename, force_update): + if not install_packages(sys_value, force_update): flag = False - # handle download error packages. - if not handle_download_error_packages(package_filename, bsp_packages_path, force_update): + # 3.handle download error packages. + if not handle_download_error_packages(sys_value, force_update): flag = False - # update the software packages, which the version is 'latest' - try: - update_latest_packages(package_filename, bsp_packages_path) - except Exception as e: - print('Error message:%s\t' % e) + # 4.update the software packages, which the version is 'latest' + if not update_latest_packages(sys_value): flag = False if flag: diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 5b56d0bf..920cca3f 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -46,11 +46,14 @@ def execute_command(cmd_string, cwd=None, shell=True): def git_pull_repo(repo_path, repo_url=''): - if platform.system() == "Windows": - cmd = r'git config --local core.autocrlf true' + try: + if platform.system() == "Windows": + cmd = r'git config --local core.autocrlf true' + execute_command(cmd, cwd=repo_path) + cmd = r'git pull ' + repo_url execute_command(cmd, cwd=repo_path) - cmd = r'git pull ' + repo_url - execute_command(cmd, cwd=repo_path) + except Exception as e: + print('Error message:%s' % e) def get_url_from_mirror_server(package_name, package_version): From e3fde60df1494690550c8fb352e0d9eb21bc94a7 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 9 Apr 2020 15:32:44 +0800 Subject: [PATCH 20/48] [optimize] cmds/cmd_package_update.py --- cmds/cmd_package_update.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 3f563bd9..88ae72ab 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -530,12 +530,12 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file print("\nThe packages in the list above are accidentally deleted or renamed.") print("\nIf you manually delete the version suffix of the package folder name, ") print("you can use command to re-download these packages.") - print("\nIn case of accidental deletion, the ENV tool will automatically re-download these packages.") + print("In case of accidental deletion, the ENV tool will automatically re-download these packages.") # re-download the packages in error_packages_list for pkg in error_packages_list: if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): - print("==============================> %s %s is redownloaded successfully. \n" + print("\n==============================> %s %s update done \n" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: error_packages_redownload_error_list.append(pkg) @@ -614,7 +614,7 @@ def handle_download_error_packages(sys_value, force_update): if os.path.exists(remove_path): continue else: - print("Error packages add: ", pkg) + print("Error package : %s" % pkg) error_packages_list.append(pkg) # Handle the failed download packages @@ -625,7 +625,6 @@ def handle_download_error_packages(sys_value, force_update): def delete_useless_packages(sys_value): package_delete_error_list = sys_value[2] - package_filename = sys_value[3] bsp_packages_path = sys_value[5] # try to delete useless packages, exit command if fails @@ -643,7 +642,7 @@ def delete_useless_packages(sys_value): return True -def remove_packages(sys_value, isDeleteOld): +def remove_packages(sys_value, force_update): old_package = sys_value[0] new_package = sys_value[1] package_error_list_filename = sys_value[4] @@ -662,7 +661,7 @@ def remove_packages(sys_value, isDeleteOld): git_folder_to_remove = remove_path_with_version print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) - if isDeleteOld: + if force_update: if not rm_package(git_folder_to_remove): print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) print("Please delete this folder manually.") @@ -706,10 +705,7 @@ def install_packages(sys_value, force_update): old_package = sys_value[0] new_package = sys_value[1] - package_delete_error_list = sys_value[2] package_filename = sys_value[3] - bsp_packages_path = sys_value[5] - bsp_root = Import('bsp_root') pkgs_root = Import('pkgs_root') env_root = Import('env_root') @@ -738,7 +734,7 @@ def install_packages(sys_value, force_update): print("You need to reuse the command to download again.") - # update pkgs.json and SConscript + # Update pkgs.json and SConscript with open(package_filename, 'w') as f: f.write(str(json.dumps(new_package, indent=1))) From 7186ed65fc3de771c10bdc0d953caa0e59c9595a Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 10 Apr 2020 09:40:43 +0800 Subject: [PATCH 21/48] [optimize] code clear up --- cmds/__init__.py | 2 +- cmds/cmd_menuconfig.py | 10 ++++++---- cmds/cmd_package_wizard.py | 1 - env.py | 12 ++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmds/__init__.py b/cmds/__init__.py index def30d7e..c5688f7c 100644 --- a/cmds/__init__.py +++ b/cmds/__init__.py @@ -35,4 +35,4 @@ "* $ pip install requests\n" "* command install step:\n" "* $ sudo apt-get install python-requests\n" - "****************************************\n") \ No newline at end of file + "****************************************\n") diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 7cb61081..189a8964 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -28,7 +28,7 @@ import os import platform import re -from vars import Import, Export +from vars import Import def is_pkg_special_config(config_str): @@ -44,6 +44,7 @@ def mk_rtconfig(filename): try: config = open(filename, 'r') except Exception as e: + print('Error message:%s' % e) print('open config:%s failed' % filename) return @@ -103,6 +104,7 @@ def find_macro_in_config(filename, macro_name): try: config = open(filename, "r") except Exception as e: + print('Error message:%s' % e) print('open .config failed') return @@ -194,7 +196,7 @@ def cmd(args): os.system('kconfig-mconf Kconfig -n') elif args.menuconfig_setting: - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') beforepath = os.getcwd() os.chdir(env_kconfig_path) @@ -228,7 +230,7 @@ def cmd(args): mk_rtconfig(fn) if platform.system() == "Windows": - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') fn = os.path.join(env_kconfig_path, '.config') if not os.path.isfile(fn): @@ -276,7 +278,7 @@ def add_parser(sub): dest='menuconfig_setting') parser.add_argument('--easy', - help='easy mode,place kconfig file everywhere,just modify the option env="RTT_ROOT" default "../.."', + help='easy mode, place kconfig everywhere, modify the option env="RTT_ROOT" default "../.."', action='store_true', default=False, dest='menuconfig_easy') diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py index 949a3ef6..d4a650d0 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package_wizard.py @@ -25,7 +25,6 @@ import os import re -import sys from package import Kconfig_file, Package_json_file from string import Template from .cmd_package_utils import user_input diff --git a/env.py b/env.py index 3dbb3e9a..c122eb9d 100644 --- a/env.py +++ b/env.py @@ -32,7 +32,7 @@ from cmds import * from vars import Export -__version__ = 'rt-thread packages v1.1.0' +__version__ = 'rt-thread packages v1.2.0' def init_argparse(): @@ -60,7 +60,7 @@ def main(): env_root = os.path.join(os.getenv('USERPROFILE'), '.env') sys.path = sys.path + [os.path.join(script_root)] - + pkgs_root = os.getenv("PKGS_ROOT") if pkgs_root is None: pkgs_root = os.path.join(env_root, 'packages') @@ -76,10 +76,10 @@ def main(): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m警告：\033[0m") - print ("\033[1;31;40m当前路径不支持非英文字符，请修改当前路径为纯英文路径。\033[0m") - print ("\033[1;31;40mThe current path does not support non-English characters.\033[0m") - print ("\033[1;31;40mPlease modify the current path to a pure English path.\033[0m") + print("\n\033[1;31;40m警告：\033[0m") + print("\033[1;31;40m当前路径不支持非英文字符，请修改当前路径为纯英文路径。\033[0m") + print("\033[1;31;40mThe current path does not support non-English characters.\033[0m") + print("\033[1;31;40mPlease modify the current path to a pure English path.\033[0m") if platform.system() == "Windows": os.system('chcp 437 > nul') From d0aaee02ee3ba33a30704a18ddcd0e72f57671e1 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 10 Apr 2020 09:53:53 +0800 Subject: [PATCH 22/48] [optimize] package database operation --- archive.py | 27 ++++++++-------- cmds/cmd_package_printenv.py | 2 +- cmds/cmd_package_update.py | 3 +- cmds/cmd_system.py | 13 +++----- package.py | 4 +-- pkgsdb.py | 60 +++++++++++++++--------------------- vars.py | 2 ++ 7 files changed, 51 insertions(+), 60 deletions(-) diff --git a/archive.py b/archive.py index b0ea9f1b..56218b94 100644 --- a/archive.py +++ b/archive.py @@ -21,6 +21,7 @@ # Change Logs: # Date Author Notes # 2018-5-28 SummerGift Add copyright information +# 2020-4-10 SummerGift Code clear up # import tarfile @@ -35,10 +36,10 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): pkg_ver = pkg['ver'] flag = True - iswindows = False - if platform.system() == "Windows": - iswindows = True + is_windows = True + else: + is_windows = False if ".tar.bz2" in archive_fn: arch = tarfile.open(archive_fn, "r:bz2") @@ -46,12 +47,12 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): arch.extract(tarinfo, path) a = tarinfo.name if not os.path.isdir(os.path.join(path, a)): - if iswindows: + if is_windows: right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.savetodb(a, archive_fn) + pkgsdb.save_to_database(a, archive_fn) arch.close() if ".tar.gz" in archive_fn: @@ -60,12 +61,12 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): arch.extract(tarinfo, path) a = tarinfo.name if not os.path.isdir(os.path.join(path, a)): - if iswindows: + if is_windows: right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.savetodb(a, archive_fn) + pkgsdb.save_to_database(a, archive_fn) arch.close() if ".zip" in archive_fn: @@ -73,7 +74,7 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): for item in arch.namelist(): arch.extract(item, path) if not os.path.isdir(os.path.join(path, item)): - if iswindows: + if is_windows: right_path = item.replace('/', '\\') else: right_path = item @@ -85,14 +86,14 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): flag = False right_name_to_db = right_path.replace(dir_name, change_dirname, 1) - pkgsdb.savetodb(right_name_to_db, archive_fn, right_path) + pkgsdb.save_to_database(right_name_to_db, archive_fn, right_path) arch.close() # Change the folder name change_dirname = pkgs_name_in_json + '-' + pkg_ver if os.path.isdir(os.path.join(path, change_dirname)): - if iswindows: + if is_windows: cmd = 'rd /s /q ' + os.path.join(path, change_dirname) os.system(cmd) else: @@ -101,7 +102,7 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): os.rename(os.path.join(path, dir_name), os.path.join(path, change_dirname)) -def packtest(path): +def package_integrity_test(path): ret = True if path.find(".zip") != -1: @@ -116,7 +117,7 @@ def packtest(path): ret = False print('package check error. \n') except Exception as e: - print('packtest error message:%s\t' % e) + print('Package test error message:%s\t' % e) print("The archive package is broken. \n") arch.close() ret = False @@ -127,6 +128,7 @@ def packtest(path): if not tarfile.is_tarfile(path): ret = False except Exception as e: + print('Error message:%s' % e) ret = False # if ".tar.gz" in path: @@ -135,6 +137,7 @@ def packtest(path): if not tarfile.is_tarfile(path): ret = False except Exception as e: + print('Error message:%s' % e) ret = False return ret diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package_printenv.py index 377ead46..36291da5 100644 --- a/cmds/cmd_package_printenv.py +++ b/cmds/cmd_package_printenv.py @@ -40,7 +40,7 @@ def package_print_env(): if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') - print("ENV_ROOT:%s" % (env_root)) + print("ENV_ROOT:%s" % env_root) def package_print_help(): diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 88ae72ab..fe32acd4 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -194,6 +194,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): git_check_cmd = 'git checkout -q ' + ver_sha execute_command(git_check_cmd, cwd=repo_path) except Exception as e: + print('Error message:%s' % e) print("\nFailed to download software package with git. Please check the network connection.") return False @@ -240,7 +241,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): pkg_dir = os.path.splitext(pkg_dir)[0] package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - if not archive.packtest(package_path): + if not archive.package_integrity_test(package_path): print("package : %s is invalid" % package_path.encode("utf-8")) return False diff --git a/cmds/cmd_system.py b/cmds/cmd_system.py index 4b116202..4f682744 100644 --- a/cmds/cmd_system.py +++ b/cmds/cmd_system.py @@ -35,14 +35,11 @@ def cmd(args): if args.system_update: dir_list = os.listdir(packages_root) - kconfig = file(os.path.join(packages_root, 'Kconfig'), 'w') - - for item in dir_list: - if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): - kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') - kconfig.write('\n') - - kconfig.close() + with open(os.path.join(packages_root, 'Kconfig'), 'w') as kconfig: + for item in dir_list: + if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): + kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') + kconfig.write('\n') def add_parser(sub): diff --git a/package.py b/package.py index 027bf529..824a9b4e 100644 --- a/package.py +++ b/package.py @@ -186,7 +186,7 @@ def download(self, ver, path, url_from_srv): if not os.path.getsize(path): os.remove(path) else: - if archive.packtest(path): + if archive.package_integrity_test(path): # print "The file is rigit." return True else: @@ -219,7 +219,7 @@ def download(self, ver, path, url_from_srv): retry_count = retry_count + 1 - if archive.packtest(path): # make sure the file is right + if archive.package_integrity_test(path): # make sure the file is right ret = True print("\rDownloded %d KB " % flush_count) print('Start to unpack. Please wait...') diff --git a/pkgsdb.py b/pkgsdb.py index 715d54cb..72e8fa4a 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -21,6 +21,7 @@ # Change Logs: # Date Author Notes # 2018-5-28 SummerGift Add copyright information +# 2020-4-10 SummerGift Code clear up # import sqlite3 @@ -33,27 +34,25 @@ SHOW_SQL = False -def GetFileMd5(filename): +def get_file_md5(filename): if not os.path.isfile(filename): return - myhash = hashlib.md5() + hash_value = hashlib.md5() f = open(filename, 'rb') while True: b = f.read(8096) if not b: break - myhash.update(b) + hash_value.update(b) f.close() - return myhash.hexdigest() + return hash_value.hexdigest() def get_conn(path): conn = sqlite3.connect(path) if os.path.exists(path) and os.path.isfile(path): - # print('on disk:[{}]'.format(path)) return conn else: - conn = None print('on memory:[:memory:]') return sqlite3.connect(':memory:') @@ -74,17 +73,16 @@ def create_table(conn, sql): if sql is not None and sql != '': cu = get_cursor(conn) if SHOW_SQL: - print('执行sql:[{}]'.format(sql)) + print('execute :[{}]'.format(sql)) cu.execute(sql) conn.commit() - # print('create data table successful!') close_all(conn) else: print('the [{}] is empty or equal None!'.format(sql)) def save(conn, sql, data): - '''insert data to database''' + """insert data to database""" if sql is not None and sql != '': if data is not None: cu = get_cursor(conn) @@ -100,14 +98,15 @@ def save(conn, sql, data): def isdataexist(pathname): ret = True - dbpathname = Import('dbsqlite_pathname') + dbfilename = Import('dbsqlite_pathname') - conn = get_conn(dbpathname) + conn = get_conn(dbfilename) c = get_cursor(conn) sql = 'SELECT md5 from packagefile where pathname = "' + pathname + '"' cursor = c.execute(sql) for row in cursor: dbmd5 = row[0] + if dbmd5: ret = False conn.close() @@ -115,29 +114,26 @@ def isdataexist(pathname): # 将数据添加到数据库，如果数据库中已经存在则不重复添加 -def savetodb(pathname, pkgspathname, before_change_name): +def save_to_database(pathname, package_pathname, before_change_name): dbpathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') bsppkgs = os.path.join(bsp_root, 'packages') conn = get_conn(dbpathname) save_sql = '''insert into packagefile values (?, ?, ?)''' - package = os.path.basename(pkgspathname) + package = os.path.basename(package_pathname) md5pathname = os.path.join(bsppkgs, before_change_name) - # print("pathname to save : %s"%pathname) - # print("md5pathname : %s"%md5pathname) if not os.path.isfile(md5pathname): print("md5pathname is Invalid") - md5 = GetFileMd5(md5pathname) - # print("md5 to save : %s"%md5) + md5 = get_file_md5(md5pathname) data = [(pathname, package, md5)] save(conn, save_sql, data) -def dbdump(dbpathname): - conn = get_conn(dbpathname) +def dbdump(dbfilename): + conn = get_conn(dbfilename) c = get_cursor(conn) cursor = c.execute("SELECT pathname, package, md5 from packagefile") for row in cursor: @@ -147,39 +143,33 @@ def dbdump(dbpathname): conn.close() -def remove_unchangedfile(pathname, dbpathname, dbsqlname): +def remove_unchanged_file(pathname, dbfilename, dbsqlname): """delete unchanged files""" flag = True - conn = get_conn(dbpathname) + conn = get_conn(dbfilename) c = get_cursor(conn) - - # print('pathname : %s'%pathname) - # print('dbsqlname : %s'%dbsqlname) - - filemd5 = GetFileMd5(pathname) - # print("filemd5 : %s"%filemd5) + filemd5 = get_file_md5(pathname) dbmd5 = 0 sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' # print sql cursor = c.execute(sql) for row in cursor: - dbmd5 = row[0] # fetch md5 from databas - # print("dbmd5 : %s"%dbmd5) + # fetch md5 from database + dbmd5 = row[0] if dbmd5 == filemd5: # delete file info from database sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - cursor = c.execute(sql) conn.commit() os.remove(pathname) else: print("%s has been changed." % pathname) print('Are you sure you want to permanently delete the file: %s?' % os.path.basename(pathname)) - print( - 'If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') + print('If you choose to keep the changed file,you should copy the file to another folder. ' + '\nbecaues it may be covered by the next update.') if sys.version_info < (3, 0): rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') @@ -187,7 +177,6 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): rc = input('Press the Y Key to delete the file or just press Enter to keep the file.') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - cursor = c.execute(sql) conn.commit() os.remove(pathname) print("%s has been removed.\n" % pathname) @@ -202,7 +191,7 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): def deletepackdir(dirpath, dbpathname): flag = getdirdisplay(dirpath, dbpathname) - if flag == True: + if flag: if os.path.exists(dirpath): for root, dirs, files in os.walk(dirpath, topdown=False): for name in files: @@ -226,8 +215,7 @@ def displaydir(filepath, basepath, length, dbpathname): else: pathname = os.path.join(filepath, fi_d) dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] - # print("dbsqlname : %s"%dbsqlname) - if not remove_unchangedfile(pathname, dbpathname, dbsqlname): + if not remove_unchanged_file(pathname, dbpathname, dbsqlname): flag = False return flag diff --git a/vars.py b/vars.py index 6d443b01..b70c3dbc 100644 --- a/vars.py +++ b/vars.py @@ -27,9 +27,11 @@ env_vars = {} + def Export(var): f = sys._getframe(1).f_locals env_vars[var] = f[var] + def Import(var): return env_vars[var] From a265aa92a769ec6762776ebfc0f2c3300c3e6366 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 10 Apr 2020 14:39:31 +0800 Subject: [PATCH 23/48] [optimize] code clear up --- archive.py | 6 +++--- pkgsdb.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/archive.py b/archive.py index 56218b94..a5c992d8 100644 --- a/archive.py +++ b/archive.py @@ -32,7 +32,7 @@ import shutil -def unpack(archive_fn, path, pkg, pkgs_name_in_json): +def unpack(archive_fn, path, pkg, package_name): pkg_ver = pkg['ver'] flag = True @@ -82,7 +82,7 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): # Gets the folder name and change_dirname only once if flag: dir_name = os.path.split(right_path)[0] - change_dirname = pkgs_name_in_json + '-' + pkg_ver + change_dirname = package_name + '-' + pkg_ver flag = False right_name_to_db = right_path.replace(dir_name, change_dirname, 1) @@ -90,7 +90,7 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): arch.close() # Change the folder name - change_dirname = pkgs_name_in_json + '-' + pkg_ver + change_dirname = package_name + '-' + pkg_ver if os.path.isdir(os.path.join(path, change_dirname)): if is_windows: diff --git a/pkgsdb.py b/pkgsdb.py index 72e8fa4a..b556b881 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -115,14 +115,14 @@ def isdataexist(pathname): # 将数据添加到数据库，如果数据库中已经存在则不重复添加 def save_to_database(pathname, package_pathname, before_change_name): - dbpathname = Import('dbsqlite_pathname') + db_pathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') - bsppkgs = os.path.join(bsp_root, 'packages') + bsp_packages_path = os.path.join(bsp_root, 'packages') - conn = get_conn(dbpathname) + conn = get_conn(db_pathname) save_sql = '''insert into packagefile values (?, ?, ?)''' package = os.path.basename(package_pathname) - md5pathname = os.path.join(bsppkgs, before_change_name) + md5pathname = os.path.join(bsp_packages_path, before_change_name) if not os.path.isfile(md5pathname): print("md5pathname is Invalid") From 0014f2e6b0556d273372fae6051f24c0bdf7d6d5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Sat, 11 Apr 2020 11:41:07 +0800 Subject: [PATCH 24/48] [optimize] convert cmd package feature to a package --- cmds/cmd_menuconfig.py | 48 +------------------ .../__init__.py} | 1 + cmds/{ => cmd_package}/cmd_package_list.py | 0 .../{ => cmd_package}/cmd_package_printenv.py | 0 cmds/{ => cmd_package}/cmd_package_update.py | 3 +- cmds/{ => cmd_package}/cmd_package_upgrade.py | 3 +- cmds/{ => cmd_package}/cmd_package_utils.py | 48 +++++++++++++++++++ cmds/{ => cmd_package}/cmd_package_wizard.py | 1 + 8 files changed, 53 insertions(+), 51 deletions(-) rename cmds/{cmd_package.py => cmd_package/__init__.py} (99%) rename cmds/{ => cmd_package}/cmd_package_list.py (100%) rename cmds/{ => cmd_package}/cmd_package_printenv.py (100%) rename cmds/{ => cmd_package}/cmd_package_update.py (99%) rename cmds/{ => cmd_package}/cmd_package_upgrade.py (98%) rename cmds/{ => cmd_package}/cmd_package_utils.py (79%) rename cmds/{ => cmd_package}/cmd_package_wizard.py (99%) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 189a8964..541278ec 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -29,6 +29,7 @@ import platform import re from vars import Import +from .cmd_package.cmd_package_utils import find_macro_in_config def is_pkg_special_config(config_str): @@ -100,53 +101,6 @@ def mk_rtconfig(filename): rtconfig.close() -def find_macro_in_config(filename, macro_name): - try: - config = open(filename, "r") - except Exception as e: - print('Error message:%s' % e) - print('open .config failed') - return - - empty_line = 1 - - for line in config: - line = line.lstrip(' ').replace('\n', '').replace('\r', '') - - if len(line) == 0: - continue - - if line[0] == '#': - if len(line) == 1: - if empty_line: - continue - - empty_line = 1 - continue - - # comment_line = line[1:] - if line.startswith('# CONFIG_'): - line = ' ' + line[9:] - else: - line = line[1:] - - # print line - - empty_line = 0 - else: - empty_line = 0 - setting = line.split('=') - if len(setting) >= 2: - if setting[0].startswith('CONFIG_'): - setting[0] = setting[0][7:] - - if setting[0] == macro_name and setting[1] == 'y': - return True - - config.close() - return False - - def cmd(args): env_root = Import('env_root') os_version = platform.platform(True)[10:13] diff --git a/cmds/cmd_package.py b/cmds/cmd_package/__init__.py similarity index 99% rename from cmds/cmd_package.py rename to cmds/cmd_package/__init__.py index 1ec8fe9b..68361a40 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package/__init__.py @@ -94,3 +94,4 @@ def add_parser(sub): dest='package_print_env') parser.set_defaults(func=run_env_cmd) + diff --git a/cmds/cmd_package_list.py b/cmds/cmd_package/cmd_package_list.py similarity index 100% rename from cmds/cmd_package_list.py rename to cmds/cmd_package/cmd_package_list.py diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package/cmd_package_printenv.py similarity index 100% rename from cmds/cmd_package_printenv.py rename to cmds/cmd_package/cmd_package_printenv.py diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py similarity index 99% rename from cmds/cmd_package_update.py rename to cmds/cmd_package/cmd_package_update.py index fe32acd4..07ac4f1f 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -34,8 +34,7 @@ import requests from package import Package, Bridge_SConscript from vars import Import, Export -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input -from .cmd_menuconfig import find_macro_in_config +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, find_macro_in_config def determine_support_chinese(env_root): diff --git a/cmds/cmd_package_upgrade.py b/cmds/cmd_package/cmd_package_upgrade.py similarity index 98% rename from cmds/cmd_package_upgrade.py rename to cmds/cmd_package/cmd_package_upgrade.py index 982cfe5d..fd46a212 100644 --- a/cmds/cmd_package_upgrade.py +++ b/cmds/cmd_package/cmd_package_upgrade.py @@ -25,8 +25,7 @@ import os from vars import Import -from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server -from .cmd_menuconfig import find_macro_in_config +from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server, find_macro_in_config def upgrade_packages_index(): diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package/cmd_package_utils.py similarity index 79% rename from cmds/cmd_package_utils.py rename to cmds/cmd_package/cmd_package_utils.py index 920cca3f..7e7433d9 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package/cmd_package_utils.py @@ -127,3 +127,51 @@ def user_input(msg=None): value = input() return value + + +def find_macro_in_config(filename, macro_name): + try: + config = open(filename, "r") + except Exception as e: + print('Error message:%s' % e) + print('open .config failed') + return + + empty_line = 1 + + for line in config: + line = line.lstrip(' ').replace('\n', '').replace('\r', '') + + if len(line) == 0: + continue + + if line[0] == '#': + if len(line) == 1: + if empty_line: + continue + + empty_line = 1 + continue + + # comment_line = line[1:] + if line.startswith('# CONFIG_'): + line = ' ' + line[9:] + else: + line = line[1:] + + # print line + + empty_line = 0 + else: + empty_line = 0 + setting = line.split('=') + if len(setting) >= 2: + if setting[0].startswith('CONFIG_'): + setting[0] = setting[0][7:] + + if setting[0] == macro_name and setting[1] == 'y': + return True + + config.close() + return False + diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package/cmd_package_wizard.py similarity index 99% rename from cmds/cmd_package_wizard.py rename to cmds/cmd_package/cmd_package_wizard.py index d4a650d0..41b6a8e8 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package/cmd_package_wizard.py @@ -152,3 +152,4 @@ def package_wizard(): print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') print('Please \033[5;34;40mupdate\033[0m other information of this package ' 'based on Kconfig and package.json in directory ' + name + '.') + From abdff816363941ab34fae46bca4a4592b5115dd8 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:02:43 +0800 Subject: [PATCH 25/48] [update] using user_input to replace raw_inuput or input --- pkgsdb.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkgsdb.py b/pkgsdb.py index b556b881..a6db58d1 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -27,8 +27,7 @@ import sqlite3 import os import hashlib -import sys - +from cmds.cmd_package.cmd_package_utils import user_input from vars import Import SHOW_SQL = False @@ -171,10 +170,7 @@ def remove_unchanged_file(pathname, dbfilename, dbsqlname): print('If you choose to keep the changed file,you should copy the file to another folder. ' '\nbecaues it may be covered by the next update.') - if sys.version_info < (3, 0): - rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') - else: - rc = input('Press the Y Key to delete the file or just press Enter to keep the file.') + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" conn.commit() From da7fe3c0d3a840108820a45c2239c49db01c7573 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:04:08 +0800 Subject: [PATCH 26/48] [update] reduce some unnecessary log --- cmds/cmd_package/cmd_package_update.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 07ac4f1f..6f810329 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -112,7 +112,8 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - print('Error message:%s\t' % e) + # print('Error message:%s\t' % e) + # So much error message should be ignore print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -375,7 +376,7 @@ def update_latest_packages(sys_value): result = False except Exception as e: - print("Error message : %s" % e) + # print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") if not right_path_flag: From 28f72189272a96cd2394cdf696b1194fe92817ef Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 11:08:12 +0800 Subject: [PATCH 27/48] [optimize] code clear up --- cmds/cmd_menuconfig.py | 2 +- cmds/cmd_package/cmd_package_update.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 541278ec..d1eb2dea 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -35,7 +35,7 @@ def is_pkg_special_config(config_str): """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER""" - if type(config_str) == type('a'): + if isinstance(config_str, str): if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')): return True return False diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 6f810329..a2752f20 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -34,7 +34,8 @@ import requests from package import Package, Bridge_SConscript from vars import Import, Export -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, find_macro_in_config +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ + find_macro_in_config def determine_support_chinese(env_root): @@ -98,6 +99,7 @@ def determine_url_valid(url_from_srv): 'Accept': '*/*', 'User-Agent': 'curl/7.54.0'} + # noinspection PyBroadException try: for i in range(0, 3): r = requests.get(url_from_srv, stream=True, headers=headers) @@ -346,6 +348,7 @@ def update_latest_packages(sys_value): repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) + # noinspection PyBroadException try: # If mirror acceleration is enabled, get the update address from the mirror server. if (not os.path.isfile(env_config_file)) or \ From f8555deeb530e7be42130933512a5c12ec27b481 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 14:07:08 +0800 Subject: [PATCH 28/48] [optimize] code clear up and add logging system --- env.py | 62 +++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/env.py b/env.py index c122eb9d..9592964d 100644 --- a/env.py +++ b/env.py @@ -27,6 +27,8 @@ import os import sys import argparse +import logging +import time import platform from cmds import * @@ -39,8 +41,7 @@ def init_argparse(): parser = argparse.ArgumentParser(description=__doc__) subs = parser.add_subparsers() - parser.add_argument('-v', '--version', - action='version', version=__version__) + parser.add_argument('-v', '--version', action='version', version=__version__) cmd_system.add_parser(subs) cmd_menuconfig.add_parser(subs) @@ -49,9 +50,21 @@ def init_argparse(): return parser -def main(): - bsp_root = os.getcwd() - script_root = os.path.split(os.path.realpath(__file__))[0] +def init_logger(env_root): + localtime = time.asctime(time.localtime(time.time())) + log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-")) + os.makedirs(log_path) + log_name = os.path.join(log_path, "running_log.txt") + + log_format = "%(asctime)s %(name)s %(levelname)s %(pathname)s %(lineno)d %(message)s " + date_format = '%Y-%m-%d %H:%M:%S %a ' + logging.basicConfig(level=logging.DEBUG, + format=log_format, + datefmt=date_format, + filename=log_name) + + +def get_env_root(): env_root = os.getenv("ENV_ROOT") if env_root is None: if platform.system() != 'Windows': @@ -59,15 +72,18 @@ def main(): else: env_root = os.path.join(os.getenv('USERPROFILE'), '.env') - sys.path = sys.path + [os.path.join(script_root)] + return env_root - pkgs_root = os.getenv("PKGS_ROOT") - if pkgs_root is None: - pkgs_root = os.path.join(env_root, 'packages') - Export('env_root') - Export('bsp_root') - Export('pkgs_root') +def get_package_root(env_root): + package_root = os.getenv("PKGS_ROOT") + if package_root is None: + package_root = os.path.join(env_root, 'packages') + return package_root + + +def get_bsp_root(): + bsp_root = os.getcwd() # noinspection PyBroadException try: @@ -84,7 +100,26 @@ def main(): if platform.system() == "Windows": os.system('chcp 437 > nul') - return False + exit(1) + + return bsp_root + + +def export_environment_variable(): + script_root = os.path.split(os.path.realpath(__file__))[0] + sys.path = sys.path + [os.path.join(script_root)] + bsp_root = get_bsp_root() + env_root = get_env_root() + pkgs_root = get_package_root(env_root) + + Export('env_root') + Export('bsp_root') + Export('pkgs_root') + + +def main(): + export_environment_variable() + init_logger(get_env_root()) parser = init_argparse() args = parser.parse_args() @@ -93,4 +128,3 @@ def main(): if __name__ == '__main__': main() - From f6a7f5cea06da7685363c3a8cbfb9723d5a6ab36 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 14:10:09 +0800 Subject: [PATCH 29/48] [optimize] add need_using_mirror_download function to simplify code --- cmds/cmd_package/cmd_package_update.py | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index a2752f20..9d497faf 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -128,6 +128,16 @@ def is_user_mange_package(bsp_package_path, pkg): return False +def need_using_mirror_download(config_file): + """default using mirror url to download packages""" + + if not os.path.isfile(config_file): + return True + elif os.path.isfile(config_file) and find_macro_in_config(config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE'): + return True + + +# noinspection PyUnboundLocalVariable def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" @@ -167,9 +177,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): upstream_change_flag = False try: - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # Check whether the package package url is valid @@ -208,18 +216,13 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): submodule_path = os.path.join(repo_path, '.gitmodules') if os.path.isfile(submodule_path): print("Start to update submodule") - if (not os.path.isfile(env_config_file)) \ - or (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file cmd = 'git submodule update --init --recursive' execute_command(cmd, cwd=repo_path) - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and - find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - + if need_using_mirror_download(env_config_file): if len(replace_list): for item in replace_list: submod_dir_path = os.path.join(repo_path, item[2]) @@ -227,10 +230,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): cmd = 'git remote set-url origin ' + item[0] execute_command(cmd, cwd=submod_dir_path) - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - + if need_using_mirror_download(env_config_file): if os.path.isfile(submodule_path): cmd = 'git checkout .gitmodules' execute_command(cmd, cwd=repo_path) From 31d9106c7c49aeab4396a1537799c939d0bacb6c Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 15:11:17 +0800 Subject: [PATCH 30/48] [update] add some basic logging and optimize code --- cmds/cmd_package/cmd_package_update.py | 94 ++++++++++++++++---------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 9d497faf..500d3088 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -32,6 +32,7 @@ import time import archive import requests +import logging from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ @@ -181,10 +182,10 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # Check whether the package package url is valid - if get_package_url is not None and determine_url_valid(get_package_url): + if get_package_url and determine_url_valid(get_package_url): package_url = get_package_url - if get_ver_sha is not None: + if get_ver_sha: ver_sha = get_ver_sha upstream_change_flag = True @@ -318,6 +319,8 @@ def update_latest_packages(sys_value): message provided by git. """ + logging.info("Begin to update latest version packages") + result = True package_filename = sys_value[3] @@ -428,6 +431,7 @@ def get_git_root_path(repo_path): def pre_package_update(): """ Make preparations before updating the software package. """ + logging.info("Begin prepare package update") bsp_root = Import('bsp_root') env_root = Import('env_root') @@ -559,6 +563,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file def rm_package(dir_remove): + logging.info("remove dir: {0}".format(dir_remove)) + if platform.system() != "Windows": shutil.rmtree(dir_remove) else: @@ -605,15 +611,17 @@ def handle_download_error_packages(sys_value, force_update): Check to see if the packages stored in the Json file list actually exist, and then download the packages if they don't exist. """ + + logging.info("begin to handel download error packages") package_filename = sys_value[3] bsp_packages_path = sys_value[5] with open(package_filename, 'r') as f: - read_back_pkgs_json = json.load(f) + package_json = json.load(f) error_packages_list = [] - for pkg in read_back_pkgs_json: + for pkg in package_json: remove_path = get_package_remove_path(pkg, bsp_packages_path) if os.path.exists(remove_path): continue @@ -622,12 +630,13 @@ def handle_download_error_packages(sys_value, force_update): error_packages_list.append(pkg) # Handle the failed download packages - get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update) + get_flag = error_packages_handle(error_packages_list, package_json, package_filename, force_update) return get_flag def delete_useless_packages(sys_value): + logging.info("Begin to delete useless packages") package_delete_error_list = sys_value[2] bsp_packages_path = sys_value[5] @@ -646,7 +655,48 @@ def delete_useless_packages(sys_value): return True +def is_git_package(pkg, bsp_packages_path): + remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) + remove_path_git = os.path.join(remove_path_with_version, '.git') + return os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git) + + +def delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list): + git_folder_to_remove = remove_path_with_version + + print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) + if force_update: + logging.info("package force update, Begin to remove package {0}".format(git_folder_to_remove)) + if not rm_package(git_folder_to_remove): + print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) + print("Please delete this folder manually.") + else: + print("The folder is managed by git. Do you want to delete this folder?\n") + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + try: + if not rm_package(git_folder_to_remove): + package_delete_fail_list.append(pkg) + print("Error: Please delete the folder manually.") + except Exception as e: + print('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + + +def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname): + if os.path.isdir(remove_path_with_version): + print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) + try: + pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) + except Exception as e: + package_delete_fail_list.append(pkg) + print('Error message:\n%s %s. %s \n\t' % ( + "Delete folder failed, please delete the folder manually", + remove_path_with_version.encode("utf-8"), e)) + + def remove_packages(sys_value, force_update): + logging.info("Begin to remove packages") old_package = sys_value[0] new_package = sys_value[1] package_error_list_filename = sys_value[4] @@ -658,38 +708,12 @@ def remove_packages(sys_value, force_update): for pkg in case_delete: remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) - remove_path_git = os.path.join(remove_path_with_version, '.git') # delete .git directory - if os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git): - git_folder_to_remove = remove_path_with_version - - print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) - if force_update: - if not rm_package(git_folder_to_remove): - print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) - print("Please delete this folder manually.") - else: - print("The folder is managed by git. Do you want to delete this folder?\n") - rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - if rc == 'y' or rc == 'Y': - try: - if not rm_package(git_folder_to_remove): - package_delete_fail_list.append(pkg) - print("Error: Please delete the folder manually.") - except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + if is_git_package(pkg, bsp_packages_path): + delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list) else: - if os.path.isdir(remove_path_with_version): - print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) - try: - pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) - except Exception as e: - package_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", - remove_path_with_version.encode("utf-8"), e)) + delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname) # write error messages with open(package_error_list_filename, 'w') as f: @@ -707,6 +731,8 @@ def install_packages(sys_value, force_update): and then download again when the update command is executed. """ + logging.info("Begin to install packages") + old_package = sys_value[0] new_package = sys_value[1] package_filename = sys_value[3] From 176966b8a0b04937537a8e71accd793b94132ab5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:00:40 +0800 Subject: [PATCH 31/48] [add] is_git_url function --- cmds/cmd_package/cmd_package_update.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 500d3088..8bd890e6 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -138,6 +138,10 @@ def need_using_mirror_download(config_file): return True +def is_git_url(package_url): + return package_url.endswith('.git') + + # noinspection PyUnboundLocalVariable def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" @@ -165,14 +169,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): package_url = package.get_url(pkg['ver']) pkgs_name_in_json = package.get_name() - # print("==================================================>") - # print("packages name :", pkgs_name_in_json.encode("utf-8")) - # print("ver :", pkg['ver']) - # print("url :", package_url.encode("utf-8")) - # print("url_from_json : ", url_from_json.encode("utf-8")) - # print("==================================================>") - - if package_url[-4:] == '.git': + logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) + if is_git_url(package_url): ver_sha = package.get_versha(pkg['ver']) upstream_change_flag = False @@ -193,7 +191,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - if package_url.endswith('.git'): + if is_git_url(package_url): try: repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] From ad28b198fdec174d5f38e0da7670d1daa0b39950 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:11:23 +0800 Subject: [PATCH 32/48] [optimize] code clear up --- cmds/cmd_package/cmd_package_update.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 8bd890e6..c10c140b 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -173,7 +173,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if is_git_url(package_url): ver_sha = package.get_versha(pkg['ver']) - upstream_change_flag = False + upstream_changed = False try: if need_using_mirror_download(env_config_file): @@ -186,7 +186,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if get_ver_sha: ver_sha = get_ver_sha - upstream_change_flag = True + upstream_changed = True except Exception as e: print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") @@ -207,7 +207,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("\nFailed to download software package with git. Please check the network connection.") return False - if upstream_change_flag: + # change upstream to origin url + if upstream_changed: cmd = 'git remote set-url origin ' + url_from_json execute_command(cmd, cwd=repo_path) From ab9c945866c5bc31d4e6cde928e5bee2c3f80328 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:18:43 +0800 Subject: [PATCH 33/48] [update] change default log level --- env.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/env.py b/env.py index 9592964d..06eb18a5 100644 --- a/env.py +++ b/env.py @@ -56,12 +56,13 @@ def init_logger(env_root): os.makedirs(log_path) log_name = os.path.join(log_path, "running_log.txt") - log_format = "%(asctime)s %(name)s %(levelname)s %(pathname)s %(lineno)d %(message)s " + log_format = "%(pathname)s %(lineno)d %(message)s " date_format = '%Y-%m-%d %H:%M:%S %a ' - logging.basicConfig(level=logging.DEBUG, + logging.basicConfig(level=logging.WARNING, format=log_format, datefmt=date_format, - filename=log_name) + # filename=log_name + ) def get_env_root(): From 2dd4b05b2dee587639094cdcd5c885d816665fba Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:25:42 +0800 Subject: [PATCH 34/48] [optimize] install_pkg function --- cmds/cmd_package/cmd_package_update.py | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index c10c140b..5a863380 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -143,7 +143,7 @@ def is_git_url(package_url): # noinspection PyUnboundLocalVariable -def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): +def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): """Install the required packages.""" ret = True @@ -151,33 +151,32 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: - if is_user_mange_package(bsp_package_path, pkg): + if is_user_mange_package(bsp_package_path, package_info): return ret # get the .config file from env - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') package = Package() - pkg_path = pkg['path'] + pkg_path = package_info['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') package.parse(pkg_path) - url_from_json = package.get_url(pkg['ver']) - package_url = package.get_url(pkg['ver']) + url_from_json = package.get_url(package_info['ver']) + package_url = package.get_url(package_info['ver']) pkgs_name_in_json = package.get_name() logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) if is_git_url(package_url): - ver_sha = package.get_versha(pkg['ver']) + ver_sha = package.get_versha(package_info['ver']) upstream_changed = False try: if need_using_mirror_download(env_config_file): - get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) + get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, package_info['ver']) # Check whether the package package url is valid if get_package_url and determine_url_valid(get_package_url): @@ -194,10 +193,11 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): if is_git_url(package_url): try: repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) - repo_path = repo_path + '-' + pkg['ver'] + repo_path = repo_path + '-' + package_info['ver'] repo_path_full = '"' + repo_path + '"' clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + logging.info(clone_cmd) execute_command(clone_cmd, cwd=bsp_package_path) git_check_cmd = 'git checkout -q ' + ver_sha @@ -236,12 +236,12 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): execute_command(cmd, cwd=repo_path) else: # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path, package_url): + if not package.download(package_info['ver'], local_pkgs_path, package_url): return False - pkg_dir = package.get_filename(pkg['ver']) + pkg_dir = package.get_filename(package_info['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) if not archive.package_integrity_test(package_path): print("package : %s is invalid" % package_path.encode("utf-8")) @@ -250,7 +250,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): # unpack package if not os.path.exists(pkg_dir): try: - if not package.unpack(package_path, bsp_package_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): ret = False except Exception as e: os.remove(package_path) @@ -742,14 +742,14 @@ def install_packages(sys_value, force_update): case_download = sub_list(new_package, old_package) packages_download_fail_list = [] - for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): + for package in case_download: + if install_pkg(env_root, pkgs_root, bsp_root, package, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( - pkg['name'], pkg['ver'])) + package['name'], package['ver'])) else: # if package download fails, record it in the packages_download_fail_list - packages_download_fail_list.append(pkg) - print(pkg, 'download failed.') + packages_download_fail_list.append(package) + print(package, 'download failed.') return False # Get the currently updated configuration. From 94b1f79be316ebed65db0bde23263f1dd5e01783 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:48:59 +0800 Subject: [PATCH 35/48] [optimize] extract git_package_install and non_git_package_install process --- cmds/cmd_package/cmd_package_update.py | 186 ++++++++++++++----------- 1 file changed, 101 insertions(+), 85 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 5a863380..9d85185e 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -91,7 +91,7 @@ def modify_submod_file_to_mirror(submodule_path): return replace_list except Exception as e: - print('Error message:%s\t' % e) + logging.warning('Error message:%s\t' % e) def determine_url_valid(url_from_srv): @@ -115,9 +115,8 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - # print('Error message:%s\t' % e) # So much error message should be ignore - print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) + logging.error('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) def is_user_mange_package(bsp_package_path, pkg): @@ -142,17 +141,94 @@ def is_git_url(package_url): return package_url.endswith('.git') +def install_git_package(bsp_package_path, package_name, package_info, package_url, ver_sha, upstream_changed, + url_origin, env_config_file): + try: + repo_path = os.path.join(bsp_package_path, package_name) + repo_path = repo_path + '-' + package_info['ver'] + repo_path_full = '"' + repo_path + '"' + + clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + logging.info(clone_cmd) + execute_command(clone_cmd, cwd=bsp_package_path) + + git_check_cmd = 'git checkout -q ' + ver_sha + execute_command(git_check_cmd, cwd=repo_path) + except Exception as e: + print('Error message:%s' % e) + print("\nFailed to download software package with git. Please check the network connection.") + return False + + # change upstream to origin url + if upstream_changed: + cmd = 'git remote set-url origin ' + url_origin + execute_command(cmd, cwd=repo_path) + + # If there is a .gitmodules file in the package, prepare to update submodule. + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): + print("Start to update submodule") + if need_using_mirror_download(env_config_file): + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file + + cmd = 'git submodule update --init --recursive' + execute_command(cmd, cwd=repo_path) + + if need_using_mirror_download(env_config_file): + if len(replace_list): + for item in replace_list: + submod_dir_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submod_dir_path): + cmd = 'git remote set-url origin ' + item[0] + execute_command(cmd, cwd=submod_dir_path) + + if need_using_mirror_download(env_config_file): + if os.path.isfile(submodule_path): + cmd = 'git checkout .gitmodules' + execute_command(cmd, cwd=repo_path) + + return True + + +def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json): + # Download a package of compressed package type. + if not package.download(package_info['ver'], local_pkgs_path, package_url): + return False + + pkg_dir = package.get_filename(package_info['ver']) + pkg_dir = os.path.splitext(pkg_dir)[0] + package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) + + if not archive.package_integrity_test(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) + return False + + # unpack package + if not os.path.exists(pkg_dir): + try: + if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): + result = False + except Exception as e: + os.remove(package_path) + result = False + print('Error message: %s\t' % e) + else: + print("The file does not exist.") + + return True + + # noinspection PyUnboundLocalVariable -def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): +def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): """Install the required packages.""" - ret = True + result = True local_pkgs_path = os.path.join(env_root, 'local_pkgs') bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: if is_user_mange_package(bsp_package_path, package_info): - return ret + return result # get the .config file from env env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') @@ -174,6 +250,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): upstream_changed = False + # noinspection PyBroadException try: if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, package_info['ver']) @@ -187,78 +264,18 @@ def install_pkg(env_root, pkgs_root, bsp_root, package_info, force_update): upstream_changed = True except Exception as e: - print('Error message:%s\t' % e) - print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + logging.warning("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") if is_git_url(package_url): - try: - repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) - repo_path = repo_path + '-' + package_info['ver'] - repo_path_full = '"' + repo_path + '"' - - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - logging.info(clone_cmd) - execute_command(clone_cmd, cwd=bsp_package_path) - - git_check_cmd = 'git checkout -q ' + ver_sha - execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: - print('Error message:%s' % e) - print("\nFailed to download software package with git. Please check the network connection.") - return False - - # change upstream to origin url - if upstream_changed: - cmd = 'git remote set-url origin ' + url_from_json - execute_command(cmd, cwd=repo_path) - - # If there is a .gitmodules file in the package, prepare to update submodule. - submodule_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submodule_path): - print("Start to update submodule") - if need_using_mirror_download(env_config_file): - replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file - - cmd = 'git submodule update --init --recursive' - execute_command(cmd, cwd=repo_path) - - if need_using_mirror_download(env_config_file): - if len(replace_list): - for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): - cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) - - if need_using_mirror_download(env_config_file): - if os.path.isfile(submodule_path): - cmd = 'git checkout .gitmodules' - execute_command(cmd, cwd=repo_path) + if not install_git_package(bsp_package_path, pkgs_name_in_json, package_info, package_url, ver_sha, + upstream_changed, + url_from_json, env_config_file): + result = False else: - # Download a package of compressed package type. - if not package.download(package_info['ver'], local_pkgs_path, package_url): - return False - - pkg_dir = package.get_filename(package_info['ver']) - pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) - - if not archive.package_integrity_test(package_path): - print("package : %s is invalid" % package_path.encode("utf-8")) - return False - - # unpack package - if not os.path.exists(pkg_dir): - try: - if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): - ret = False - except Exception as e: - os.remove(package_path) - ret = False - print('Error message: %s\t' % e) - else: - print("The file does not exist.") - return ret + if not install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, + pkgs_name_in_json): + result = False + return result def sub_list(aList, bList): @@ -293,7 +310,7 @@ def update_submodule(repo_path): execute_command(cmd, cwd=repo_path) print("Submodule update successful") except Exception as e: - print('Error message:%s' % e) + logging.warning('Error message:%s' % e) def get_pkg_folder_by_orign_path(orign_path, version): @@ -304,7 +321,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + logging.warning('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -381,8 +398,7 @@ def update_latest_packages(sys_value): result = False except Exception as e: - # print("Error message : %s" % e) - print("Failed to connect to the mirror server, using non-mirror server to update.") + logging.warning("Failed to connect to the mirror server, using non-mirror server to update.") if not right_path_flag: continue @@ -420,7 +436,7 @@ def get_git_root_path(repo_path): os.chdir(before) return get_git_root except Exception as e: - print("Error message : %s" % e) + logging.warning("Error message : %s" % e) return None else: print("Missing path %s" % repo_path) @@ -541,7 +557,7 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file # re-download the packages in error_packages_list for pkg in error_packages_list: - if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): + if install_package(env_root, pkgs_root, bsp_root, pkg, force_update): print("\n==============================> %s %s update done \n" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: @@ -678,8 +694,8 @@ def delete_git_package(pkg, remove_path_with_version, force_update, package_dele package_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + logging.warning('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname): @@ -689,7 +705,7 @@ def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) except Exception as e: package_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( + logging.warning('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", remove_path_with_version.encode("utf-8"), e)) @@ -743,7 +759,7 @@ def install_packages(sys_value, force_update): packages_download_fail_list = [] for package in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, package, force_update): + if install_package(env_root, pkgs_root, bsp_root, package, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( package['name'], package['ver'])) else: From ce04de292bced35cffc2ceb107d2ca58e5f1462d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 17:50:46 +0800 Subject: [PATCH 36/48] [update] change pathname to filename --- env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env.py b/env.py index 06eb18a5..ca9daf83 100644 --- a/env.py +++ b/env.py @@ -56,7 +56,7 @@ def init_logger(env_root): os.makedirs(log_path) log_name = os.path.join(log_path, "running_log.txt") - log_format = "%(pathname)s %(lineno)d %(message)s " + log_format = "%(filename)s %(lineno)d %(message)s " date_format = '%Y-%m-%d %H:%M:%S %a ' logging.basicConfig(level=logging.WARNING, format=log_format, From b5a549312d93afc7d6b4bd4d0d4832e350f007e3 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 18:02:46 +0800 Subject: [PATCH 37/48] [update] rename Package class --- cmds/cmd_package/cmd_package_list.py | 4 ++-- cmds/cmd_package/cmd_package_update.py | 14 ++++++-------- package.py | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmds/cmd_package/cmd_package_list.py b/cmds/cmd_package/cmd_package_list.py index 5e7c17b6..1b0cbcc8 100644 --- a/cmds/cmd_package/cmd_package_list.py +++ b/cmds/cmd_package/cmd_package_list.py @@ -26,7 +26,7 @@ import os import platform import kconfig -from package import Package +from package import PackageOperation from vars import Import @@ -55,7 +55,7 @@ def list_packages(): packages = kconfig.parse(config_file) for pkg in packages: - package = Package() + package = PackageOperation() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 9d85185e..64385e13 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -21,6 +21,7 @@ # Change Logs: # Date Author Notes # 2020-04-08 SummerGift Optimize program structure +# 2020-04-13 SummerGift refactoring # import os @@ -33,7 +34,7 @@ import archive import requests import logging -from package import Package, Bridge_SConscript +from package import PackageOperation, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ find_macro_in_config @@ -233,7 +234,7 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): # get the .config file from env env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') - package = Package() + package = PackageOperation() pkg_path = package_info['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] @@ -345,15 +346,14 @@ def update_latest_packages(sys_value): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') with open(package_filename, 'r') as f: read_back_pkgs_json = json.load(f) for pkg in read_back_pkgs_json: right_path_flag = True - package = Package() + package = PackageOperation() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': pkg_path = pkg_path[1:] @@ -370,9 +370,7 @@ def update_latest_packages(sys_value): # noinspection PyBroadException try: # If mirror acceleration is enabled, get the update address from the mirror server. - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") # Change repo's upstream address. diff --git a/package.py b/package.py index 824a9b4e..dd99625a 100644 --- a/package.py +++ b/package.py @@ -130,7 +130,7 @@ ''' -class Package: +class PackageOperation: pkg = None def parse(self, filename): From 9e09486d5988b84ca84d7ae145accfe15f7b8935 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 13 Apr 2020 18:49:58 +0800 Subject: [PATCH 38/48] [update] fix overwrite package problem --- archive.py | 26 +++++++++++++++++++++----- cmds/cmd_package/cmd_package_update.py | 3 ++- env.py | 1 + 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/archive.py b/archive.py index a5c992d8..c6c3d86e 100644 --- a/archive.py +++ b/archive.py @@ -30,12 +30,19 @@ import pkgsdb import platform import shutil +import logging -def unpack(archive_fn, path, pkg, package_name): - pkg_ver = pkg['ver'] +def unpack(archive_fn, path, package_info, package_name): + pkg_ver = package_info['ver'] flag = True + package_temp_path = os.path.join(path, "package_temp") + os.makedirs(package_temp_path) + + logging.info("archive_fn", archive_fn) + logging.info("path", path) + if platform.system() == "Windows": is_windows = True else: @@ -52,6 +59,7 @@ def unpack(archive_fn, path, pkg, package_name): else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) + pkgsdb.save_to_database(a, archive_fn) arch.close() @@ -72,8 +80,8 @@ def unpack(archive_fn, path, pkg, package_name): if ".zip" in archive_fn: arch = zipfile.ZipFile(archive_fn, "r") for item in arch.namelist(): - arch.extract(item, path) - if not os.path.isdir(os.path.join(path, item)): + arch.extract(item, package_temp_path) + if not os.path.isdir(os.path.join(package_temp_path, item)): if is_windows: right_path = item.replace('/', '\\') else: @@ -86,6 +94,7 @@ def unpack(archive_fn, path, pkg, package_name): flag = False right_name_to_db = right_path.replace(dir_name, change_dirname, 1) + right_path = os.path.join("package_temp", right_path) pkgsdb.save_to_database(right_name_to_db, archive_fn, right_path) arch.close() @@ -99,7 +108,14 @@ def unpack(archive_fn, path, pkg, package_name): else: shutil.rmtree(os.path.join(path, change_dirname)) - os.rename(os.path.join(path, dir_name), os.path.join(path, change_dirname)) + rename_path = os.path.join(package_temp_path, change_dirname) + os.rename(os.path.join(package_temp_path, dir_name), rename_path) + + # copy to bsp packages path. + shutil.move(rename_path, os.path.join(path, change_dirname)) + + # remove temp dir + shutil.rmtree(package_temp_path) def package_integrity_test(path): diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 64385e13..d9b3ea9b 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -192,6 +192,7 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json): + result = True # Download a package of compressed package type. if not package.download(package_info['ver'], local_pkgs_path, package_url): return False @@ -216,7 +217,7 @@ def install_not_git_package(package, package_info, local_pkgs_path, package_url, else: print("The file does not exist.") - return True + return result # noinspection PyUnboundLocalVariable diff --git a/env.py b/env.py index ca9daf83..96b54db1 100644 --- a/env.py +++ b/env.py @@ -22,6 +22,7 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2019-1-16 SummerGift Add chinese detection +# 2020-4-13 SummerGift refactoring # import os From b423e83ab6a2d92ff13aca905d97aaecedbce08e Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 14 Apr 2020 18:28:23 +0800 Subject: [PATCH 39/48] [optimize] code clear up --- cmds/cmd_package/cmd_package_update.py | 36 ++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index d9b3ea9b..c011b5db 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -229,8 +229,12 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: + logging.info("Begin to check if it's an user managed package {0}, {1} \n".format(bsp_package_path, package_info)) if is_user_mange_package(bsp_package_path, package_info): + logging.info("User managed package {0}, {1} no need install. \n".format(bsp_package_path, package_info)) return result + else: + logging.info("NOT User managed package {0}, {1} need install. \n".format(bsp_package_path, package_info)) # get the .config file from env env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') @@ -315,8 +319,8 @@ def update_submodule(repo_path): logging.warning('Error message:%s' % e) -def get_pkg_folder_by_orign_path(orign_path, version): - return orign_path + '-' + version +def get_package_folder(origin_path, version): + return origin_path + '-' + version def git_cmd_exec(cmd, cwd): @@ -366,7 +370,7 @@ def update_latest_packages(sys_value): # Find out the packages which version is 'latest' if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) - repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) + repo_path = get_package_folder(repo_path, pkg['ver']) # noinspection PyBroadException try: @@ -427,7 +431,7 @@ def get_git_root_path(repo_path): try: before = os.getcwd() os.chdir(repo_path) - result = os.popen('git rev-parse --show-toplevel') + result = os.popen("git rev-parse --show-toplevel") result = result.read() for line in result.splitlines()[:5]: get_git_root = line @@ -542,7 +546,7 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file bsp_root = Import('bsp_root') env_root = Import('env_root') pkgs_root = Import('pkgs_root') - error_packages_redownload_error_list = [] + download_error = [] flag = True if len(error_packages_list): @@ -560,18 +564,20 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file print("\n==============================> %s %s update done \n" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: - error_packages_redownload_error_list.append(pkg) + download_error.append(pkg) print(pkg, 'download failed.') flag = False - if len(error_packages_redownload_error_list): - print("%s" % error_packages_redownload_error_list) - print("Packages:%s,%s re-download error, you need to use command again to re-download them." - % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + if len(download_error): + print("%s" % download_error) + + for pkg in download_error: + print("Packages:%s, %s re-download error, you can use command to re-download them." + % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_package_json = sub_list(read_back_pkgs_json, error_packages_redownload_error_list) + error_write_back = sub_list(read_back_pkgs_json, download_error) with open(package_filename, 'w') as f: - f.write(json.dumps(write_back_package_json, indent=1)) + f.write(json.dumps(error_write_back, indent=1)) return flag @@ -615,7 +621,7 @@ def get_package_remove_path(pkg, bsp_packages_path): # Handles the deletion of git repository folders with version Numbers remove_path = os.path.join(bsp_packages_path, dir_path) - remove_path_ver = get_pkg_folder_by_orign_path(remove_path, ver) + remove_path_ver = get_package_folder(remove_path, ver) return remove_path_ver @@ -749,6 +755,10 @@ def install_packages(sys_value, force_update): old_package = sys_value[0] new_package = sys_value[1] + + logging.info("old_package {0}".format(old_package)) + logging.info("new_package {0}".format(new_package)) + package_filename = sys_value[3] bsp_root = Import('bsp_root') pkgs_root = Import('pkgs_root') From 9f9490bac45c59c133919641fc740350116d72f3 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 20 Apr 2020 13:53:25 +0800 Subject: [PATCH 40/48] [update] logging --- archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive.py b/archive.py index c6c3d86e..757cc2f7 100644 --- a/archive.py +++ b/archive.py @@ -40,8 +40,8 @@ def unpack(archive_fn, path, package_info, package_name): package_temp_path = os.path.join(path, "package_temp") os.makedirs(package_temp_path) - logging.info("archive_fn", archive_fn) - logging.info("path", path) + logging.info("archive filename : {0}".format(archive_fn)) + logging.info("path {0}".format(path)) if platform.system() == "Windows": is_windows = True From e3eedc7bbce9b706028f243eea0366a93426bfe0 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 20 Apr 2020 15:02:07 +0800 Subject: [PATCH 41/48] [optimize] package unpack process --- archive.py | 124 ++++++++++++++----------- cmds/cmd_package/cmd_package_update.py | 30 +++--- env.py | 6 +- package.py | 12 +-- 4 files changed, 89 insertions(+), 83 deletions(-) diff --git a/archive.py b/archive.py index 757cc2f7..b18b5605 100644 --- a/archive.py +++ b/archive.py @@ -33,89 +33,105 @@ import logging -def unpack(archive_fn, path, package_info, package_name): - pkg_ver = package_info['ver'] - flag = True +def is_windows(): + if platform.system() == "Windows": + return True + else: + return False + + +def remove_folder(folder_path): + if os.path.isdir(folder_path): + if is_windows(): + cmd = 'rd /s /q ' + folder_path + os.system(cmd) + else: + shutil.rmtree(folder_path) + + +def unpack(archive_filename, path, package_info, package_name): + package_version = package_info['ver'] package_temp_path = os.path.join(path, "package_temp") os.makedirs(package_temp_path) - logging.info("archive filename : {0}".format(archive_fn)) - logging.info("path {0}".format(path)) - - if platform.system() == "Windows": - is_windows = True - else: - is_windows = False + logging.info("BSP packages path {0}".format(path)) + logging.info("BSP package temp path: {0}".format(package_temp_path)) + logging.info("archive filename : {0}".format(archive_filename)) - if ".tar.bz2" in archive_fn: - arch = tarfile.open(archive_fn, "r:bz2") + if ".tar.bz2" in archive_filename: + arch = tarfile.open(archive_filename, "r:bz2") for tarinfo in arch: arch.extract(tarinfo, path) a = tarinfo.name if not os.path.isdir(os.path.join(path, a)): - if is_windows: + if is_windows(): right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.save_to_database(a, archive_fn) + pkgsdb.save_to_database(a, archive_filename) arch.close() - if ".tar.gz" in archive_fn: - arch = tarfile.open(archive_fn, "r:gz") + if ".tar.gz" in archive_filename: + arch = tarfile.open(archive_filename, "r:gz") for tarinfo in arch: arch.extract(tarinfo, path) a = tarinfo.name if not os.path.isdir(os.path.join(path, a)): - if is_windows: + if is_windows(): right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.save_to_database(a, archive_fn) + pkgsdb.save_to_database(a, archive_filename) arch.close() - if ".zip" in archive_fn: - arch = zipfile.ZipFile(archive_fn, "r") - for item in arch.namelist(): - arch.extract(item, package_temp_path) - if not os.path.isdir(os.path.join(package_temp_path, item)): - if is_windows: - right_path = item.replace('/', '\\') - else: - right_path = item - - # Gets the folder name and change_dirname only once - if flag: - dir_name = os.path.split(right_path)[0] - change_dirname = package_name + '-' + pkg_ver - flag = False - - right_name_to_db = right_path.replace(dir_name, change_dirname, 1) - right_path = os.path.join("package_temp", right_path) - pkgsdb.save_to_database(right_name_to_db, archive_fn, right_path) - arch.close() - - # Change the folder name - change_dirname = package_name + '-' + pkg_ver - - if os.path.isdir(os.path.join(path, change_dirname)): - if is_windows: - cmd = 'rd /s /q ' + os.path.join(path, change_dirname) - os.system(cmd) - else: - shutil.rmtree(os.path.join(path, change_dirname)) - - rename_path = os.path.join(package_temp_path, change_dirname) + try: + if ".zip" in archive_filename: + flag = True + dir_name = "" + package_name_with_version = "" + + arch = zipfile.ZipFile(archive_filename, "r") + for item in arch.namelist(): + arch.extract(item, package_temp_path) + if not os.path.isdir(os.path.join(package_temp_path, item)): + if is_windows(): + right_path = item.replace('/', '\\') + else: + right_path = item + + # Gets the folder name and changed folder name only once + if flag: + dir_name = os.path.split(right_path)[0] + package_name_with_version = package_name + '-' + package_version + flag = False + + right_name_to_db = right_path.replace(dir_name, package_name_with_version, 1) + right_path = os.path.join("package_temp", right_path) + pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path) + arch.close() + except Exception as e: + # remove temp folder and archive file + logging.warning('unpack error message : {0}'.format(e)) + logging.warning('unpack {0} failed'.format(os.path.basename(archive_filename))) + remove_folder(package_temp_path) + os.remove(archive_filename) + return False + + # rename package folder name + package_name_with_version = package_name + '-' + package_version + rename_path = os.path.join(package_temp_path, package_name_with_version) os.rename(os.path.join(package_temp_path, dir_name), rename_path) - # copy to bsp packages path. - shutil.move(rename_path, os.path.join(path, change_dirname)) + # copy package to bsp packages path. + shutil.move(rename_path, os.path.join(path, package_name_with_version)) - # remove temp dir - shutil.rmtree(package_temp_path) + # remove temp folder + remove_folder(package_temp_path) + return True def package_integrity_test(path): diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index c011b5db..c3282389 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -191,7 +191,7 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur return True -def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json): +def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, package_name): result = True # Download a package of compressed package type. if not package.download(package_info['ver'], local_pkgs_path, package_url): @@ -208,10 +208,9 @@ def install_not_git_package(package, package_info, local_pkgs_path, package_url, # unpack package if not os.path.exists(pkg_dir): try: - if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json): + if not archive.unpack(package_path, bsp_package_path, package_info, package_name): result = False except Exception as e: - os.remove(package_path) result = False print('Error message: %s\t' % e) else: @@ -229,7 +228,8 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): bsp_package_path = os.path.join(bsp_root, 'packages') if not force_update: - logging.info("Begin to check if it's an user managed package {0}, {1} \n".format(bsp_package_path, package_info)) + logging.info( + "Begin to check if it's an user managed package {0}, {1} \n".format(bsp_package_path, package_info)) if is_user_mange_package(bsp_package_path, package_info): logging.info("User managed package {0}, {1} no need install. \n".format(bsp_package_path, package_info)) return result @@ -531,14 +531,14 @@ def pre_package_update(): # read data back from pkgs_error.json with open(pkgs_error_list_fn, 'r') as f: - pkgs_error = json.load(f) + package_error = json.load(f) # create SConscript file if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) - return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, + return [oldpkgs, newpkgs, package_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] @@ -551,29 +551,29 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file if len(error_packages_list): print("\n==============================> Packages list to download : \n") - for pkg in error_packages_list: - print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + for package in error_packages_list: + print("Package name : %s, Ver : %s" % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) print("\nThe packages in the list above are accidentally deleted or renamed.") print("\nIf you manually delete the version suffix of the package folder name, ") print("you can use command to re-download these packages.") print("In case of accidental deletion, the ENV tool will automatically re-download these packages.") # re-download the packages in error_packages_list - for pkg in error_packages_list: - if install_package(env_root, pkgs_root, bsp_root, pkg, force_update): + for package in error_packages_list: + if install_package(env_root, pkgs_root, bsp_root, package, force_update): print("\n==============================> %s %s update done \n" - % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) else: - download_error.append(pkg) - print(pkg, 'download failed.') + download_error.append(package) + print(package, 'download failed.') flag = False if len(download_error): print("%s" % download_error) - for pkg in download_error: + for package in download_error: print("Packages:%s, %s re-download error, you can use command to re-download them." - % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) error_write_back = sub_list(read_back_pkgs_json, download_error) with open(package_filename, 'w') as f: diff --git a/env.py b/env.py index 96b54db1..40875b58 100644 --- a/env.py +++ b/env.py @@ -53,9 +53,9 @@ def init_argparse(): def init_logger(env_root): localtime = time.asctime(time.localtime(time.time())) - log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-")) - os.makedirs(log_path) - log_name = os.path.join(log_path, "running_log.txt") + # log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-")) + # os.makedirs(log_path) + # log_name = os.path.join(log_path, "running_log.txt") log_format = "%(filename)s %(lineno)d %(message)s " date_format = '%Y-%m-%d %H:%M:%S %a ' diff --git a/package.py b/package.py index dd99625a..fb46efc0 100644 --- a/package.py +++ b/package.py @@ -245,14 +245,4 @@ def download(self, ver, path, url_from_srv): return False return ret - @staticmethod - def unpack(package_path, path, pkg, package_name_in_json): - try: - # ignore the return value - archive.unpack(package_path, path, pkg, package_name_in_json) - return True - except Exception as e: - print('unpack error message :%s' % e) - print('unpack %s failed' % os.path.basename(package_path)) - os.remove(package_path) - return False + From 60912ff557ae482b2a2bf89d03e294e245356854 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 20 Apr 2020 15:30:00 +0800 Subject: [PATCH 42/48] [optimize] update logging --- cmds/cmd_package/cmd_package_update.py | 5 +++-- env.py | 7 +------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index c3282389..78860ecd 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -384,7 +384,7 @@ def update_latest_packages(sys_value): # if git root is same as repo path, then change the upstream get_git_root = get_git_root_path(repo_path) - if get_git_root is not None: + if get_git_root: if os.path.normcase(repo_path) == os.path.normcase(get_git_root): if mirror_url[0] is not None: cmd = 'git remote set-url origin ' + mirror_url[0] @@ -442,7 +442,8 @@ def get_git_root_path(repo_path): logging.warning("Error message : %s" % e) return None else: - print("Missing path %s" % repo_path) + logging.warning("Missing path {0}".format(repo_path)) + logging.warning("If you manage this package manually, Env tool will not update it.") return None diff --git a/env.py b/env.py index 40875b58..eabdeba6 100644 --- a/env.py +++ b/env.py @@ -52,12 +52,7 @@ def init_argparse(): def init_logger(env_root): - localtime = time.asctime(time.localtime(time.time())) - # log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-")) - # os.makedirs(log_path) - # log_name = os.path.join(log_path, "running_log.txt") - - log_format = "%(filename)s %(lineno)d %(message)s " + log_format = "%(module)s %(lineno)d %(levelname)s %(message)s \n" date_format = '%Y-%m-%d %H:%M:%S %a ' logging.basicConfig(level=logging.WARNING, format=log_format, From 97cf8eca6e0369d3e5233fa7f1ef95d1425abd91 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Mon, 20 Apr 2020 17:25:42 +0800 Subject: [PATCH 43/48] [fix] package update error --- archive.py | 47 ++++++++++++-------------- cmds/cmd_package/cmd_package_update.py | 24 ++++++++----- cmds/cmd_package/cmd_package_utils.py | 21 ++++++++++-- env.py | 1 - package.py | 15 +++++--- 5 files changed, 67 insertions(+), 41 deletions(-) diff --git a/archive.py b/archive.py index b18b5605..21c4ba4a 100644 --- a/archive.py +++ b/archive.py @@ -24,36 +24,23 @@ # 2020-4-10 SummerGift Code clear up # +import logging +import os +import shutil import tarfile import zipfile -import os import pkgsdb -import platform -import shutil -import logging - - -def is_windows(): - if platform.system() == "Windows": - return True - else: - return False - - -def remove_folder(folder_path): - if os.path.isdir(folder_path): - if is_windows(): - cmd = 'rd /s /q ' + folder_path - os.system(cmd) - else: - shutil.rmtree(folder_path) +from cmds.cmd_package.cmd_package_utils import is_windows, remove_folder def unpack(archive_filename, path, package_info, package_name): package_version = package_info['ver'] - package_temp_path = os.path.join(path, "package_temp") - os.makedirs(package_temp_path) + try: + remove_folder(package_temp_path) + os.makedirs(package_temp_path) + except Exception as e: + logging.warning('Error message : {0}'.format(e)) logging.info("BSP packages path {0}".format(path)) logging.info("BSP package temp path: {0}".format(package_temp_path)) @@ -124,13 +111,23 @@ def unpack(archive_filename, path, package_info, package_name): # rename package folder name package_name_with_version = package_name + '-' + package_version rename_path = os.path.join(package_temp_path, package_name_with_version) - os.rename(os.path.join(package_temp_path, dir_name), rename_path) - # copy package to bsp packages path. - shutil.move(rename_path, os.path.join(path, package_name_with_version)) + logging.info("origin name: {0}".format(os.path.join(package_temp_path, dir_name))) + logging.info("rename name: {0}".format(rename_path)) + + try: + os.rename(os.path.join(package_temp_path, dir_name), rename_path) + except Exception as e: + # remove temp folder and archive file + logging.warning('{0}'.format(e)) + + if not os.path.isdir(os.path.join(path, package_name_with_version)): + # copy package to bsp packages path. + shutil.move(rename_path, os.path.join(path, package_name_with_version)) # remove temp folder remove_folder(package_temp_path) + return True diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 78860ecd..d11cbce5 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -24,16 +24,18 @@ # 2020-04-13 SummerGift refactoring # -import os import json -import kconfig -import pkgsdb -import shutil +import logging +import os import platform +import shutil import time -import archive + import requests -import logging + +import archive +import kconfig +import pkgsdb from package import PackageOperation, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ @@ -212,7 +214,7 @@ def install_not_git_package(package, package_info, local_pkgs_path, package_url, result = False except Exception as e: result = False - print('Error message: %s\t' % e) + logging.error('Error message: {0}'.format(e)) else: print("The file does not exist.") @@ -247,9 +249,13 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): package.parse(pkg_path) url_from_json = package.get_url(package_info['ver']) - package_url = package.get_url(package_info['ver']) - pkgs_name_in_json = package.get_name() + if not url_from_json: + return False + + package_url = url_from_json + + pkgs_name_in_json = package.get_name() logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) if is_git_url(package_url): ver_sha = package.get_versha(package_info['ver']) diff --git a/cmds/cmd_package/cmd_package_utils.py b/cmds/cmd_package/cmd_package_utils.py index 7e7433d9..7821d0d7 100644 --- a/cmds/cmd_package/cmd_package_utils.py +++ b/cmds/cmd_package/cmd_package_utils.py @@ -23,11 +23,13 @@ # 2020-04-08 SummerGift Optimize program structure # +import json +import os import platform import subprocess -import time -import json import sys +import time +import shutil import requests @@ -175,3 +177,18 @@ def find_macro_in_config(filename, macro_name): config.close() return False + +def is_windows(): + if platform.system() == "Windows": + return True + else: + return False + + +def remove_folder(folder_path): + if os.path.isdir(folder_path): + if is_windows(): + cmd = 'rd /s /q ' + folder_path + os.system(cmd) + else: + shutil.rmtree(folder_path) diff --git a/env.py b/env.py index eabdeba6..4a55b309 100644 --- a/env.py +++ b/env.py @@ -29,7 +29,6 @@ import sys import argparse import logging -import time import platform from cmds import * diff --git a/package.py b/package.py index fb46efc0..fded16ef 100644 --- a/package.py +++ b/package.py @@ -25,12 +25,15 @@ # 2020-4-7 SummerGift Code improvement # -import os import json -import archive +import logging +import os import sys + import requests +import archive + """Template for creating a new file""" Bridge_SConscript = '''import os @@ -151,11 +154,15 @@ def get_filename(self, ver): return None def get_url(self, ver): + url = None for item in self.pkg['site']: if item['version'].lower() == ver.lower(): - return item['URL'] + url = item['URL'] - return None + if not url: + logging.warning("Can't find right url {0}, please check {1}".format(ver.lower(), self.pkg['site'])) + + return url def get_versha(self, ver): for item in self.pkg['site']: From a0f8bf1a64c39f5a76c98f6766c556133d2c11bc Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 21 Apr 2020 11:06:04 +0800 Subject: [PATCH 44/48] [update] code clear up --- cmds/cmd_package/cmd_package_update.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index d11cbce5..47e7658c 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -149,9 +149,9 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur try: repo_path = os.path.join(bsp_package_path, package_name) repo_path = repo_path + '-' + package_info['ver'] - repo_path_full = '"' + repo_path + '"' + repo_name_with_version = '"' + repo_path + '"' - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + clone_cmd = 'git clone ' + package_url + ' ' + repo_name_with_version logging.info(clone_cmd) execute_command(clone_cmd, cwd=bsp_package_path) @@ -180,10 +180,10 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur if need_using_mirror_download(env_config_file): if len(replace_list): for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): + submodule_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submodule_path): cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) + execute_command(cmd, cwd=submodule_path) if need_using_mirror_download(env_config_file): if os.path.isfile(submodule_path): From cc5ca61249c20925cd9bc359260f3b70ed75a1f8 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 21 Apr 2020 11:15:45 +0800 Subject: [PATCH 45/48] [optimize] remove_folder function --- cmds/cmd_package/cmd_package_utils.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_package/cmd_package_utils.py b/cmds/cmd_package/cmd_package_utils.py index 7821d0d7..0ae95bcd 100644 --- a/cmds/cmd_package/cmd_package_utils.py +++ b/cmds/cmd_package/cmd_package_utils.py @@ -31,6 +31,7 @@ import time import shutil import requests +import logging def execute_command(cmd_string, cwd=None, shell=True): @@ -186,9 +187,16 @@ def is_windows(): def remove_folder(folder_path): - if os.path.isdir(folder_path): - if is_windows(): - cmd = 'rd /s /q ' + folder_path - os.system(cmd) + try: + if os.path.isdir(folder_path): + if is_windows(): + cmd = 'rd /s /q ' + folder_path + os.system(cmd) + else: + shutil.rmtree(folder_path) + return True else: - shutil.rmtree(folder_path) + return True + except Exception as e: + logging.warning('Error message : {0}'.format(e)) + return False From 528783d0058c2421680fde2cb65eed034216b5ce Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 21 Apr 2020 11:18:00 +0800 Subject: [PATCH 46/48] [optimize] package install process, add more error handle --- archive.py | 116 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/archive.py b/archive.py index 21c4ba4a..ea4996f3 100644 --- a/archive.py +++ b/archive.py @@ -33,25 +33,13 @@ from cmds.cmd_package.cmd_package_utils import is_windows, remove_folder -def unpack(archive_filename, path, package_info, package_name): - package_version = package_info['ver'] - package_temp_path = os.path.join(path, "package_temp") - try: - remove_folder(package_temp_path) - os.makedirs(package_temp_path) - except Exception as e: - logging.warning('Error message : {0}'.format(e)) - - logging.info("BSP packages path {0}".format(path)) - logging.info("BSP package temp path: {0}".format(package_temp_path)) - logging.info("archive filename : {0}".format(archive_filename)) - +def unpack(archive_filename, bsp_package_path, package_info, package_name): if ".tar.bz2" in archive_filename: arch = tarfile.open(archive_filename, "r:bz2") for tarinfo in arch: - arch.extract(tarinfo, path) + arch.extract(tarinfo, bsp_package_path) a = tarinfo.name - if not os.path.isdir(os.path.join(path, a)): + if not os.path.isdir(os.path.join(bsp_package_path, a)): if is_windows(): right_path = a.replace('/', '\\') else: @@ -64,9 +52,9 @@ def unpack(archive_filename, path, package_info, package_name): if ".tar.gz" in archive_filename: arch = tarfile.open(archive_filename, "r:gz") for tarinfo in arch: - arch.extract(tarinfo, path) + arch.extract(tarinfo, bsp_package_path) a = tarinfo.name - if not os.path.isdir(os.path.join(path, a)): + if not os.path.isdir(os.path.join(bsp_package_path, a)): if is_windows(): right_path = a.replace('/', '\\') else: @@ -75,55 +63,85 @@ def unpack(archive_filename, path, package_info, package_name): pkgsdb.save_to_database(a, archive_filename) arch.close() + if ".zip" in archive_filename: + if not handle_zip_package(archive_filename, bsp_package_path, package_name, package_info): + return False + + return True + + +def handle_zip_package(archive_filename, bsp_package_path, package_name, package_info): + package_version = package_info['ver'] + package_temp_path = os.path.join(bsp_package_path, "package_temp") + try: - if ".zip" in archive_filename: - flag = True - dir_name = "" - package_name_with_version = "" - - arch = zipfile.ZipFile(archive_filename, "r") - for item in arch.namelist(): - arch.extract(item, package_temp_path) - if not os.path.isdir(os.path.join(package_temp_path, item)): - if is_windows(): - right_path = item.replace('/', '\\') - else: - right_path = item - - # Gets the folder name and changed folder name only once - if flag: - dir_name = os.path.split(right_path)[0] - package_name_with_version = package_name + '-' + package_version - flag = False - - right_name_to_db = right_path.replace(dir_name, package_name_with_version, 1) - right_path = os.path.join("package_temp", right_path) - pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path) - arch.close() + if remove_folder(package_temp_path): + os.makedirs(package_temp_path) + except Exception as e: + logging.warning('Error message : {0}'.format(e)) + + logging.info("BSP packages path {0}".format(bsp_package_path)) + logging.info("BSP package temp path: {0}".format(package_temp_path)) + logging.info("archive filename : {0}".format(archive_filename)) + + try: + flag = True + package_folder_name = "" + package_name_with_version = "" + arch = zipfile.ZipFile(archive_filename, "r") + for item in arch.namelist(): + arch.extract(item, package_temp_path) + if not os.path.isdir(os.path.join(package_temp_path, item)): + if is_windows(): + right_path = item.replace('/', '\\') + else: + right_path = item + + # Gets the folder name and changed folder name only once + if flag: + package_folder_name = os.path.split(right_path)[0] + package_name_with_version = package_name + '-' + package_version + flag = False + + right_name_to_db = right_path.replace(package_folder_name, package_name_with_version, 1) + right_path = os.path.join("package_temp", right_path) + pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path) + arch.close() + + if not move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, + bsp_package_path): + return False except Exception as e: - # remove temp folder and archive file logging.warning('unpack error message : {0}'.format(e)) logging.warning('unpack {0} failed'.format(os.path.basename(archive_filename))) + # remove temp folder and archive file remove_folder(package_temp_path) os.remove(archive_filename) return False + return True + + +def move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, path): # rename package folder name package_name_with_version = package_name + '-' + package_version rename_path = os.path.join(package_temp_path, package_name_with_version) - - logging.info("origin name: {0}".format(os.path.join(package_temp_path, dir_name))) + logging.info("origin name: {0}".format(os.path.join(package_temp_path, package_folder_name))) logging.info("rename name: {0}".format(rename_path)) try: - os.rename(os.path.join(package_temp_path, dir_name), rename_path) + os.rename(os.path.join(package_temp_path, package_folder_name), rename_path) except Exception as e: - # remove temp folder and archive file logging.warning('{0}'.format(e)) - if not os.path.isdir(os.path.join(path, package_name_with_version)): - # copy package to bsp packages path. - shutil.move(rename_path, os.path.join(path, package_name_with_version)) + try: + # if there is no specified version package in the bsp package path, + # then move package from package_temp to bsp packages path + if not os.path.isdir(os.path.join(path, package_name_with_version)): + shutil.move(rename_path, os.path.join(path, package_name_with_version)) + except Exception as e: + logging.warning('{0}'.format(e)) + return False # remove temp folder remove_folder(package_temp_path) From 66555089cc453ea1d23281c612c2ac1b036a1fb0 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 21 Apr 2020 11:58:43 +0800 Subject: [PATCH 47/48] [optimize] move package function --- archive.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/archive.py b/archive.py index ea4996f3..4dfd64b6 100644 --- a/archive.py +++ b/archive.py @@ -122,31 +122,33 @@ def handle_zip_package(archive_filename, bsp_package_path, package_name, package return True -def move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, path): - # rename package folder name +def move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, + bsp_packages_path): + """move package in temp folder to bsp packages folder.""" + origin_package_folder_path = os.path.join(package_temp_path, package_folder_name) package_name_with_version = package_name + '-' + package_version - rename_path = os.path.join(package_temp_path, package_name_with_version) - logging.info("origin name: {0}".format(os.path.join(package_temp_path, package_folder_name))) - logging.info("rename name: {0}".format(rename_path)) + package_folder_in_temp = os.path.join(package_temp_path, package_name_with_version) + bsp_package_path = os.path.join(bsp_packages_path, package_name_with_version) + logging.info("origin name: {0}".format(origin_package_folder_path)) + logging.info("rename name: {0}".format(package_folder_in_temp)) + result = True try: - os.rename(os.path.join(package_temp_path, package_folder_name), rename_path) - except Exception as e: - logging.warning('{0}'.format(e)) + # rename package folder name to package name with version + os.rename(origin_package_folder_path, package_folder_in_temp) - try: # if there is no specified version package in the bsp package path, - # then move package from package_temp to bsp packages path - if not os.path.isdir(os.path.join(path, package_name_with_version)): - shutil.move(rename_path, os.path.join(path, package_name_with_version)) + # then move package from package_folder_in_temp to bsp_package_path + if not os.path.isdir(bsp_package_path): + shutil.move(package_folder_in_temp, bsp_package_path) except Exception as e: logging.warning('{0}'.format(e)) - return False - - # remove temp folder - remove_folder(package_temp_path) + result = False + finally: + # must remove temp folder + remove_folder(package_temp_path) - return True + return result def package_integrity_test(path): From 0674cdb788861817e65eb9de952e478f09879b08 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 22 Apr 2020 10:54:51 +0800 Subject: [PATCH 48/48] [optimize] package install handle --- cmds/cmd_package/cmd_package_update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py index 47e7658c..25fdc9f5 100644 --- a/cmds/cmd_package/cmd_package_update.py +++ b/cmds/cmd_package/cmd_package_update.py @@ -125,7 +125,10 @@ def determine_url_valid(url_from_srv): def is_user_mange_package(bsp_package_path, pkg): for root, dirs, files in os.walk(bsp_package_path, topdown=True): for name in dirs: - if name.lower() == pkg["name"].lower(): + package_name_lower = pkg["name"].lower() + folder_name_lower = name.lower() + folder_name_common = folder_name_lower.replace("-", "_") + if folder_name_lower == package_name_lower or folder_name_common == package_name_lower: return True break return False