From d5cf840718f9bbbc5bc50abc42ff482b0f0007fe Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 16:14:24 +0800 Subject: [PATCH 01/18] [optimize] code clear up --- archive.py | 8 +++--- cmds/cmd_menuconfig.py | 23 +++++++++-------- env.py | 5 ++-- init_env.py | 11 ++++++--- kconfig.py | 10 +++++--- package.py | 9 ++++--- pkgsdb.py | 56 +++++++++++++++++++++--------------------- 7 files changed, 64 insertions(+), 58 deletions(-) diff --git a/archive.py b/archive.py index 24117f41..b0ea9f1b 100644 --- a/archive.py +++ b/archive.py @@ -83,11 +83,11 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): dir_name = os.path.split(right_path)[0] change_dirname = pkgs_name_in_json + '-' + pkg_ver flag = False - + right_name_to_db = right_path.replace(dir_name, change_dirname, 1) pkgsdb.savetodb(right_name_to_db, archive_fn, right_path) arch.close() - + # Change the folder name change_dirname = pkgs_name_in_json + '-' + pkg_ver @@ -97,8 +97,8 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): os.system(cmd) else: shutil.rmtree(os.path.join(path, change_dirname)) - - os.rename(os.path.join(path, dir_name),os.path.join(path, change_dirname)) + + os.rename(os.path.join(path, dir_name), os.path.join(path, change_dirname)) def packtest(path): diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index a8a75d60..7cb61081 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -32,7 +32,7 @@ def is_pkg_special_config(config_str): - ''' judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER''' + """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER""" if type(config_str) == type('a'): if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')): @@ -43,7 +43,7 @@ def is_pkg_special_config(config_str): def mk_rtconfig(filename): try: config = open(filename, 'r') - except: + except Exception as e: print('open config:%s failed' % filename) return @@ -89,7 +89,7 @@ def mk_rtconfig(filename): if setting[1] == 'y': rtconfig.write('#define %s\n' % setting[0]) else: - rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$",line)[0])) + rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$", line)[0])) if os.path.isfile('rtconfig_project.h'): rtconfig.write('#include "rtconfig_project.h"\n') @@ -102,7 +102,7 @@ def mk_rtconfig(filename): def find_macro_in_config(filename, macro_name): try: config = open(filename, "r") - except: + except Exception as e: print('open .config failed') return @@ -122,7 +122,7 @@ def find_macro_in_config(filename, macro_name): empty_line = 1 continue - #comment_line = line[1:] + # comment_line = line[1:] if line.startswith('# CONFIG_'): line = ' ' + line[9:] else: @@ -146,7 +146,6 @@ def find_macro_in_config(filename, macro_name): def cmd(args): - env_root = Import('env_root') os_version = platform.platform(True)[10:13] kconfig_win7_path = os.path.join( @@ -159,9 +158,9 @@ def cmd(args): print("\n\033[1;31;40m 命令应当在某一特定 BSP 目录下执行，例如：\"rt-thread/bsp/stm32/stm32f091-st-nucleo\"\033[0m") print("\033[1;31;40m请确保当前目录为 BSP 根目录，并且该目录中有 Kconfig 文件。\033[0m\n") - print (" command should be used in a bsp root path with a Kconfig file.") - print ("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") - print ("You should check if there is a Kconfig file in your bsp root first.") + print(" command should be used in a bsp root path with a Kconfig file.") + print("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") + print("You should check if there is a Kconfig file in your bsp root first.") if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -242,13 +241,13 @@ def cmd(args): if find_macro_in_config(fn, 'SYS_CREATE_MDK_IAR_PROJECT'): if find_macro_in_config(fn, 'SYS_CREATE_MDK4'): os.system('scons --target=mdk4 -s') - print("Create mdk4 project done") + print("Create mdk4 project done") elif find_macro_in_config(fn, 'SYS_CREATE_MDK5'): os.system('scons --target=mdk5 -s') - print("Create mdk5 project done") + print("Create mdk5 project done") elif find_macro_in_config(fn, 'SYS_CREATE_IAR'): os.system('scons --target=iar -s') - print("Create iar project done") + print("Create iar project done") def add_parser(sub): diff --git a/env.py b/env.py index 18a4efa7..3dbb3e9a 100644 --- a/env.py +++ b/env.py @@ -53,7 +53,7 @@ def main(): bsp_root = os.getcwd() script_root = os.path.split(os.path.realpath(__file__))[0] env_root = os.getenv("ENV_ROOT") - if env_root == None: + if env_root is None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') else: @@ -62,13 +62,14 @@ def main(): sys.path = sys.path + [os.path.join(script_root)] pkgs_root = os.getenv("PKGS_ROOT") - if pkgs_root == None: + if pkgs_root is None: pkgs_root = os.path.join(env_root, 'packages') Export('env_root') Export('bsp_root') Export('pkgs_root') + # noinspection PyBroadException try: bsp_root.encode('utf-8').decode("ascii") except Exception as e: diff --git a/init_env.py b/init_env.py index 500fb0d9..0c024212 100644 --- a/init_env.py +++ b/init_env.py @@ -25,24 +25,27 @@ from multiprocessing import Process import os -import sys + def run_proc(name, env_root): - exec_file = os.path.join(env_root, "tools\scripts\env.py") + exec_file = os.path.join(env_root, r"tools\scripts\env.py") log_std = os.path.join(env_root, "env_log_std") log_err = os.path.join(env_root, "env_log_err") + # noinspection PyBroadException try: - os.system("python %s package --upgrade 1>%s 2>%s"%(exec_file, log_std, log_err)) + os.system("python %s package --upgrade 1>%s 2>%s" % (exec_file, log_std, log_err)) except Exception as e: print("Auto upgrade failed, please check your network.") pass + def main(): env_root = env_root = os.getenv("ENV_ROOT") p = Process(target=run_proc, args=('upgrade', env_root)) p.start() p.join() -if __name__=='__main__': + +if __name__ == '__main__': main() diff --git a/kconfig.py b/kconfig.py index 0e904203..7c7022b3 100644 --- a/kconfig.py +++ b/kconfig.py @@ -1,4 +1,4 @@ - # -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- # # File : kconfig.py # This file is part of RT-Thread RTOS @@ -50,10 +50,12 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): ret = [] + + # noinspection PyBroadException try: - config = open(filename, "r") - except: - print('open .config failed') + config = open(filename, "r") + except Exception as e: + print('open .config failed') return ret for line in config: diff --git a/package.py b/package.py index 99e83383..c3674eb4 100644 --- a/package.py +++ b/package.py @@ -128,6 +128,7 @@ Return('group') ''' + class Package: pkg = None @@ -185,7 +186,7 @@ def download(self, ver, path, url_from_srv): os.remove(path) else: if archive.packtest(path): - #print "The file is rigit." + # print "The file is rigit." return True else: os.remove(path) @@ -200,7 +201,7 @@ def download(self, ver, path, url_from_srv): print('Start to download package : %s ' % filename.encode("utf-8")) while True: - #print("retryCount : %d"%retryCount) + # print("retryCount : %d"%retryCount) try: r = requests.get(url_from_srv, stream=True, headers=headers) @@ -233,8 +234,8 @@ def download(self, ver, path, url_from_srv): ret = False break except Exception as e: - print(url_from_srv) - print('error message:%s\t' %e) + print(url_from_srv) + print('error message:%s\t' % e) retryCount = retryCount + 1 if retryCount > 5: print('%s download fail!\n' % path.encode("utf-8")) diff --git a/pkgsdb.py b/pkgsdb.py index 488008cb..715d54cb 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -29,6 +29,7 @@ import sys from vars import Import + SHOW_SQL = False @@ -49,7 +50,7 @@ def GetFileMd5(filename): def get_conn(path): conn = sqlite3.connect(path) if os.path.exists(path) and os.path.isfile(path): - #print('on disk:[{}]'.format(path)) + # print('on disk:[{}]'.format(path)) return conn else: conn = None @@ -76,7 +77,7 @@ def create_table(conn, sql): print('执行sql:[{}]'.format(sql)) cu.execute(sql) conn.commit() - #print('create data table successful!') + # print('create data table successful!') close_all(conn) else: print('the [{}] is empty or equal None!'.format(sql)) @@ -113,7 +114,7 @@ def isdataexist(pathname): return ret -#将数据添加到数据库，如果数据库中已经存在则不重复添加 +# 将数据添加到数据库，如果数据库中已经存在则不重复添加 def savetodb(pathname, pkgspathname, before_change_name): dbpathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') @@ -123,14 +124,14 @@ def savetodb(pathname, pkgspathname, before_change_name): save_sql = '''insert into packagefile values (?, ?, ?)''' package = os.path.basename(pkgspathname) md5pathname = os.path.join(bsppkgs, before_change_name) - #print("pathname to save : %s"%pathname) - #print("md5pathname : %s"%md5pathname) - + # print("pathname to save : %s"%pathname) + # print("md5pathname : %s"%md5pathname) + if not os.path.isfile(md5pathname): print("md5pathname is Invalid") - + md5 = GetFileMd5(md5pathname) - #print("md5 to save : %s"%md5) + # print("md5 to save : %s"%md5) data = [(pathname, package, md5)] save(conn, save_sql, data) @@ -140,33 +141,32 @@ def dbdump(dbpathname): c = get_cursor(conn) cursor = c.execute("SELECT pathname, package, md5 from packagefile") for row in cursor: - print("pathname = ", row[0]) - print("package = ", row[1]) - print("md5 = ", row[2], "\n") + print("pathname = ", row[0]) + print("package = ", row[1]) + print("md5 = ", row[2], "\n") conn.close() -#delete the unchanged file - def remove_unchangedfile(pathname, dbpathname, dbsqlname): + """delete unchanged files""" flag = True conn = get_conn(dbpathname) c = get_cursor(conn) - #print('pathname : %s'%pathname) - #print('dbsqlname : %s'%dbsqlname) + # print('pathname : %s'%pathname) + # print('dbsqlname : %s'%dbsqlname) filemd5 = GetFileMd5(pathname) - #print("filemd5 : %s"%filemd5) + # print("filemd5 : %s"%filemd5) dbmd5 = 0 sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' - #print sql + # print sql cursor = c.execute(sql) for row in cursor: dbmd5 = row[0] # fetch md5 from databas - #print("dbmd5 : %s"%dbmd5) + # print("dbmd5 : %s"%dbmd5) if dbmd5 == filemd5: # delete file info from database @@ -175,10 +175,11 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): conn.commit() os.remove(pathname) else: - print ("%s has been changed." % pathname) - print ('Are you sure you want to permanently delete the file: %s?' % - os.path.basename(pathname)) - print ('If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') + print("%s has been changed." % pathname) + print('Are you sure you want to permanently delete the file: %s?' % + os.path.basename(pathname)) + print( + 'If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') if sys.version_info < (3, 0): rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') @@ -196,8 +197,8 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): return flag -#删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 -#如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false +# 删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 +# 如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false def deletepackdir(dirpath, dbpathname): flag = getdirdisplay(dirpath, dbpathname) @@ -209,11 +210,11 @@ def deletepackdir(dirpath, dbpathname): for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(dirpath) - #print "the dir should be delete" + # print "the dir should be delete" return flag -#遍历filepath下所有文件，包括子目录 +# 遍历filepath下所有文件，包括子目录 def displaydir(filepath, basepath, length, dbpathname): flag = True if os.path.isdir(filepath): @@ -225,7 +226,7 @@ def displaydir(filepath, basepath, length, dbpathname): else: pathname = os.path.join(filepath, fi_d) dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] - #print("dbsqlname : %s"%dbsqlname) + # print("dbsqlname : %s"%dbsqlname) if not remove_unchangedfile(pathname, dbpathname, dbsqlname): flag = False return flag @@ -235,6 +236,5 @@ def getdirdisplay(filepath, dbpathname): display = filepath length = len(display) basepath = os.path.basename(filepath) - #print "basepath:",basepath flag = displaydir(filepath, basepath, length, dbpathname) return flag From 08910db124706d32ccda5d2adc6c19368d845d8f Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 16:52:42 +0800 Subject: [PATCH 02/18] [optimize] Code improvement --- package.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/package.py b/package.py index c3674eb4..027bf529 100644 --- a/package.py +++ b/package.py @@ -22,6 +22,7 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2018-12-28 Ernest Chen Add package information and enjoy package maker +# 2020-4-7 SummerGift Code improvement # import os @@ -191,7 +192,7 @@ def download(self, ver, path, url_from_srv): else: os.remove(path) - retryCount = 0 + retry_count = 0 headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', @@ -201,7 +202,7 @@ def download(self, ver, path, url_from_srv): print('Start to download package : %s ' % filename.encode("utf-8")) while True: - # print("retryCount : %d"%retryCount) + # print("retry_count : %d"%retry_count) try: r = requests.get(url_from_srv, stream=True, headers=headers) @@ -216,7 +217,7 @@ def download(self, ver, path, url_from_srv): sys.stdout.write("\rDownloding %d KB" % flush_count) sys.stdout.flush() - retryCount = retryCount + 1 + retry_count = retry_count + 1 if archive.packtest(path): # make sure the file is right ret = True @@ -226,7 +227,7 @@ def download(self, ver, path, url_from_srv): else: if os.path.isfile(path): os.remove(path) - if retryCount > 5: + if retry_count > 5: print( "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) if os.path.isfile(path): @@ -236,21 +237,22 @@ def download(self, ver, path, url_from_srv): except Exception as e: print(url_from_srv) print('error message:%s\t' % e) - retryCount = retryCount + 1 - if retryCount > 5: + retry_count = retry_count + 1 + if retry_count > 5: print('%s download fail!\n' % path.encode("utf-8")) if os.path.isfile(path): os.remove(path) return False return ret - def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): + @staticmethod + def unpack(package_path, path, pkg, package_name_in_json): try: # ignore the return value - archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) + archive.unpack(package_path, path, pkg, package_name_in_json) return True except Exception as e: print('unpack error message :%s' % e) - print('unpack %s failed' % os.path.basename(fullpkg_path)) - os.remove(fullpkg_path) + print('unpack %s failed' % os.path.basename(package_path)) + os.remove(package_path) return False From e32ed122b796545b1d6fa7aec4617159b9399661 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 17:08:59 +0800 Subject: [PATCH 03/18] [optimize] code improvement --- cmds/cmd_package.py | 267 ++++++++++++++++++++++++-------------------- 1 file changed, 144 insertions(+), 123 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 2689b022..182ffa36 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -54,7 +54,6 @@ "* $ sudo apt-get install python-requests\n" "****************************************\n") -"""package command""" def execute_command(cmdstring, cwd=None, shell=True): """Execute the system command at the specified address.""" @@ -72,6 +71,7 @@ def execute_command(cmdstring, cwd=None, shell=True): return stdout_str + def git_pull_repo(repo_path, repo_url=''): if platform.system() == "Windows": cmd = r'git config --local core.autocrlf true' @@ -79,6 +79,7 @@ def git_pull_repo(repo_path, repo_url=''): cmd = r'git pull ' + repo_url execute_command(cmd, cwd=repo_path) + def determine_support_chinese(env_root): get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') if os.path.isfile(get_flag_file_path): @@ -86,6 +87,7 @@ def determine_support_chinese(env_root): else: return False + def user_input(msg, default_value): """Gets the user's keyboard input.""" @@ -103,7 +105,8 @@ def user_input(msg, default_value): return value -def union_input(msg = None): + +def union_input(msg=None): """Gets the union keyboard input.""" if sys.version_info < (3, 0): @@ -226,8 +229,8 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None -def determine_url_valid(url_from_srv): +def determine_url_valid(url_from_srv): headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', @@ -289,14 +292,16 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = False try: - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) # determine whether the package package url is valid - if get_package_url != None and determine_url_valid(get_package_url): + if get_package_url is not None and determine_url_valid(get_package_url): package_url = get_package_url - if get_ver_sha != None: + if get_ver_sha is not None: ver_sha = get_ver_sha upstream_change_flag = True @@ -315,7 +320,6 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): git_check_cmd = 'git checkout -q ' + ver_sha execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: print("\nFailed to download software package with git. Please check the network connection.") return False @@ -325,20 +329,26 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): execute_command(cmd, cwd=repo_path) # If there is a .gitmodules file in the package, prepare to update submodule. - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): print("Start to update submodule") # print("开始更新软件包子模块") - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) \ + or (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + # print("开启了镜像加速，开始修改 .gitmodules 文件") - replace_list = modify_submod_file_to_mirror(submod_path) # Modify .gitmodules file + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file # print("开始执行更新动作") cmd = 'git submodule update --init --recursive' execute_command(cmd, cwd=repo_path) - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and + find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if len(replace_list): for item in replace_list: submod_dir_path = os.path.join(repo_path, item[2]) @@ -346,11 +356,13 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): cmd = 'git remote set-url origin ' + item[0] execute_command(cmd, cwd=submod_dir_path) - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - if os.path.isfile(submod_path): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if os.path.isfile(submodule_path): cmd = 'git checkout .gitmodules' execute_command(cmd, cwd=repo_path) - else: # Download a package of compressed package type. if not package.download(pkg['ver'], local_pkgs_path, package_url): @@ -358,20 +370,20 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): pkg_dir = package.get_filename(pkg['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] - pkg_fullpath = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - if not archive.packtest(pkg_fullpath): - print("package : %s is invalid"%pkg_fullpath.encode("utf-8")) + if not archive.packtest(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) return False - + # unpack package if not os.path.exists(pkg_dir): try: - if not package.unpack(pkg_fullpath, bsp_pkgs_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): ret = False except Exception as e: - os.remove(pkg_fullpath) + os.remove(package_path) ret = False print('error message: %s\t' % e) else: @@ -394,11 +406,11 @@ def package_list(): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - print ('\033[1;31;40mNo system configuration file : .config.\033[0m') - print ('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + print('\033[1;31;40mNo system configuration file : .config.\033[0m') + print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -417,12 +429,12 @@ def package_list(): package.parse(pkg_path) pkgs_name_in_json = package.get_name() - print ("package name : %s, ver : %s "%(pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) if not pkgs: - print ("Packages list is empty.") - print ('You can use < menuconfig > command to select online packages.') - print ('Then use < pkgs --update > command to install them.') + print("Packages list is empty.") + print('You can use < menuconfig > command to select online packages.') + print('Then use < pkgs --update > command to install them.') return @@ -471,7 +483,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e)) + print('error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -512,14 +524,16 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): try: # If mirror acceleration is enabled, get the update address from # the mirror server. - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") # Change repo's upstream address. mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) - if mirror_url[0] != None: + if mirror_url[0] is not None: cmd = 'git remote set-url origin ' + mirror_url[0] git_cmd_exec(cmd, repo_path) @@ -536,13 +550,13 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): # recover origin url to the path which get from packages.json file if package.get_url(pkg['ver']): cmd = 'git remote set-url origin ' + \ - package.get_url(pkg['ver']) + package.get_url(pkg['ver']) git_cmd_exec(cmd, repo_path) else: print("Can't find the package : %s's url in file : %s" % (payload_pkgs_name_in_json, pkg_path)) - print("==============================> %s update done\n" %(pkgs_name_in_json)) + print("==============================> %s update done\n" % (pkgs_name_in_json)) def pre_package_update(): @@ -555,11 +569,11 @@ def pre_package_update(): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - print ('No system configuration file : .config.') - print ('You should use < menuconfig > command to config bsp first.') + print('No system configuration file : .config.') + print('You should use < menuconfig > command to config bsp first.') if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -583,7 +597,7 @@ def pre_package_update(): dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') Export('dbsqlite_pathname') dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - + # Avoid creating tables more than one time if not os.path.isfile(dbsqlite_pathname): conn = pkgsdb.get_conn(dbsqlite_pathname) @@ -634,7 +648,7 @@ def pre_package_update(): # create SConscript file if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - with open(os.path.join(bsp_packages_path, 'SConscript'),'w') as f: + with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] @@ -652,11 +666,11 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): if len(error_packages_list): print("\n==============================> Packages list to download : \n") for pkg in error_packages_list: - print("Package name : %s, Ver : %s"%(pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) print("\nThe packages in the list above are accidentally deleted, env will redownload them.") print("Warning: Packages should be deleted in command.\n") - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + for pkg in error_packages_list: # Redownloaded the packages in error_packages_list if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) @@ -667,8 +681,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): if len(error_packages_redownload_error_list): print("%s" % error_packages_redownload_error_list) - print ("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % + (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) write_back_pkgs_json = sub_list( read_back_pkgs_json, error_packages_redownload_error_list) read_back_pkgs_json = write_back_pkgs_json @@ -684,7 +698,7 @@ def rm_package(dir): if platform.system() != "Windows": shutil.rmtree(dir) else: - dir = '"' + dir + '"' + dir = '"' + dir + '"' cmd = 'rd /s /q ' + dir os.system(cmd) @@ -692,15 +706,15 @@ def rm_package(dir): if platform.system() != "Windows": shutil.rmtree(dir) else: - dir = '"' + dir + '"' + dir = '"' + dir + '"' cmd = 'rmdir /s /q ' + dir os.system(cmd) if os.path.isdir(dir): - print ("Folder path: %s" % dir.encode("utf-8")) + print("Folder path: %s" % dir.encode("utf-8")) return False else: - print ("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) return True @@ -741,7 +755,7 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): continue else: error_packages_list.append(pkg) - + # Handle the failed download packages get_flag = error_packages_handle( error_packages_list, read_back_pkgs_json, pkgs_fn) @@ -756,7 +770,7 @@ def write_storage_file(pkgs_fn, newpkgs): next update. """ - with open(pkgs_fn,'w') as f: + with open(pkgs_fn, 'w') as f: f.write(str(json.dumps(newpkgs, indent=1))) @@ -782,7 +796,7 @@ def package_update(isDeleteOld=False): # According to the env version, whether Chinese output is supported or not if determine_support_chinese(env_root): if platform.system() == "Windows": - os.system('chcp 65001 > nul') + os.system('chcp 65001 > nul') oldpkgs = sys_value[0] newpkgs = sys_value[1] @@ -801,11 +815,12 @@ def package_update(isDeleteOld=False): error_package, bsp_packages_path) if os.path.isdir(removepath_ver): - print("\nError: %s package delete failed, begin to remove it."% + print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) if rm_package(removepath_ver) == False: - print("Error: Delete package %s failed! Please delete the folder manually.\n"%error_package['name'].encode("utf-8")) + print("Error: Delete package %s failed! Please delete the folder manually.\n" % + error_package['name'].encode("utf-8")) return # 1.in old ,not in new : Software packages that need to be removed. @@ -821,13 +836,13 @@ def package_update(isDeleteOld=False): if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): gitdir = removepath_ver - print ("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) + print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) if isDeleteOld: if rm_package(gitdir) == False: print("Floder delete fail: %s" % gitdir.encode("utf-8")) print("Please delete this folder manually.") else: - print ("The folder is managed by git. Do you want to delete this folder?\n") + print("The folder is managed by git. Do you want to delete this folder?\n") if sys.version_info < (3, 0): rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') else: @@ -859,7 +874,7 @@ def package_update(isDeleteOld=False): return else: # write error messages - with open(pkgs_error_list_fn,'w') as f: + with open(pkgs_error_list_fn, 'w') as f: f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) # 2.in new not in old : Software packages to be installed. @@ -878,7 +893,7 @@ def package_update(isDeleteOld=False): # If the PKG download fails, record it in the # pkgs_download_fail_list. pkgs_download_fail_list.append(pkg) - print(pkg, 'download failed.') + print(pkg, 'download failed.') flag = False # Get the currently updated configuration. @@ -886,7 +901,7 @@ def package_update(isDeleteOld=False): # Give hints based on the success of the download. if len(pkgs_download_fail_list): - print("\nPackage download failed list:" ) + print("\nPackage download failed list:") for item in pkgs_download_fail_list: print(item) @@ -909,104 +924,107 @@ def package_update(isDeleteOld=False): flag = False if flag: - print ("Operation completed successfully.") + print("Operation completed successfully.") else: - print ("Operation failed.") - + print("Operation failed.") + + def package_wizard(): """Packages creation wizard. The user enters the package name, version number, category, and automatically generates the package index file. """ # Welcome - print ('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') - - #Simple introduction about the wizard - print ('note :') - print (' \033[5;35;40m[ ]\033[0m means default setting or optional information.') - print (' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') - - #first step - print ('\033[5;33;40m\n1.Please input a new package name :\033[0m') + print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') + + # Simple introduction about the wizard + print('note :') + print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') + print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') + + # first step + print('\033[5;33;40m\n1.Please input a new package name :\033[0m') name = union_input() regular_obj = re.compile('\W') while name == '' or name.isspace() == True or regular_obj.search(name.strip()): if name == '' or name.isspace(): - print ('\033[1;31;40mError: you must input a package name. Try again.\033[0m') + print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') name = union_input() - else: - print ('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') + else: + print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') name = union_input() default_description = 'Please add description of ' + name + ' in English.' - #description = user_input('menuconfig option name,default:\n',default_description) + # description = user_input('menuconfig option name,default:\n',default_description) description = default_description - description_zh = "请添加软件包 " + name +" 的中文描述。" - - #second step + description_zh = "请添加软件包 " + name + " 的中文描述。" + + # second step ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') ver_standard = ver.replace('.', '') - #keyword = user_input('keyword,default:\n', name) + # keyword = user_input('keyword,default:\n', name) keyword = name - #third step + # third step packageclass = ('iot', 'language', 'misc', 'multimedia', 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') - print ('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print ("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') + print( + "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") classnu = union_input() - while classnu == '' or classnu.isdigit()== False or int(classnu) < 1 or int(classnu) >9: - if classnu == '' : - print ('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') - else : - print ('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') + while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: + if classnu == '': + print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') + else: + print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') classnu = union_input() - - pkgsclass = packageclass[int(classnu) - 1] - #fourth step - print ("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") + pkgsclass = packageclass[int(classnu) - 1] + + # fourth step + print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") authorname = union_input() while authorname == '': - print ("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") + print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") authorname = union_input() - #fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + # fifth step + authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') while authoremail == '': - print ('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() - - #sixth step - print ('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') - print ("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") + print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') + authoremail = union_input() + + # sixth step + print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') + print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') license_class = union_input() - while license_class == '' : - print ('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() + while license_class == '': + print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') + license_class = union_input() - if license_class.isdigit()== True and int(license_class) >= 1 and int(license_class) <= 4: + if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: license = license_index[int(license_class) - 1] - else : - license = license_class - - #seventh step - print ('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') - print ("\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - + else: + license = license_class + + # seventh step + print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') + print( + "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + repository = union_input() while repository == '': - print ('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() + print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') + repository = union_input() pkg_path = name if not os.path.exists(pkg_path): os.mkdir(pkg_path) else: - print ("\033[1;31;40mError: the package directory is exits!\033[0m") + print("\033[1;31;40mError: the package directory is exits!\033[0m") s = Template(Kconfig_file) uppername = str.upper(name) @@ -1017,7 +1035,9 @@ def package_wizard(): f.close() s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass,authorname=authorname,authoremail=authoremail, description=description, description_zh=description_zh,version=ver, keyword=keyword,license=license, repository=repository, pkgs_using_name=uppername) + package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + description=description, description_zh=description_zh, version=ver, keyword=keyword, + license=license, repository=repository, pkgs_using_name=uppername) f = open(os.path.join(pkg_path, 'package.json'), 'w') f.write(package) f.close() @@ -1025,9 +1045,10 @@ def package_wizard(): print ('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') print ('Please \033[5;34;40mupdate\033[0m other information of this package based on Kconfig and package.json in directory '+name+'.') + def upgrade_packages_index(): """Update the package repository index.""" - + env_root = Import('env_root') pkgs_root = Import('pkgs_root') env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') @@ -1091,26 +1112,26 @@ def upgrade_env_script(): def package_upgrade(): """Update the package repository directory and env function scripts.""" - + upgrade_packages_index() upgrade_env_script() def package_print_env(): - print ("Here are some environmental variables.") - print ( + print("Here are some environmental variables.") + print( "If you meet some problems,please check them. Make sure the configuration is correct.") - print ("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) - print ("RTT_CC:%s" % (os.getenv("RTT_CC"))) - print ("SCONS:%s" % (os.getenv("SCONS"))) - print ("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) + print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) + print("RTT_CC:%s" % (os.getenv("RTT_CC"))) + print("SCONS:%s" % (os.getenv("SCONS"))) + print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) env_root = os.getenv('ENV_ROOT') if env_root == None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') - print ("ENV_ROOT:%s" % (env_root)) + print("ENV_ROOT:%s" % (env_root)) def cmd(args): From 3be7a9d8c762ea4aa086307aded9e59c0a814b75 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 17:21:52 +0800 Subject: [PATCH 04/18] [optimize] clear up --- cmds/cmd_package.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 182ffa36..24912b8e 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -818,7 +818,7 @@ def package_update(isDeleteOld=False): print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) - if rm_package(removepath_ver) == False: + if not rm_package(removepath_ver): print("Error: Delete package %s failed! Please delete the folder manually.\n" % error_package['name'].encode("utf-8")) return @@ -1042,8 +1042,9 @@ def package_wizard(): f.write(package) f.close() - print ('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') - print ('Please \033[5;34;40mupdate\033[0m other information of this package based on Kconfig and package.json in directory '+name+'.') + print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') + print('Please \033[5;34;40mupdate\033[0m other information of this package ' + 'based on Kconfig and package.json in directory ' + name + '.') def upgrade_packages_index(): @@ -1051,25 +1052,28 @@ def upgrade_packages_index(): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') - if get_package_url != None: + + if get_package_url is not None: git_repo = get_package_url else: print("Failed to get url from mirror server. Using default url.") git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' else: git_repo = 'https://github.com/RT-Thread/packages.git' - + packages_root = pkgs_root pkgs_path = os.path.join(packages_root, 'packages') if not os.path.isdir(pkgs_path): cmd = 'git clone ' + git_repo + ' ' + pkgs_path os.system(cmd) - print ("upgrade from :%s" % (git_repo.encode("utf-8"))) + print("upgrade from :%s" % (git_repo.encode("utf-8"))) else: print("Begin to upgrade env packages.") git_pull_repo(pkgs_path, git_repo) @@ -1093,11 +1097,13 @@ def upgrade_env_script(): print("Begin to upgrade env scripts.") env_root = Import('env_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') - if get_package_url != None: + + if get_package_url is not None: env_scripts_repo = get_package_url else: print("Failed to get url from mirror server. Using default url.") @@ -1119,15 +1125,14 @@ def package_upgrade(): def package_print_env(): print("Here are some environmental variables.") - print( - "If you meet some problems,please check them. Make sure the configuration is correct.") + print("If you meet some problems,please check them. Make sure the configuration is correct.") print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) print("RTT_CC:%s" % (os.getenv("RTT_CC"))) print("SCONS:%s" % (os.getenv("SCONS"))) print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) env_root = os.getenv('ENV_ROOT') - if env_root == None: + if env_root is None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') @@ -1135,7 +1140,7 @@ def package_print_env(): def cmd(args): - """Env's pkgs command execution options.""" + """Env's packages command execution options.""" if args.package_update_y: package_update(True) @@ -1154,12 +1159,12 @@ def cmd(args): def add_parser(sub): - """The pkgs command parser for env.""" + """The packages command parser for env.""" parser = sub.add_parser('package', help=__doc__, description=__doc__) parser.add_argument('--force-update', - help='force update and clean packages, install or remove the packages by your settings in menuconfig', + help='force update and clean packages, install or remove packages by settings in menuconfig', action='store_true', default=False, dest='package_update_y') From 8e4bc4d0f8aad25bd0edd490ad194902f08352cd Mon Sep 17 00:00:00 2001 From: SummerGift Date: Tue, 7 Apr 2020 18:05:07 +0800 Subject: [PATCH 05/18] [update] cmds/cmd_package.py --- cmds/cmd_package.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 24912b8e..c8f550b2 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -110,12 +110,12 @@ def union_input(msg=None): """Gets the union keyboard input.""" if sys.version_info < (3, 0): - if msg != None: + if msg is not None: value = raw_input(msg) else: value = raw_input() else: - if msg != None: + if msg is not None: value = input(msg) else: value = input() @@ -263,7 +263,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): bsp_pkgs_path = os.path.join(bsp_root, 'packages') # get the .config file from env - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') package = Package() @@ -297,7 +297,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - # determine whether the package package url is valid + # Check whether the package package url is valid if get_package_url is not None and determine_url_valid(get_package_url): package_url = get_package_url @@ -500,7 +500,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): env_root = Import('env_root') pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') with open(pkgs_fn, 'r') as f: @@ -632,8 +632,7 @@ def pre_package_update(): # print("oldpkgs", oldpkgs) # regenerate file : packages/pkgs_error.json - pkgs_error_list_fn = os.path.join( - bsp_packages_path, 'pkgs_error.json') + pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') if not os.path.exists(pkgs_error_list_fn): os.chdir(bsp_packages_path) @@ -914,7 +913,7 @@ def package_update(isDeleteOld=False): get_flag = handle_download_error_packages( pkgs_fn, bsp_packages_path) - if get_flag != None: + if get_flag is not None: flag = get_flag # Update the software packages, which the version is 'latest' From 28fdab19581dc5bad4562ec500e80e608ad14011 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:32:31 +0800 Subject: [PATCH 06/18] [optimize] Optimize program structure --- cmds/__init__.py | 12 +++++++ cmds/cmd_package.py | 67 +++------------------------------------- cmds/cmd_package_list.py | 49 +++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 63 deletions(-) create mode 100644 cmds/cmd_package_list.py diff --git a/cmds/__init__.py b/cmds/__init__.py index 1006f6a6..def30d7e 100644 --- a/cmds/__init__.py +++ b/cmds/__init__.py @@ -24,3 +24,15 @@ # __all__ = ['cmd_package', 'cmd_system', 'cmd_menuconfig'] + +try: + import requests +except ImportError: + print("****************************************\n" + "* Import requests module error.\n" + "* Please install requests module first.\n" + "* pip install step:\n" + "* $ pip install requests\n" + "* command install step:\n" + "* $ sudo apt-get install python-requests\n" + "****************************************\n") \ No newline at end of file diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index c8f550b2..01201a0a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -33,7 +33,6 @@ import platform import subprocess import time -import logging import archive import sys import re @@ -41,18 +40,7 @@ from vars import Import, Export from string import Template from .cmd_menuconfig import find_macro_in_config - -try: - import requests -except ImportError: - print("****************************************\n" - "* Import requests module error.\n" - "* Please install requests module first.\n" - "* pip install step:\n" - "* $ pip install requests\n" - "* command install step:\n" - "* $ sudo apt-get install python-requests\n" - "****************************************\n") +from .cmd_package_list import list_packages def execute_command(cmdstring, cwd=None, shell=True): @@ -391,53 +379,6 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): return ret -def package_list(): - """Print the packages list in env. - - Read the.config file in the BSP directory, - and list the version number of the selected package. - """ - - fn = '.config' - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - if not os.path.isfile(fn): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print('\033[1;31;40mNo system configuration file : .config.\033[0m') - print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') - - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return - - pkgs = kconfig.parse(fn) - - for pkg in pkgs: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - pkgs_name_in_json = package.get_name() - print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) - - if not pkgs: - print("Packages list is empty.") - print('You can use < menuconfig > command to select online packages.') - print('Then use < pkgs --update > command to install them.') - return - - def sub_list(aList, bList): """Return the items in aList but not in bList.""" @@ -1147,8 +1088,8 @@ def cmd(args): package_update() elif args.package_create: package_wizard() - elif args.package_list: - package_list() + elif args.list_packages: + list_packages() elif args.package_upgrade: package_upgrade() elif args.package_print_env: @@ -1178,7 +1119,7 @@ def add_parser(sub): help='list target packages', action='store_true', default=False, - dest='package_list') + dest='list_packages') parser.add_argument('--wizard', help='create a new package with wizard', diff --git a/cmds/cmd_package_list.py b/cmds/cmd_package_list.py new file mode 100644 index 00000000..a48f2761 --- /dev/null +++ b/cmds/cmd_package_list.py @@ -0,0 +1,49 @@ +import os +import platform +import kconfig +from package import Package +from vars import Import + + +def list_packages(): + """Print the packages list in env. + + Read the.config file in the BSP directory, + and list the version number of the selected package. + """ + + fn = '.config' + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + if not os.path.isfile(fn): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\033[1;31;40mCan't find system configuration file : .config.\033[0m") + print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return + + pkgs = kconfig.parse(fn) + + for pkg in pkgs: + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + pkgs_name_in_json = package.get_name() + print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + + if not pkgs: + print("Packages list is empty.") + print('You can use < menuconfig > command to select online packages.') + print('Then use < pkgs --update > command to install them.') + return From 8cb4f595ba446a931ed5424b40b2d7a1c85ded6d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:41:41 +0800 Subject: [PATCH 07/18] [update] cmds/cmd_package_list.py --- cmds/cmd_package_list.py | 44 +++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/cmds/cmd_package_list.py b/cmds/cmd_package_list.py index a48f2761..5e7c17b6 100644 --- a/cmds/cmd_package_list.py +++ b/cmds/cmd_package_list.py @@ -1,3 +1,28 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + import os import platform import kconfig @@ -12,25 +37,24 @@ def list_packages(): and list the version number of the selected package. """ - fn = '.config' - env_root = Import('env_root') + config_file = '.config' pkgs_root = Import('pkgs_root') - if not os.path.isfile(fn): + if not os.path.isfile(config_file): if platform.system() == "Windows": os.system('chcp 65001 > nul') - print("\033[1;31;40mCan't find system configuration file : .config.\033[0m") - print('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') + print("\033[1;31;40mWarning: Can't find .config.\033[0m") + print('\033[1;31;40mYou should use command to config bsp first.\033[0m') if platform.system() == "Windows": os.system('chcp 437 > nul') return - pkgs = kconfig.parse(fn) + packages = kconfig.parse(config_file) - for pkg in pkgs: + for pkg in packages: package = Package() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': @@ -39,10 +63,10 @@ def list_packages(): pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') package.parse(pkg_path) - pkgs_name_in_json = package.get_name() - print("package name : %s, ver : %s " % (pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) + package_name_in_json = package.get_name().encode("utf-8") + print("package name : %s, ver : %s " % (package_name_in_json, pkg['ver'].encode("utf-8"))) - if not pkgs: + if not packages: print("Packages list is empty.") print('You can use < menuconfig > command to select online packages.') print('Then use < pkgs --update > command to install them.') From 73ca2acbcbe9fcc238d44e43f1ffbda348ff66b8 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:48:07 +0800 Subject: [PATCH 08/18] [add] cmd_package_wizard.py --- cmds/cmd_package.py | 155 +------------------------------ cmds/cmd_package_wizard.py | 183 +++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 154 deletions(-) create mode 100644 cmds/cmd_package_wizard.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 01201a0a..246ada9a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -35,12 +35,12 @@ import time import archive import sys -import re from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file from vars import Import, Export from string import Template from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages +from .cmd_package_wizard import package_wizard def execute_command(cmdstring, cwd=None, shell=True): @@ -76,41 +76,6 @@ def determine_support_chinese(env_root): return False -def user_input(msg, default_value): - """Gets the user's keyboard input.""" - - if default_value != '': - msg = '%s[%s]' % (msg, default_value) - - print(msg) - if sys.version_info < (3, 0): - value = raw_input() - else: - value = input() - - if value == '': - value = default_value - - return value - - -def union_input(msg=None): - """Gets the union keyboard input.""" - - if sys.version_info < (3, 0): - if msg is not None: - value = raw_input(msg) - else: - value = raw_input() - else: - if msg is not None: - value = input(msg) - else: - value = input() - - return value - - def get_mirror_giturl(submod_name): """Gets the submodule's url on mirror server. @@ -869,124 +834,6 @@ def package_update(isDeleteOld=False): print("Operation failed.") -def package_wizard(): - """Packages creation wizard. - - The user enters the package name, version number, category, and automatically generates the package index file. - """ - # Welcome - print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') - - # Simple introduction about the wizard - print('note :') - print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') - print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') - - # first step - print('\033[5;33;40m\n1.Please input a new package name :\033[0m') - - name = union_input() - regular_obj = re.compile('\W') - while name == '' or name.isspace() == True or regular_obj.search(name.strip()): - if name == '' or name.isspace(): - print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = union_input() - else: - print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') - name = union_input() - - default_description = 'Please add description of ' + name + ' in English.' - # description = user_input('menuconfig option name,default:\n',default_description) - description = default_description - description_zh = "请添加软件包 " + name + " 的中文描述。" - - # second step - ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') - ver_standard = ver.replace('.', '') - # keyword = user_input('keyword,default:\n', name) - keyword = name - - # third step - packageclass = ('iot', 'language', 'misc', 'multimedia', - 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') - print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print( - "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = union_input() - while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: - if classnu == '': - print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') - else: - print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = union_input() - - pkgsclass = packageclass[int(classnu) - 1] - - # fourth step - print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") - - authorname = union_input() - while authorname == '': - print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") - authorname = union_input() - - # fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') - while authoremail == '': - print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() - - # sixth step - print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') - print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") - license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = union_input() - while license_class == '': - print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() - - if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: - license = license_index[int(license_class) - 1] - else: - license = license_class - - # seventh step - print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') - print( - "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - - repository = union_input() - while repository == '': - print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() - - pkg_path = name - if not os.path.exists(pkg_path): - os.mkdir(pkg_path) - else: - print("\033[1;31;40mError: the package directory is exits!\033[0m") - - s = Template(Kconfig_file) - uppername = str.upper(name) - kconfig = s.substitute(name=uppername, description=description, version=ver, - pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) - f = open(os.path.join(pkg_path, 'Kconfig'), 'w') - f.write(kconfig) - f.close() - - s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, - description=description, description_zh=description_zh, version=ver, keyword=keyword, - license=license, repository=repository, pkgs_using_name=uppername) - f = open(os.path.join(pkg_path, 'package.json'), 'w') - f.write(package) - f.close() - - print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') - print('Please \033[5;34;40mupdate\033[0m other information of this package ' - 'based on Kconfig and package.json in directory ' + name + '.') - - def upgrade_packages_index(): """Update the package repository index.""" diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py new file mode 100644 index 00000000..0a3e6a8e --- /dev/null +++ b/cmds/cmd_package_wizard.py @@ -0,0 +1,183 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import re +import sys +from package import Kconfig_file, Package_json_file +from string import Template + + +def user_input(msg, default_value): + """Gets the user's keyboard input.""" + + if default_value != '': + msg = '%s[%s]' % (msg, default_value) + + print(msg) + if sys.version_info < (3, 0): + value = raw_input() + else: + value = input() + + if value == '': + value = default_value + + return value + + +def union_input(msg=None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg is not None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg is not None: + value = input(msg) + else: + value = input() + + return value + + +def package_wizard(): + """Packages creation wizard. + + The user enters the package name, version number, category, and automatically generates the package index file. + """ + # Welcome + print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') + + # Simple introduction about the wizard + print('note :') + print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') + print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') + + # first step + print('\033[5;33;40m\n1.Please input a new package name :\033[0m') + + name = union_input() + regular_obj = re.compile('\W') + while name == '' or name.isspace() == True or regular_obj.search(name.strip()): + if name == '' or name.isspace(): + print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') + name = union_input() + else: + print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') + name = union_input() + + default_description = 'Please add description of ' + name + ' in English.' + # description = user_input('menuconfig option name,default:\n',default_description) + description = default_description + description_zh = "请添加软件包 " + name + " 的中文描述。" + + # second step + ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') + ver_standard = ver.replace('.', '') + # keyword = user_input('keyword,default:\n', name) + keyword = name + + # third step + packageclass = ('iot', 'language', 'misc', 'multimedia', + 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') + print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') + print( + "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + classnu = union_input() + while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: + if classnu == '': + print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') + else: + print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') + classnu = union_input() + + pkgsclass = packageclass[int(classnu) - 1] + + # fourth step + print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") + + authorname = union_input() + while authorname == '': + print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") + authorname = union_input() + + # fifth step + authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + while authoremail == '': + print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') + authoremail = union_input() + + # sixth step + print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') + print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") + license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') + license_class = union_input() + while license_class == '': + print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') + license_class = union_input() + + if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: + license = license_index[int(license_class) - 1] + else: + license = license_class + + # seventh step + print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') + print( + "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + + repository = union_input() + while repository == '': + print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') + repository = union_input() + + pkg_path = name + if not os.path.exists(pkg_path): + os.mkdir(pkg_path) + else: + print("\033[1;31;40mError: the package directory is exits!\033[0m") + + s = Template(Kconfig_file) + uppername = str.upper(name) + kconfig = s.substitute(name=uppername, description=description, version=ver, + pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) + f = open(os.path.join(pkg_path, 'Kconfig'), 'w') + f.write(kconfig) + f.close() + + s = Template(Package_json_file) + package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + description=description, description_zh=description_zh, version=ver, keyword=keyword, + license=license, repository=repository, pkgs_using_name=uppername) + f = open(os.path.join(pkg_path, 'package.json'), 'w') + f.write(package) + f.close() + + print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') + print('Please \033[5;34;40mupdate\033[0m other information of this package ' + 'based on Kconfig and package.json in directory ' + name + '.') From f3cfd2b7b4b23fe79f43922b9ede0f07cd6b9d33 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 09:55:14 +0800 Subject: [PATCH 09/18] [add] cmds/cmd_package_printenv.py --- cmds/cmd_package.py | 18 +-------------- cmds/cmd_package_printenv.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 cmds/cmd_package_printenv.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 246ada9a..22afc2e4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -37,10 +37,10 @@ import sys from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file from vars import Import, Export -from string import Template from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard +from .cmd_package_printenv import package_print_env def execute_command(cmdstring, cwd=None, shell=True): @@ -910,22 +910,6 @@ def package_upgrade(): upgrade_env_script() -def package_print_env(): - print("Here are some environmental variables.") - print("If you meet some problems,please check them. Make sure the configuration is correct.") - print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) - print("RTT_CC:%s" % (os.getenv("RTT_CC"))) - print("SCONS:%s" % (os.getenv("SCONS"))) - print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) - - env_root = os.getenv('ENV_ROOT') - if env_root is None: - if platform.system() != 'Windows': - env_root = os.path.join(os.getenv('HOME'), '.env') - - print("ENV_ROOT:%s" % (env_root)) - - def cmd(args): """Env's packages command execution options.""" diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package_printenv.py new file mode 100644 index 00000000..f3e1a529 --- /dev/null +++ b/cmds/cmd_package_printenv.py @@ -0,0 +1,43 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import platform + + +def package_print_env(): + print("Here are some environmental variables.") + print("If you meet some problems,please check them. Make sure the configuration is correct.") + print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) + print("RTT_CC:%s" % (os.getenv("RTT_CC"))) + print("SCONS:%s" % (os.getenv("SCONS"))) + print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) + + env_root = os.getenv('ENV_ROOT') + if env_root is None: + if platform.system() != 'Windows': + env_root = os.path.join(os.getenv('HOME'), '.env') + + print("ENV_ROOT:%s" % (env_root)) From 662ed0b0d9cfa6f0fe33d783149ef7a5863df7f2 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:06:34 +0800 Subject: [PATCH 10/18] [optimize] add cmd_package_upgrade.py and cmd_package_utils.py --- cmds/cmd_package.py | 160 +----------------------------------- cmds/cmd_package_upgrade.py | 105 +++++++++++++++++++++++ cmds/cmd_package_utils.py | 113 +++++++++++++++++++++++++ cmds/cmd_package_wizard.py | 2 +- 4 files changed, 220 insertions(+), 160 deletions(-) create mode 100644 cmds/cmd_package_upgrade.py create mode 100644 cmds/cmd_package_utils.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 22afc2e4..1d6dd09a 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -41,31 +41,7 @@ from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env - - -def execute_command(cmdstring, cwd=None, shell=True): - """Execute the system command at the specified address.""" - - if shell: - cmdstring_list = cmdstring - - sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, shell=shell, bufsize=4096) - - stdout_str = '' - while sub.poll() is None: - stdout_str += str(sub.stdout.read()) - time.sleep(0.1) - - return stdout_str - - -def git_pull_repo(repo_path, repo_url=''): - if platform.system() == "Windows": - cmd = r'git config --local core.autocrlf true' - execute_command(cmd, cwd=repo_path) - cmd = r'git pull ' + repo_url - execute_command(cmd, cwd=repo_path) +from .cmd_package_upgrade import package_upgrade def determine_support_chinese(env_root): @@ -125,64 +101,6 @@ def modify_submod_file_to_mirror(submod_path): print('error message:%s\t' % e) -def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): - """Get the download address from the mirror server based on the package name.""" - - try: - if type(pkgs_name_in_json) != type("str"): - if sys.version_info < (3, 0): - pkgs_name_in_json = str(pkgs_name_in_json) - else: - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - - payload = { - "userName": "RT-Thread", - "packages": [ - { - "name": "NULL", - } - ] - } - payload["packages"][0]['name'] = pkgs_name_in_json - - # print(payload) - - try: - r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) - - if r.status_code == requests.codes.ok: - package_info = json.loads(r.text) - - # Can't find package,change git package SHA if it's a git - # package - if len(package_info['packages']) == 0: - print("Package was NOT found on mirror server. Using a non-mirrored address to download.") - return None, None - else: - for item in package_info['packages'][0]['packages_info']['site']: - if item['version'] == pkgs_ver: - # Change download url - download_url = item['URL'] - if download_url[-4:] == '.git': - # Change git package SHA - repo_sha = item['VER_SHA'] - return download_url, repo_sha - return download_url, None - - print("\nTips : \nThe system needs to be upgraded.") - print("Please use the command to upgrade packages index.\n") - return None, None - - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - - def determine_url_valid(url_from_srv): headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', @@ -834,82 +752,6 @@ def package_update(isDeleteOld=False): print("Operation failed.") -def upgrade_packages_index(): - """Update the package repository index.""" - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') - - if get_package_url is not None: - git_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' - else: - git_repo = 'https://github.com/RT-Thread/packages.git' - - packages_root = pkgs_root - pkgs_path = os.path.join(packages_root, 'packages') - - if not os.path.isdir(pkgs_path): - cmd = 'git clone ' + git_repo + ' ' + pkgs_path - os.system(cmd) - print("upgrade from :%s" % (git_repo.encode("utf-8"))) - else: - print("Begin to upgrade env packages.") - git_pull_repo(pkgs_path, git_repo) - print("==============================> Env packages upgrade done \n") - - for filename in os.listdir(packages_root): - package_path = os.path.join(packages_root, filename) - if os.path.isdir(package_path): - - if package_path == pkgs_path: - continue - - if os.path.isdir(os.path.join(package_path, '.git')): - print("Begin to upgrade %s." % filename) - git_pull_repo(package_path) - print("==============================> Env %s update done \n" % filename) - - -def upgrade_env_script(): - """Update env function scripts.""" - - print("Begin to upgrade env scripts.") - env_root = Import('env_root') - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') - - if get_package_url is not None: - env_scripts_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' - else: - env_scripts_repo = 'https://github.com/RT-Thread/env.git' - - env_scripts_root = os.path.join(env_root, 'tools', 'scripts') - git_pull_repo(env_scripts_root, env_scripts_repo) - print("==============================> Env scripts upgrade done \n") - - -def package_upgrade(): - """Update the package repository directory and env function scripts.""" - - upgrade_packages_index() - upgrade_env_script() - - def cmd(args): """Env's packages command execution options.""" diff --git a/cmds/cmd_package_upgrade.py b/cmds/cmd_package_upgrade.py new file mode 100644 index 00000000..982cfe5d --- /dev/null +++ b/cmds/cmd_package_upgrade.py @@ -0,0 +1,105 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +from vars import Import +from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server +from .cmd_menuconfig import find_macro_in_config + + +def upgrade_packages_index(): + """Update the package repository index.""" + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') + + if get_package_url is not None: + git_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' + else: + git_repo = 'https://github.com/RT-Thread/packages.git' + + packages_root = pkgs_root + pkgs_path = os.path.join(packages_root, 'packages') + + if not os.path.isdir(pkgs_path): + cmd = 'git clone ' + git_repo + ' ' + pkgs_path + os.system(cmd) + print("upgrade from :%s" % (git_repo.encode("utf-8"))) + else: + print("Begin to upgrade env packages.") + git_pull_repo(pkgs_path, git_repo) + print("==============================> Env packages upgrade done \n") + + for filename in os.listdir(packages_root): + package_path = os.path.join(packages_root, filename) + if os.path.isdir(package_path): + + if package_path == pkgs_path: + continue + + if os.path.isdir(os.path.join(package_path, '.git')): + print("Begin to upgrade %s." % filename) + git_pull_repo(package_path) + print("==============================> Env %s update done \n" % filename) + + +def upgrade_env_script(): + """Update env function scripts.""" + + print("Begin to upgrade env scripts.") + env_root = Import('env_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') + + if get_package_url is not None: + env_scripts_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' + else: + env_scripts_repo = 'https://github.com/RT-Thread/env.git' + + env_scripts_root = os.path.join(env_root, 'tools', 'scripts') + git_pull_repo(env_scripts_root, env_scripts_repo) + print("==============================> Env scripts upgrade done \n") + + +def package_upgrade(): + """Update the package repository directory and env function scripts.""" + + upgrade_packages_index() + upgrade_env_script() diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py new file mode 100644 index 00000000..306215ca --- /dev/null +++ b/cmds/cmd_package_utils.py @@ -0,0 +1,113 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import platform +import subprocess +import time +import json +import requests + + +def execute_command(cmdstring, cwd=None, shell=True): + """Execute the system command at the specified address.""" + + if shell: + cmdstring_list = cmdstring + + sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, shell=shell, bufsize=4096) + + stdout_str = '' + while sub.poll() is None: + stdout_str += str(sub.stdout.read()) + time.sleep(0.1) + + return stdout_str + + +def git_pull_repo(repo_path, repo_url=''): + if platform.system() == "Windows": + cmd = r'git config --local core.autocrlf true' + execute_command(cmd, cwd=repo_path) + cmd = r'git pull ' + repo_url + execute_command(cmd, cwd=repo_path) + + +def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): + """Get the download address from the mirror server based on the package name.""" + + try: + if type(pkgs_name_in_json) != type("str"): + if sys.version_info < (3, 0): + pkgs_name_in_json = str(pkgs_name_in_json) + else: + pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] + except Exception as e: + print('error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None + + payload = { + "userName": "RT-Thread", + "packages": [ + { + "name": "NULL", + } + ] + } + payload["packages"][0]['name'] = pkgs_name_in_json + + # print(payload) + + try: + r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) + + if r.status_code == requests.codes.ok: + package_info = json.loads(r.text) + + # Can't find package,change git package SHA if it's a git + # package + if len(package_info['packages']) == 0: + print("Package was NOT found on mirror server. Using a non-mirrored address to download.") + return None, None + else: + for item in package_info['packages'][0]['packages_info']['site']: + if item['version'] == pkgs_ver: + # Change download url + download_url = item['URL'] + if download_url[-4:] == '.git': + # Change git package SHA + repo_sha = item['VER_SHA'] + return download_url, repo_sha + return download_url, None + + print("\nTips : \nThe system needs to be upgraded.") + print("Please use the command to upgrade packages index.\n") + return None, None + + except Exception as e: + print('error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py index 0a3e6a8e..10808d83 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package_wizard.py @@ -2,7 +2,7 @@ # # File : cmd_package.py # This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From e4ed8412165b5f00e4a63099fd5cca2bda117002 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:31:05 +0800 Subject: [PATCH 11/18] [update] cmd_package.py --- cmds/cmd_package.py | 6 ++++-- cmds/cmd_package_utils.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 1d6dd09a..eb493fe4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -23,6 +23,7 @@ # 2018-05-28 SummerGift Add copyright information # 2018-12-28 Ernest Chen Add package information and enjoy package maker # 2019-01-07 SummerGift The prompt supports utf-8 encoding +# 2020-04-08 SummerGift Optimize program structure # import os @@ -31,17 +32,18 @@ import pkgsdb import shutil import platform -import subprocess import time import archive import sys -from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file +import requests +from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_menuconfig import find_macro_in_config from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo def determine_support_chinese(env_root): diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 306215ca..5a8944a3 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -27,6 +27,7 @@ import subprocess import time import json +import sys import requests From cfda04f5d74d77474344cf13b96728dffeff50d5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 10:41:14 +0800 Subject: [PATCH 12/18] [optimize] error message --- cmds/cmd_package.py | 12 ++++++------ cmds/cmd_package_utils.py | 25 ++++++++++--------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index eb493fe4..1b807767 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -100,7 +100,7 @@ def modify_submod_file_to_mirror(submod_path): return replace_list except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) def determine_url_valid(url_from_srv): @@ -123,7 +123,7 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -179,7 +179,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = True except Exception as e: - print('error message:%s\t' % e) + print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") if package_url[-4:] == '.git': @@ -258,7 +258,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): except Exception as e: os.remove(package_path) ret = False - print('error message: %s\t' % e) + print('Error message: %s\t' % e) else: print("The file does not exist.") return ret @@ -309,7 +309,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -364,7 +364,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): git_cmd_exec(cmd, repo_path) except Exception as e: - print("error message : %s" % e) + print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") # Update the package repository from upstream. diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 5a8944a3..80b50e32 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -31,13 +31,10 @@ import requests -def execute_command(cmdstring, cwd=None, shell=True): +def execute_command(cmd_string, cwd=None, shell=True): """Execute the system command at the specified address.""" - if shell: - cmdstring_list = cmdstring - - sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, + sub = subprocess.Popen(cmd_string, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=shell, bufsize=4096) stdout_str = '' @@ -56,17 +53,17 @@ def git_pull_repo(repo_path, repo_url=''): execute_command(cmd, cwd=repo_path) -def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): +def get_url_from_mirror_server(package_name, package_version): """Get the download address from the mirror server based on the package name.""" try: - if type(pkgs_name_in_json) != type("str"): + if isinstance(package_name, str): if sys.version_info < (3, 0): - pkgs_name_in_json = str(pkgs_name_in_json) + package_name = str(package_name) else: - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] + package_name = str(package_name)[2:-1] except Exception as e: - print('error message:%s' % e) + print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None @@ -78,9 +75,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } ] } - payload["packages"][0]['name'] = pkgs_name_in_json - - # print(payload) + payload["packages"][0]['name'] = package_name try: r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) @@ -95,7 +90,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None else: for item in package_info['packages'][0]['packages_info']['site']: - if item['version'] == pkgs_ver: + if item['version'] == package_version: # Change download url download_url = item['URL'] if download_url[-4:] == '.git': @@ -109,6 +104,6 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None except Exception as e: - print('error message:%s' % e) + print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None From 4a7a9ffd5170e0522fc73193c916247e1ec39ec5 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 11:34:27 +0800 Subject: [PATCH 13/18] [optimize] extract user_input to cmd_package_utils.py --- cmds/cmd_package.py | 12 ++-- cmds/cmd_package_utils.py | 17 ++++++ cmds/cmd_package_wizard.py | 114 ++++++++++++++----------------------- 3 files changed, 64 insertions(+), 79 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 1b807767..54a074b4 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -43,7 +43,7 @@ from .cmd_package_wizard import package_wizard from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input def determine_support_chinese(env_root): @@ -663,19 +663,15 @@ def package_update(isDeleteOld=False): print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) if isDeleteOld: - if rm_package(gitdir) == False: + if not rm_package(gitdir): print("Floder delete fail: %s" % gitdir.encode("utf-8")) print("Please delete this folder manually.") else: print("The folder is managed by git. Do you want to delete this folder?\n") - if sys.version_info < (3, 0): - rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - else: - rc = input('Press the Y Key to delete the folder or just press Enter to keep it : ') - + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': try: - if rm_package(gitdir) == False: + if not rm_package(gitdir): pkgs_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: diff --git a/cmds/cmd_package_utils.py b/cmds/cmd_package_utils.py index 80b50e32..5b56d0bf 100644 --- a/cmds/cmd_package_utils.py +++ b/cmds/cmd_package_utils.py @@ -107,3 +107,20 @@ def get_url_from_mirror_server(package_name, package_version): print('Error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None + + +def user_input(msg=None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg is not None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg is not None: + value = input(msg) + else: + value = input() + + return value diff --git a/cmds/cmd_package_wizard.py b/cmds/cmd_package_wizard.py index 10808d83..949a3ef6 100644 --- a/cmds/cmd_package_wizard.py +++ b/cmds/cmd_package_wizard.py @@ -28,41 +28,7 @@ import sys from package import Kconfig_file, Package_json_file from string import Template - - -def user_input(msg, default_value): - """Gets the user's keyboard input.""" - - if default_value != '': - msg = '%s[%s]' % (msg, default_value) - - print(msg) - if sys.version_info < (3, 0): - value = raw_input() - else: - value = input() - - if value == '': - value = default_value - - return value - - -def union_input(msg=None): - """Gets the union keyboard input.""" - - if sys.version_info < (3, 0): - if msg is not None: - value = raw_input(msg) - else: - value = raw_input() - else: - if msg is not None: - value = input(msg) - else: - value = input() - - return value +from .cmd_package_utils import user_input def package_wizard(): @@ -70,6 +36,7 @@ def package_wizard(): The user enters the package name, version number, category, and automatically generates the package index file. """ + # Welcome print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') @@ -81,80 +48,85 @@ def package_wizard(): # first step print('\033[5;33;40m\n1.Please input a new package name :\033[0m') - name = union_input() + name = user_input() regular_obj = re.compile('\W') while name == '' or name.isspace() == True or regular_obj.search(name.strip()): if name == '' or name.isspace(): print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = union_input() + name = user_input() else: print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') - name = union_input() + name = user_input() default_description = 'Please add description of ' + name + ' in English.' - # description = user_input('menuconfig option name,default:\n',default_description) description = default_description description_zh = "请添加软件包 " + name + " 的中文描述。" # second step - ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') + print("\033[5;33;40m\n2.Please input this package version, default : '1.0.0' \033[0m") + ver = user_input() + if ver == '': + print("using default version 1.0.0") + ver = '1.0.0' + ver_standard = ver.replace('.', '') - # keyword = user_input('keyword,default:\n', name) keyword = name # third step - packageclass = ('iot', 'language', 'misc', 'multimedia', - 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') + package_class_list = ('iot', 'language', 'misc', 'multimedia', + 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print( - "\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = union_input() - while classnu == '' or classnu.isdigit() == False or int(classnu) < 1 or int(classnu) > 9: - if classnu == '': + print("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|" + "[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + + class_number = user_input() + while class_number == '' or class_number.isdigit() is False or int(class_number) < 1 or int(class_number) > 9: + if class_number == '': print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') else: print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = union_input() + class_number = user_input() - pkgsclass = packageclass[int(classnu) - 1] + package_class = package_class_list[int(class_number) - 1] # fourth step print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") - authorname = union_input() - while authorname == '': + author_name = user_input() + while author_name == '': print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") - authorname = union_input() + author_name = user_input() # fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') - while authoremail == '': + author_email = user_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + while author_email == '': print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() + author_email = user_input() - # sixth step + # sixth step print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = union_input() + license_class = user_input() while license_class == '': print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() + license_class = user_input() - if license_class.isdigit() == True and int(license_class) >= 1 and int(license_class) <= 4: - license = license_index[int(license_class) - 1] + if license_class.isdigit() and 1 <= int(license_class) <= 4: + license_choice = license_index[int(license_class) - 1] else: - license = license_class + license_choice = license_class - # seventh step + # seventh step print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') print( - "\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + "\033[1;32;40mFor example, hello package's repository url " + "is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - repository = union_input() + repository = user_input() while repository == '': print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() + repository = user_input() pkg_path = name if not os.path.exists(pkg_path): @@ -163,17 +135,17 @@ def package_wizard(): print("\033[1;31;40mError: the package directory is exits!\033[0m") s = Template(Kconfig_file) - uppername = str.upper(name) - kconfig = s.substitute(name=uppername, description=description, version=ver, - pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) + upper_name = str.upper(name) + kconfig = s.substitute(name=upper_name, description=description, version=ver, + pkgs_class=package_class, lowercase_name=name, version_standard=ver_standard) f = open(os.path.join(pkg_path, 'Kconfig'), 'w') f.write(kconfig) f.close() s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass, authorname=authorname, authoremail=authoremail, + package = s.substitute(name=name, pkgsclass=package_class, authorname=author_name, authoremail=author_email, description=description, description_zh=description_zh, version=ver, keyword=keyword, - license=license, repository=repository, pkgs_using_name=uppername) + license=license_choice, repository=repository, pkgs_using_name=upper_name) f = open(os.path.join(pkg_path, 'package.json'), 'w') f.write(package) f.close() From 9b26ef42ae6854ac7c645c39eebc3186cbac0357 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 11:50:24 +0800 Subject: [PATCH 14/18] [optimize] cmds/cmd_package.py --- cmds/cmd_package.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 54a074b4..b3e8a6f7 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -34,7 +34,6 @@ import platform import time import archive -import sys import requests from package import Package, Bridge_SConscript from vars import Import, Export @@ -687,26 +686,22 @@ def package_update(isDeleteOld=False): print('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) + # write error messages + with open(pkgs_error_list_fn, 'w') as f: + f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + if len(pkgs_delete_fail_list): - # write error messages - pkgs_file = file(pkgs_error_list_fn, 'w') - pkgs_file.write(json.dumps(pkgs_delete_fail_list, indent=1)) - pkgs_file.close() return - else: - # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) # 2.in new not in old : Software packages to be installed. # If the package download fails, record it, and then download again when # the update command is executed. - casedownload = sub_list(newpkgs, oldpkgs) + case_download = sub_list(newpkgs, oldpkgs) # print 'in new not in old:', casedownload pkgs_download_fail_list = [] - for pkg in casedownload: + for pkg in case_download: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) From 01a62b46ed5f35bf6533daf99c57e63a01052977 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 14:03:14 +0800 Subject: [PATCH 15/18] [optimize] add cmd_package_update.py and code clear up --- cmds/cmd_package.py | 733 +--------------------------------- cmds/cmd_package_printenv.py | 4 + cmds/cmd_package_update.py | 737 +++++++++++++++++++++++++++++++++++ 3 files changed, 750 insertions(+), 724 deletions(-) create mode 100644 cmds/cmd_package_update.py diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index b3e8a6f7..1ec8fe9b 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -2,7 +2,7 @@ # # File : cmd_package.py # This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,729 +26,17 @@ # 2020-04-08 SummerGift Optimize program structure # -import os -import json -import kconfig -import pkgsdb -import shutil -import platform -import time -import archive -import requests -from package import Package, Bridge_SConscript -from vars import Import, Export -from .cmd_menuconfig import find_macro_in_config +from .cmd_package_printenv import package_print_env, package_print_help from .cmd_package_list import list_packages from .cmd_package_wizard import package_wizard -from .cmd_package_printenv import package_print_env from .cmd_package_upgrade import package_upgrade -from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input +from .cmd_package_update import package_update -def determine_support_chinese(env_root): - get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') - if os.path.isfile(get_flag_file_path): - return True - else: - return False - - -def get_mirror_giturl(submod_name): - """Gets the submodule's url on mirror server. - - Retrurn the download address of the submodule on the mirror server from the submod_name. - """ - - mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' - return mirror_url - - -def modify_submod_file_to_mirror(submod_path): - """Modify the.gitmodules file based on the submodule to be updated""" - - replace_list = [] - try: - with open(submod_path, 'r') as f: - for line in f: - line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') - if line.startswith('url'): - submod_git_url = line.split('=')[1] - submodule_name = submod_git_url.split('/')[-1].replace('.git', '') - replace_url = get_mirror_giturl(submodule_name) - # print(replace_url) - query_submodule_name = 'submod_' + submodule_name - # print(query_submodule_name) - get_package_url, get_ver_sha = get_url_from_mirror_server( - query_submodule_name, 'latest') - - if get_package_url != None and determine_url_valid(get_package_url): - replace_list.append( - (submod_git_url, replace_url, submodule_name)) - - with open(submod_path, 'r+') as f: - submod_file_count = f.read() - - write_content = submod_file_count - - for item in replace_list: - write_content = write_content.replace(item[0], item[1]) - - with open(submod_path, 'w') as f: - f.write(str(write_content)) - - return replace_list - - except Exception as e: - print('Error message:%s\t' % e) - - -def determine_url_valid(url_from_srv): - headers = {'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip, deflate', - 'Accept': '*/*', - 'User-Agent': 'curl/7.54.0'} - - try: - for i in range(0, 3): - r = requests.get(url_from_srv, stream=True, headers=headers) - if r.status_code == requests.codes.not_found: - if i == 2: - print("Warning : %s is invalid." % url_from_srv) - return False - time.sleep(1) - else: - break - - return True - - except Exception as e: - print('Error message:%s\t' % e) - print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) - - -def install_pkg(env_root, pkgs_root, bsp_root, pkg): - """Install the required packages.""" - - # default true - ret = True - local_pkgs_path = os.path.join(env_root, 'local_pkgs') - bsp_pkgs_path = os.path.join(bsp_root, 'packages') - - # get the .config file from env - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - url_from_json = package.get_url(pkg['ver']) - package_url = package.get_url(pkg['ver']) - pkgs_name_in_json = package.get_name() - - if package_url[-4:] == '.git': - ver_sha = package.get_versha(pkg['ver']) - - # print("==================================================>") - # print("packages name :"%pkgs_name_in_json.encode("utf-8")) - # print("ver :"%pkg['ver']) - # print("url :"%package_url.encode("utf-8")) - # print("url_from_json : "%url_from_json.encode("utf-8")) - # print("==================================================>") - - get_package_url = None - get_ver_sha = None - upstream_change_flag = False - - try: - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - - # Check whether the package package url is valid - if get_package_url is not None and determine_url_valid(get_package_url): - package_url = get_package_url - - if get_ver_sha is not None: - ver_sha = get_ver_sha - - upstream_change_flag = True - except Exception as e: - print('Error message:%s\t' % e) - print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - - if package_url[-4:] == '.git': - try: - repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) - repo_path = repo_path + '-' + pkg['ver'] - repo_path_full = '"' + repo_path + '"' - - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(clone_cmd, cwd=bsp_pkgs_path) - - git_check_cmd = 'git checkout -q ' + ver_sha - execute_command(git_check_cmd, cwd=repo_path) - except Exception as e: - print("\nFailed to download software package with git. Please check the network connection.") - return False - - if upstream_change_flag: - cmd = 'git remote set-url origin ' + url_from_json - execute_command(cmd, cwd=repo_path) - - # If there is a .gitmodules file in the package, prepare to update submodule. - submodule_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submodule_path): - print("Start to update submodule") - # print("开始更新软件包子模块") - - if (not os.path.isfile(env_config_file)) \ - or (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - # print("开启了镜像加速，开始修改 .gitmodules 文件") - replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file - - # print("开始执行更新动作") - cmd = 'git submodule update --init --recursive' - execute_command(cmd, cwd=repo_path) - - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and - find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - if len(replace_list): - for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): - cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) - - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - - if os.path.isfile(submodule_path): - cmd = 'git checkout .gitmodules' - execute_command(cmd, cwd=repo_path) - else: - # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path, package_url): - return False - - pkg_dir = package.get_filename(pkg['ver']) - pkg_dir = os.path.splitext(pkg_dir)[0] - package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - - if not archive.packtest(package_path): - print("package : %s is invalid" % package_path.encode("utf-8")) - return False - - # unpack package - if not os.path.exists(pkg_dir): - - try: - if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): - ret = False - except Exception as e: - os.remove(package_path) - ret = False - print('Error message: %s\t' % e) - else: - print("The file does not exist.") - return ret - - -def sub_list(aList, bList): - """Return the items in aList but not in bList.""" - - tmp = [] - for a in aList: - if a not in bList: - tmp.append(a) - return tmp - - -def and_list(aList, bList): - """Return the items in aList and in bList.""" - - tmp = [] - for a in aList: - if a in bList: - tmp.append(a) - return tmp - - -def update_submodule(repo_path): - """Update the submodules in the repository.""" - - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): - print("Please wait a few seconds in order to update the submodule.") - cmd = 'git submodule init -q' - execute_command(cmd, cwd=repo_path) - cmd = 'git submodule update' - execute_command(cmd, cwd=repo_path) - print("Submodule update successful") - - -def get_pkg_folder_by_orign_path(orign_path, version): - # TODO fix for old version project, will remove after new major version - # release - if os.path.exists(orign_path + '-' + version): - return orign_path + '-' + version - return orign_path - - -def git_cmd_exec(cmd, cwd): - try: - execute_command(cmd, cwd=cwd) - except Exception as e: - print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) - print("You can solve this problem by manually removing old packages and re-downloading them using env.") - - -def update_latest_packages(pkgs_fn, bsp_packages_path): - """ update the packages that are latest version. - - If the selected package is the latest version, - check to see if it is the latest version after the update command, - if not, then update the latest version from the remote repository. - If the download has a conflict, you are currently using the prompt - message provided by git. - """ - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - for pkg in read_back_pkgs_json: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - pkgs_name_in_json = package.get_name() - - # Find out the packages which version is 'latest' - if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": - repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) - repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) - - try: - # If mirror acceleration is enabled, get the update address from - # the mirror server. - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) - and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") - - # Change repo's upstream address. - mirror_url = get_url_from_mirror_server( - payload_pkgs_name_in_json, pkg['ver']) - - if mirror_url[0] is not None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) - - except Exception as e: - print("Error message : %s" % e) - print("Failed to connect to the mirror server, using non-mirror server to update.") - - # Update the package repository from upstream. - git_pull_repo(repo_path) - - # If the package has submodules, update the submodules. - update_submodule(repo_path) - - # recover origin url to the path which get from packages.json file - if package.get_url(pkg['ver']): - cmd = 'git remote set-url origin ' + \ - package.get_url(pkg['ver']) - git_cmd_exec(cmd, repo_path) - else: - print("Can't find the package : %s's url in file : %s" % - (payload_pkgs_name_in_json, pkg_path)) - - print("==============================> %s update done\n" % (pkgs_name_in_json)) - - -def pre_package_update(): - """ Make preparations before updating the software package. """ - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - - if not os.path.exists('.config'): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print('No system configuration file : .config.') - print('You should use < menuconfig > command to config bsp first.') +def run_env_cmd(args): + """Run packages command.""" - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return False - - bsp_packages_path = os.path.join(bsp_root, 'packages') - if not os.path.exists(bsp_packages_path): - os.mkdir("packages") - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # prepare target packages file - dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') - Export('dbsqlite_pathname') - dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - - # Avoid creating tables more than one time - if not os.path.isfile(dbsqlite_pathname): - conn = pkgsdb.get_conn(dbsqlite_pathname) - sql = '''CREATE TABLE packagefile - (pathname TEXT ,package TEXT ,md5 TEXT );''' - pkgsdb.create_table(conn, sql) - - fn = '.config' - pkgs = kconfig.parse(fn) - - # print("newpkgs", pkgs) - - newpkgs = pkgs - - if not os.path.exists(bsp_packages_path): - os.mkdir(bsp_packages_path) - - pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') - - # regenerate file : packages/pkgs.json - if not os.path.exists(pkgs_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs.json - with open(pkgs_fn, 'r') as f: - oldpkgs = json.load(f) - - # print("oldpkgs", oldpkgs) - - # regenerate file : packages/pkgs_error.json - pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') - - if not os.path.exists(pkgs_error_list_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs_error.json - with open(pkgs_error_list_fn, 'r') as f: - pkgs_error = json.load(f) - - # create SConscript file - if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: - f.write(str(Bridge_SConscript)) - - return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] - - -def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - flag = None - - error_packages_redownload_error_list = [] - - if len(error_packages_list): - print("\n==============================> Packages list to download : \n") - for pkg in error_packages_list: - print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - print("\nThe packages in the list above are accidentally deleted, env will redownload them.") - print("Warning: Packages should be deleted in command.\n") - - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is redownloaded successfully. \n" % ( - pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - else: - error_packages_redownload_error_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - if len(error_packages_redownload_error_list): - print("%s" % error_packages_redownload_error_list) - print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_pkgs_json = sub_list( - read_back_pkgs_json, error_packages_redownload_error_list) - read_back_pkgs_json = write_back_pkgs_json - # print("write_back_pkgs_json:%s"%write_back_pkgs_json) - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) - pkgs_file.close() - - return flag - - -def rm_package(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rd /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rmdir /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - print("Folder path: %s" % dir.encode("utf-8")) - return False - else: - print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) - return True - - -def get_package_remove_path(pkg, bsp_packages_path): - dirpath = pkg['path'] - ver = pkg['ver'] - if dirpath[0] == '/' or dirpath[0] == '\\': - dirpath = dirpath[1:] - - if platform.system() == "Windows": - dirpath = os.path.basename(dirpath.replace('/', '\\')) - else: - dirpath = os.path.basename(dirpath) - - removepath = os.path.join(bsp_packages_path, dirpath) - - # Handles the deletion of git repository folders with version Numbers - removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) - return removepath_ver - - -def handle_download_error_packages(pkgs_fn, bsp_packages_path): - """ handle download error packages. - - Check to see if the packages stored in the Json file list actually exist, - and then download the packages if they don't exist. - """ - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - error_packages_list = [] - - for pkg in read_back_pkgs_json: - removepath = get_package_remove_path(pkg, bsp_packages_path) - - if os.path.exists(removepath): - continue - else: - error_packages_list.append(pkg) - - # Handle the failed download packages - get_flag = error_packages_handle( - error_packages_list, read_back_pkgs_json, pkgs_fn) - - return get_flag - - -def write_storage_file(pkgs_fn, newpkgs): - """Writes the updated configuration to pkgs.json file. - - Packages that are not downloaded correctly will be redownloaded at the - next update. - """ - - with open(pkgs_fn, 'w') as f: - f.write(str(json.dumps(newpkgs, indent=1))) - - -def package_update(isDeleteOld=False): - """Update env's packages. - - Compare the old and new software package list and update the package. - Remove unwanted packages and download the newly selected package.- - Check if the files in the deleted packages have been changed, and if so, - remind the user saved the modified file. - """ - - sys_value = pre_package_update() - - if not sys_value: - return - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - flag = True - - # According to the env version, whether Chinese output is supported or not - if determine_support_chinese(env_root): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - oldpkgs = sys_value[0] - newpkgs = sys_value[1] - pkgs_delete_error_list = sys_value[2] - pkgs_fn = sys_value[3] - pkgs_error_list_fn = sys_value[4] - bsp_packages_path = sys_value[5] - dbsqlite_pathname = sys_value[6] - - # print(oldpkgs) - # print(newpkgs) - - if len(pkgs_delete_error_list): - for error_package in pkgs_delete_error_list: - removepath_ver = get_package_remove_path( - error_package, bsp_packages_path) - - if os.path.isdir(removepath_ver): - print("\nError: %s package delete failed, begin to remove it." % - error_package['name'].encode("utf-8")) - - if not rm_package(removepath_ver): - print("Error: Delete package %s failed! Please delete the folder manually.\n" % - error_package['name'].encode("utf-8")) - return - - # 1.in old ,not in new : Software packages that need to be removed. - casedelete = sub_list(oldpkgs, newpkgs) - pkgs_delete_fail_list = [] - - for pkg in casedelete: - - removepath_ver = get_package_remove_path(pkg, bsp_packages_path) - removepath_git = os.path.join(removepath_ver, '.git') - - # Delete. Git directory. - if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): - gitdir = removepath_ver - - print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) - if isDeleteOld: - if not rm_package(gitdir): - print("Floder delete fail: %s" % gitdir.encode("utf-8")) - print("Please delete this folder manually.") - else: - print("The folder is managed by git. Do you want to delete this folder?\n") - rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - if rc == 'y' or rc == 'Y': - try: - if not rm_package(gitdir): - pkgs_delete_fail_list.append(pkg) - print("Error: Please delete the folder manually.") - except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", gitdir.encode("utf-8"), e)) - else: - if os.path.isdir(removepath_ver): - print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) - try: - pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) - except Exception as e: - pkgs_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) - - # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) - - if len(pkgs_delete_fail_list): - return - - # 2.in new not in old : Software packages to be installed. - # If the package download fails, record it, and then download again when - # the update command is executed. - - case_download = sub_list(newpkgs, oldpkgs) - # print 'in new not in old:', casedownload - pkgs_download_fail_list = [] - - for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is downloaded successfully. \n" % ( - pkg['name'], pkg['ver'])) - else: - # If the PKG download fails, record it in the - # pkgs_download_fail_list. - pkgs_download_fail_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - # Get the currently updated configuration. - newpkgs = sub_list(newpkgs, pkgs_download_fail_list) - - # Give hints based on the success of the download. - if len(pkgs_download_fail_list): - print("\nPackage download failed list:") - for item in pkgs_download_fail_list: - print(item) - - print("You need to reuse the command to download again.") - - # update pkgs.json and SConscript - write_storage_file(pkgs_fn, newpkgs) - - # handle download error packages. - get_flag = handle_download_error_packages( - pkgs_fn, bsp_packages_path) - - if get_flag is not None: - flag = get_flag - - # Update the software packages, which the version is 'latest' - try: - update_latest_packages(pkgs_fn, bsp_packages_path) - except KeyboardInterrupt: - flag = False - - if flag: - print("Operation completed successfully.") - else: - print("Operation failed.") - - -def cmd(args): - """Env's packages command execution options.""" - - if args.package_update_y: + if args.package_update_force: package_update(True) elif args.package_update: package_update() @@ -761,7 +49,7 @@ def cmd(args): elif args.package_print_env: package_print_env() else: - os.system('pkgs -h') + package_print_help() def add_parser(sub): @@ -773,7 +61,7 @@ def add_parser(sub): help='force update and clean packages, install or remove packages by settings in menuconfig', action='store_true', default=False, - dest='package_update_y') + dest='package_update_force') parser.add_argument('--update', help='update packages, install or remove the packages by your settings in menuconfig', @@ -805,7 +93,4 @@ def add_parser(sub): default=False, dest='package_print_env') - # parser.add_argument('--upgrade', dest='reposource', required=False, - # help='add source & update packages repo ') - - parser.set_defaults(func=cmd) + parser.set_defaults(func=run_env_cmd) diff --git a/cmds/cmd_package_printenv.py b/cmds/cmd_package_printenv.py index f3e1a529..377ead46 100644 --- a/cmds/cmd_package_printenv.py +++ b/cmds/cmd_package_printenv.py @@ -41,3 +41,7 @@ def package_print_env(): env_root = os.path.join(os.getenv('HOME'), '.env') print("ENV_ROOT:%s" % (env_root)) + + +def package_print_help(): + os.system('pkgs -h') diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py new file mode 100644 index 00000000..fb7b3274 --- /dev/null +++ b/cmds/cmd_package_update.py @@ -0,0 +1,737 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import json +import kconfig +import pkgsdb +import shutil +import platform +import time +import archive +import requests +from package import Package, Bridge_SConscript +from vars import Import, Export +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input + + +def determine_support_chinese(env_root): + get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') + if os.path.isfile(get_flag_file_path): + return True + else: + return False + + +def get_mirror_giturl(submod_name): + """Gets the submodule's url on mirror server. + + Retrurn the download address of the submodule on the mirror server from the submod_name. + """ + + mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' + return mirror_url + + +def modify_submod_file_to_mirror(submod_path): + """Modify the.gitmodules file based on the submodule to be updated""" + + replace_list = [] + try: + with open(submod_path, 'r') as f: + for line in f: + line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') + if line.startswith('url'): + submod_git_url = line.split('=')[1] + submodule_name = submod_git_url.split('/')[-1].replace('.git', '') + replace_url = get_mirror_giturl(submodule_name) + # print(replace_url) + query_submodule_name = 'submod_' + submodule_name + # print(query_submodule_name) + get_package_url, get_ver_sha = get_url_from_mirror_server( + query_submodule_name, 'latest') + + if get_package_url != None and determine_url_valid(get_package_url): + replace_list.append( + (submod_git_url, replace_url, submodule_name)) + + with open(submod_path, 'r+') as f: + submod_file_count = f.read() + + write_content = submod_file_count + + for item in replace_list: + write_content = write_content.replace(item[0], item[1]) + + with open(submod_path, 'w') as f: + f.write(str(write_content)) + + return replace_list + + except Exception as e: + print('Error message:%s\t' % e) + + +def determine_url_valid(url_from_srv): + headers = {'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + 'Accept': '*/*', + 'User-Agent': 'curl/7.54.0'} + + try: + for i in range(0, 3): + r = requests.get(url_from_srv, stream=True, headers=headers) + if r.status_code == requests.codes.not_found: + if i == 2: + print("Warning : %s is invalid." % url_from_srv) + return False + time.sleep(1) + else: + break + + return True + + except Exception as e: + print('Error message:%s\t' % e) + print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) + + +def install_pkg(env_root, pkgs_root, bsp_root, pkg): + """Install the required packages.""" + + # default true + ret = True + local_pkgs_path = os.path.join(env_root, 'local_pkgs') + bsp_pkgs_path = os.path.join(bsp_root, 'packages') + + # get the .config file from env + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + url_from_json = package.get_url(pkg['ver']) + package_url = package.get_url(pkg['ver']) + pkgs_name_in_json = package.get_name() + + if package_url[-4:] == '.git': + ver_sha = package.get_versha(pkg['ver']) + + # print("==================================================>") + # print("packages name :"%pkgs_name_in_json.encode("utf-8")) + # print("ver :"%pkg['ver']) + # print("url :"%package_url.encode("utf-8")) + # print("url_from_json : "%url_from_json.encode("utf-8")) + # print("==================================================>") + + get_package_url = None + get_ver_sha = None + upstream_change_flag = False + + try: + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) + + # Check whether the package package url is valid + if get_package_url is not None and determine_url_valid(get_package_url): + package_url = get_package_url + + if get_ver_sha is not None: + ver_sha = get_ver_sha + + upstream_change_flag = True + except Exception as e: + print('Error message:%s\t' % e) + print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + + if package_url[-4:] == '.git': + try: + repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) + repo_path = repo_path + '-' + pkg['ver'] + repo_path_full = '"' + repo_path + '"' + + clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + execute_command(clone_cmd, cwd=bsp_pkgs_path) + + git_check_cmd = 'git checkout -q ' + ver_sha + execute_command(git_check_cmd, cwd=repo_path) + except Exception as e: + print("\nFailed to download software package with git. Please check the network connection.") + return False + + if upstream_change_flag: + cmd = 'git remote set-url origin ' + url_from_json + execute_command(cmd, cwd=repo_path) + + # If there is a .gitmodules file in the package, prepare to update submodule. + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): + print("Start to update submodule") + # print("开始更新软件包子模块") + + if (not os.path.isfile(env_config_file)) \ + or (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + # print("开启了镜像加速，开始修改 .gitmodules 文件") + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file + + # print("开始执行更新动作") + cmd = 'git submodule update --init --recursive' + execute_command(cmd, cwd=repo_path) + + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and + find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if len(replace_list): + for item in replace_list: + submod_dir_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submod_dir_path): + cmd = 'git remote set-url origin ' + item[0] + execute_command(cmd, cwd=submod_dir_path) + + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if os.path.isfile(submodule_path): + cmd = 'git checkout .gitmodules' + execute_command(cmd, cwd=repo_path) + else: + # Download a package of compressed package type. + if not package.download(pkg['ver'], local_pkgs_path, package_url): + return False + + pkg_dir = package.get_filename(pkg['ver']) + pkg_dir = os.path.splitext(pkg_dir)[0] + package_path = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) + + if not archive.packtest(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) + return False + + # unpack package + if not os.path.exists(pkg_dir): + + try: + if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): + ret = False + except Exception as e: + os.remove(package_path) + ret = False + print('Error message: %s\t' % e) + else: + print("The file does not exist.") + return ret + + +def sub_list(aList, bList): + """Return the items in aList but not in bList.""" + + tmp = [] + for a in aList: + if a not in bList: + tmp.append(a) + return tmp + + +def and_list(aList, bList): + """Return the items in aList and in bList.""" + + tmp = [] + for a in aList: + if a in bList: + tmp.append(a) + return tmp + + +def update_submodule(repo_path): + """Update the submodules in the repository.""" + + submod_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submod_path): + print("Please wait a few seconds in order to update the submodule.") + cmd = 'git submodule init -q' + execute_command(cmd, cwd=repo_path) + cmd = 'git submodule update' + execute_command(cmd, cwd=repo_path) + print("Submodule update successful") + + +def get_pkg_folder_by_orign_path(orign_path, version): + # TODO fix for old version project, will remove after new major version + # release + if os.path.exists(orign_path + '-' + version): + return orign_path + '-' + version + return orign_path + + +def git_cmd_exec(cmd, cwd): + try: + execute_command(cmd, cwd=cwd) + except Exception as e: + print('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + print("You can solve this problem by manually removing old packages and re-downloading them using env.") + + +def update_latest_packages(pkgs_fn, bsp_packages_path): + """ update the packages that are latest version. + + If the selected package is the latest version, + check to see if it is the latest version after the update command, + if not, then update the latest version from the remote repository. + If the download has a conflict, you are currently using the prompt + message provided by git. + """ + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + + with open(pkgs_fn, 'r') as f: + read_back_pkgs_json = json.load(f) + + for pkg in read_back_pkgs_json: + package = Package() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + pkgs_name_in_json = package.get_name() + + # Find out the packages which version is 'latest' + if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": + repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) + repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) + + try: + # If mirror acceleration is enabled, get the update address from + # the mirror server. + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) + and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") + + # Change repo's upstream address. + mirror_url = get_url_from_mirror_server( + payload_pkgs_name_in_json, pkg['ver']) + + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + + except Exception as e: + print("Error message : %s" % e) + print("Failed to connect to the mirror server, using non-mirror server to update.") + + # Update the package repository from upstream. + git_pull_repo(repo_path) + + # If the package has submodules, update the submodules. + update_submodule(repo_path) + + # recover origin url to the path which get from packages.json file + if package.get_url(pkg['ver']): + cmd = 'git remote set-url origin ' + \ + package.get_url(pkg['ver']) + git_cmd_exec(cmd, repo_path) + else: + print("Can't find the package : %s's url in file : %s" % + (payload_pkgs_name_in_json, pkg_path)) + + print("==============================> %s update done\n" % (pkgs_name_in_json)) + + +def pre_package_update(): + """ Make preparations before updating the software package. """ + + bsp_root = Import('bsp_root') + env_root = Import('env_root') + + if not os.path.exists('.config'): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + + print('No system configuration file : .config.') + print('You should use < menuconfig > command to config bsp first.') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return False + + bsp_packages_path = os.path.join(bsp_root, 'packages') + if not os.path.exists(bsp_packages_path): + os.mkdir("packages") + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # prepare target packages file + dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') + Export('dbsqlite_pathname') + dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') + + # Avoid creating tables more than one time + if not os.path.isfile(dbsqlite_pathname): + conn = pkgsdb.get_conn(dbsqlite_pathname) + sql = '''CREATE TABLE packagefile + (pathname TEXT ,package TEXT ,md5 TEXT );''' + pkgsdb.create_table(conn, sql) + + fn = '.config' + pkgs = kconfig.parse(fn) + + # print("newpkgs", pkgs) + + newpkgs = pkgs + + if not os.path.exists(bsp_packages_path): + os.mkdir(bsp_packages_path) + + pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') + + # regenerate file : packages/pkgs.json + if not os.path.exists(pkgs_fn): + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # Reading data back from pkgs.json + with open(pkgs_fn, 'r') as f: + oldpkgs = json.load(f) + + # print("oldpkgs", oldpkgs) + + # regenerate file : packages/pkgs_error.json + pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') + + if not os.path.exists(pkgs_error_list_fn): + os.chdir(bsp_packages_path) + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # Reading data back from pkgs_error.json + with open(pkgs_error_list_fn, 'r') as f: + pkgs_error = json.load(f) + + # create SConscript file + if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): + with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: + f.write(str(Bridge_SConscript)) + + return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + + +def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): + bsp_root = Import('bsp_root') + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + flag = None + + error_packages_redownload_error_list = [] + + if len(error_packages_list): + print("\n==============================> Packages list to download : \n") + for pkg in error_packages_list: + print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + print("\nThe packages in the list above are accidentally deleted, env will redownload them.") + print("Warning: Packages should be deleted in command.\n") + + for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + if install_pkg(env_root, pkgs_root, bsp_root, pkg): + print("==============================> %s %s is redownloaded successfully. \n" % ( + pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + else: + error_packages_redownload_error_list.append(pkg) + print(pkg, 'download failed.') + flag = False + + if len(error_packages_redownload_error_list): + print("%s" % error_packages_redownload_error_list) + print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % + (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) + write_back_pkgs_json = sub_list( + read_back_pkgs_json, error_packages_redownload_error_list) + read_back_pkgs_json = write_back_pkgs_json + # print("write_back_pkgs_json:%s"%write_back_pkgs_json) + pkgs_file = file(pkgs_fn, 'w') + pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) + pkgs_file.close() + + return flag + + +def rm_package(dir): + if platform.system() != "Windows": + shutil.rmtree(dir) + else: + dir = '"' + dir + '"' + cmd = 'rd /s /q ' + dir + os.system(cmd) + + if os.path.isdir(dir): + if platform.system() != "Windows": + shutil.rmtree(dir) + else: + dir = '"' + dir + '"' + cmd = 'rmdir /s /q ' + dir + os.system(cmd) + + if os.path.isdir(dir): + print("Folder path: %s" % dir.encode("utf-8")) + return False + else: + print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + return True + + +def get_package_remove_path(pkg, bsp_packages_path): + dirpath = pkg['path'] + ver = pkg['ver'] + if dirpath[0] == '/' or dirpath[0] == '\\': + dirpath = dirpath[1:] + + if platform.system() == "Windows": + dirpath = os.path.basename(dirpath.replace('/', '\\')) + else: + dirpath = os.path.basename(dirpath) + + removepath = os.path.join(bsp_packages_path, dirpath) + + # Handles the deletion of git repository folders with version Numbers + removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) + return removepath_ver + + +def handle_download_error_packages(pkgs_fn, bsp_packages_path): + """ handle download error packages. + + Check to see if the packages stored in the Json file list actually exist, + and then download the packages if they don't exist. + """ + + with open(pkgs_fn, 'r') as f: + read_back_pkgs_json = json.load(f) + + error_packages_list = [] + + for pkg in read_back_pkgs_json: + removepath = get_package_remove_path(pkg, bsp_packages_path) + + if os.path.exists(removepath): + continue + else: + error_packages_list.append(pkg) + + # Handle the failed download packages + get_flag = error_packages_handle( + error_packages_list, read_back_pkgs_json, pkgs_fn) + + return get_flag + + +def write_storage_file(pkgs_fn, newpkgs): + """Writes the updated configuration to pkgs.json file. + + Packages that are not downloaded correctly will be redownloaded at the + next update. + """ + + with open(pkgs_fn, 'w') as f: + f.write(str(json.dumps(newpkgs, indent=1))) + + +def package_update(isDeleteOld=False): + """Update env's packages. + + Compare the old and new software package list and update the package. + Remove unwanted packages and download the newly selected package.- + Check if the files in the deleted packages have been changed, and if so, + remind the user saved the modified file. + """ + + sys_value = pre_package_update() + + if not sys_value: + return + + bsp_root = Import('bsp_root') + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + flag = True + + # According to the env version, whether Chinese output is supported or not + if determine_support_chinese(env_root): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + oldpkgs = sys_value[0] + newpkgs = sys_value[1] + pkgs_delete_error_list = sys_value[2] + pkgs_fn = sys_value[3] + pkgs_error_list_fn = sys_value[4] + bsp_packages_path = sys_value[5] + dbsqlite_pathname = sys_value[6] + + # print(oldpkgs) + # print(newpkgs) + + if len(pkgs_delete_error_list): + for error_package in pkgs_delete_error_list: + removepath_ver = get_package_remove_path( + error_package, bsp_packages_path) + + if os.path.isdir(removepath_ver): + print("\nError: %s package delete failed, begin to remove it." % + error_package['name'].encode("utf-8")) + + if not rm_package(removepath_ver): + print("Error: Delete package %s failed! Please delete the folder manually.\n" % + error_package['name'].encode("utf-8")) + return + + # 1.in old ,not in new : Software packages that need to be removed. + casedelete = sub_list(oldpkgs, newpkgs) + pkgs_delete_fail_list = [] + + for pkg in casedelete: + + removepath_ver = get_package_remove_path(pkg, bsp_packages_path) + removepath_git = os.path.join(removepath_ver, '.git') + + # Delete. Git directory. + if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): + gitdir = removepath_ver + + print("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) + if isDeleteOld: + if not rm_package(gitdir): + print("Floder delete fail: %s" % gitdir.encode("utf-8")) + print("Please delete this folder manually.") + else: + print("The folder is managed by git. Do you want to delete this folder?\n") + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + try: + if not rm_package(gitdir): + pkgs_delete_fail_list.append(pkg) + print("Error: Please delete the folder manually.") + except Exception as e: + print('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", gitdir.encode("utf-8"), e)) + else: + if os.path.isdir(removepath_ver): + print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) + try: + pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) + except Exception as e: + pkgs_delete_fail_list.append(pkg) + print('Error message:\n%s %s. %s \n\t' % ( + "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) + + # write error messages + with open(pkgs_error_list_fn, 'w') as f: + f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + + if len(pkgs_delete_fail_list): + return + + # 2.in new not in old : Software packages to be installed. + # If the package download fails, record it, and then download again when + # the update command is executed. + + case_download = sub_list(newpkgs, oldpkgs) + # print 'in new not in old:', casedownload + pkgs_download_fail_list = [] + + for pkg in case_download: + if install_pkg(env_root, pkgs_root, bsp_root, pkg): + print("==============================> %s %s is downloaded successfully. \n" % ( + pkg['name'], pkg['ver'])) + else: + # If the PKG download fails, record it in the + # pkgs_download_fail_list. + pkgs_download_fail_list.append(pkg) + print(pkg, 'download failed.') + flag = False + + # Get the currently updated configuration. + newpkgs = sub_list(newpkgs, pkgs_download_fail_list) + + # Give hints based on the success of the download. + if len(pkgs_download_fail_list): + print("\nPackage download failed list:") + for item in pkgs_download_fail_list: + print(item) + + print("You need to reuse the command to download again.") + + # update pkgs.json and SConscript + write_storage_file(pkgs_fn, newpkgs) + + # handle download error packages. + get_flag = handle_download_error_packages( + pkgs_fn, bsp_packages_path) + + if get_flag is not None: + flag = get_flag + + # Update the software packages, which the version is 'latest' + try: + update_latest_packages(pkgs_fn, bsp_packages_path) + except KeyboardInterrupt: + flag = False + + if flag: + print("Operation completed successfully.") + else: + print("Operation failed.") From 6449f2f6e9acd1a6cc73598c0e6c69597d18de1d Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 15:52:36 +0800 Subject: [PATCH 16/18] [optimize] cmds/cmd_package_update.py --- cmds/cmd_package_update.py | 283 ++++++++++++++++++------------------- 1 file changed, 141 insertions(+), 142 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index fb7b3274..12787605 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -35,6 +35,7 @@ from package import Package, Bridge_SConscript from vars import Import, Export from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input +from .cmd_menuconfig import find_macro_in_config def determine_support_chinese(env_root): @@ -45,39 +46,37 @@ def determine_support_chinese(env_root): return False -def get_mirror_giturl(submod_name): +def get_mirror_giturl(submodule_name): """Gets the submodule's url on mirror server. Retrurn the download address of the submodule on the mirror server from the submod_name. """ - mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' + mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submodule_name + '.git' return mirror_url -def modify_submod_file_to_mirror(submod_path): +def modify_submod_file_to_mirror(submodule_path): """Modify the.gitmodules file based on the submodule to be updated""" replace_list = [] try: - with open(submod_path, 'r') as f: + with open(submodule_path, 'r') as f: for line in f: line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') if line.startswith('url'): - submod_git_url = line.split('=')[1] - submodule_name = submod_git_url.split('/')[-1].replace('.git', '') + submodule_git_url = line.split('=')[1] + submodule_name = submodule_git_url.split('/')[-1].replace('.git', '') replace_url = get_mirror_giturl(submodule_name) - # print(replace_url) query_submodule_name = 'submod_' + submodule_name - # print(query_submodule_name) get_package_url, get_ver_sha = get_url_from_mirror_server( query_submodule_name, 'latest') - if get_package_url != None and determine_url_valid(get_package_url): + if get_package_url is not None and determine_url_valid(get_package_url): replace_list.append( - (submod_git_url, replace_url, submodule_name)) + (submodule_git_url, replace_url, submodule_name)) - with open(submod_path, 'r+') as f: + with open(submodule_path, 'r+') as f: submod_file_count = f.read() write_content = submod_file_count @@ -85,7 +84,7 @@ def modify_submod_file_to_mirror(submod_path): for item in replace_list: write_content = write_content.replace(item[0], item[1]) - with open(submod_path, 'w') as f: + with open(submodule_path, 'w') as f: f.write(str(write_content)) return replace_list @@ -397,6 +396,12 @@ def pre_package_update(): return False + # According to the env version, whether Chinese output is supported or not + if determine_support_chinese(env_root): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + # create packages folder bsp_packages_path = os.path.join(bsp_root, 'packages') if not os.path.exists(bsp_packages_path): os.mkdir("packages") @@ -415,7 +420,7 @@ def pre_package_update(): Export('dbsqlite_pathname') dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - # Avoid creating tables more than one time + # avoid creating tables more than one time if not os.path.isfile(dbsqlite_pathname): conn = pkgsdb.get_conn(dbsqlite_pathname) sql = '''CREATE TABLE packagefile @@ -424,18 +429,14 @@ def pre_package_update(): fn = '.config' pkgs = kconfig.parse(fn) - - # print("newpkgs", pkgs) - newpkgs = pkgs if not os.path.exists(bsp_packages_path): os.mkdir(bsp_packages_path) - pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') - # regenerate file : packages/pkgs.json - if not os.path.exists(pkgs_fn): + package_json_filename = os.path.join(bsp_packages_path, 'pkgs.json') + if not os.path.exists(package_json_filename): os.chdir(bsp_packages_path) fp = open("pkgs.json", 'w') fp.write("[]") @@ -443,11 +444,9 @@ def pre_package_update(): os.chdir(bsp_root) # Reading data back from pkgs.json - with open(pkgs_fn, 'r') as f: + with open(package_json_filename, 'r') as f: oldpkgs = json.load(f) - # print("oldpkgs", oldpkgs) - # regenerate file : packages/pkgs_error.json pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') @@ -458,7 +457,7 @@ def pre_package_update(): fp.close() os.chdir(bsp_root) - # Reading data back from pkgs_error.json + # read data back from pkgs_error.json with open(pkgs_error_list_fn, 'r') as f: pkgs_error = json.load(f) @@ -467,17 +466,15 @@ def pre_package_update(): with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) - return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] -def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): +def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename): bsp_root = Import('bsp_root') env_root = Import('env_root') pkgs_root = Import('pkgs_root') - - flag = None - error_packages_redownload_error_list = [] + flag = True if len(error_packages_list): print("\n==============================> Packages list to download : \n") @@ -486,7 +483,8 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): print("\nThe packages in the list above are accidentally deleted, env will redownload them.") print("Warning: Packages should be deleted in command.\n") - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list + # re download the packages in error_packages_list + for pkg in error_packages_list: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) @@ -499,57 +497,53 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): print("%s" % error_packages_redownload_error_list) print("Packages:%s,%s redownloed error, you need to use command again to redownload them." % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_pkgs_json = sub_list( - read_back_pkgs_json, error_packages_redownload_error_list) - read_back_pkgs_json = write_back_pkgs_json - # print("write_back_pkgs_json:%s"%write_back_pkgs_json) - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) - pkgs_file.close() + + write_back_package_json = sub_list(read_back_pkgs_json, error_packages_redownload_error_list) + with open(package_filename, 'w') as f: + f.write(json.dumps(write_back_package_json, indent=1)) return flag -def rm_package(dir): +def rm_package(dir_remove): if platform.system() != "Windows": - shutil.rmtree(dir) + shutil.rmtree(dir_remove) else: - dir = '"' + dir + '"' - cmd = 'rd /s /q ' + dir + dir_remove = '"' + dir_remove + '"' + cmd = 'rd /s /q ' + dir_remove os.system(cmd) - if os.path.isdir(dir): + if os.path.isdir(dir_remove): if platform.system() != "Windows": - shutil.rmtree(dir) + shutil.rmtree(dir_remove) else: - dir = '"' + dir + '"' - cmd = 'rmdir /s /q ' + dir + dir_remove = '"' + dir_remove + '"' + cmd = 'rmdir /s /q ' + dir_remove os.system(cmd) - if os.path.isdir(dir): - print("Folder path: %s" % dir.encode("utf-8")) + if os.path.isdir(dir_remove): + print("Folder path: %s" % dir_remove.encode("utf-8")) return False else: - print("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) + print("Path: %s \nSuccess: Folder has been removed. " % dir_remove.encode("utf-8")) return True def get_package_remove_path(pkg, bsp_packages_path): - dirpath = pkg['path'] + dir_path = pkg['path'] ver = pkg['ver'] - if dirpath[0] == '/' or dirpath[0] == '\\': - dirpath = dirpath[1:] + if dir_path[0] == '/' or dir_path[0] == '\\': + dir_path = dir_path[1:] if platform.system() == "Windows": - dirpath = os.path.basename(dirpath.replace('/', '\\')) + dir_path = os.path.basename(dir_path.replace('/', '\\')) else: - dirpath = os.path.basename(dirpath) - - removepath = os.path.join(bsp_packages_path, dirpath) + dir_path = os.path.basename(dir_path) # Handles the deletion of git repository folders with version Numbers - removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) - return removepath_ver + remove_path = os.path.join(bsp_packages_path, dir_path) + remove_path_ver = get_pkg_folder_by_orign_path(remove_path, ver) + return remove_path_ver def handle_download_error_packages(pkgs_fn, bsp_packages_path): @@ -579,57 +573,11 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): return get_flag -def write_storage_file(pkgs_fn, newpkgs): - """Writes the updated configuration to pkgs.json file. - - Packages that are not downloaded correctly will be redownloaded at the - next update. - """ - - with open(pkgs_fn, 'w') as f: - f.write(str(json.dumps(newpkgs, indent=1))) - - -def package_update(isDeleteOld=False): - """Update env's packages. - - Compare the old and new software package list and update the package. - Remove unwanted packages and download the newly selected package.- - Check if the files in the deleted packages have been changed, and if so, - remind the user saved the modified file. - """ - - sys_value = pre_package_update() - - if not sys_value: - return - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - flag = True - - # According to the env version, whether Chinese output is supported or not - if determine_support_chinese(env_root): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - oldpkgs = sys_value[0] - newpkgs = sys_value[1] - pkgs_delete_error_list = sys_value[2] - pkgs_fn = sys_value[3] - pkgs_error_list_fn = sys_value[4] - bsp_packages_path = sys_value[5] - dbsqlite_pathname = sys_value[6] - - # print(oldpkgs) - # print(newpkgs) - - if len(pkgs_delete_error_list): - for error_package in pkgs_delete_error_list: - removepath_ver = get_package_remove_path( - error_package, bsp_packages_path) - +def delete_useless_packages(package_delete_error_list, bsp_packages_path): + # try to delete useless packages, exit command if fails + if len(package_delete_error_list): + for error_package in package_delete_error_list: + removepath_ver = get_package_remove_path(error_package, bsp_packages_path) if os.path.isdir(removepath_ver): print("\nError: %s package delete failed, begin to remove it." % error_package['name'].encode("utf-8")) @@ -637,18 +585,26 @@ def package_update(isDeleteOld=False): if not rm_package(removepath_ver): print("Error: Delete package %s failed! Please delete the folder manually.\n" % error_package['name'].encode("utf-8")) - return + return False + return True + - # 1.in old ,not in new : Software packages that need to be removed. - casedelete = sub_list(oldpkgs, newpkgs) - pkgs_delete_fail_list = [] +def remove_packages(sys_value, isDeleteOld): + old_package = sys_value[0] + new_package = sys_value[1] + package_error_list_filename = sys_value[4] + bsp_packages_path = sys_value[5] + sqlite_pathname = sys_value[6] + + case_delete = sub_list(old_package, new_package) + package_delete_fail_list = [] - for pkg in casedelete: + for pkg in case_delete: removepath_ver = get_package_remove_path(pkg, bsp_packages_path) removepath_git = os.path.join(removepath_ver, '.git') - # Delete. Git directory. + # delete .git directory if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): gitdir = removepath_ver @@ -663,7 +619,7 @@ def package_update(isDeleteOld=False): if rc == 'y' or rc == 'Y': try: if not rm_package(gitdir): - pkgs_delete_fail_list.append(pkg) + package_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") except Exception as e: print('Error message:%s%s. error.message: %s\n\t' % @@ -672,63 +628,106 @@ def package_update(isDeleteOld=False): if os.path.isdir(removepath_ver): print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) try: - pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) + pkgsdb.deletepackdir(removepath_ver, sqlite_pathname) except Exception as e: - pkgs_delete_fail_list.append(pkg) + package_delete_fail_list.append(pkg) print('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) # write error messages - with open(pkgs_error_list_fn, 'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) + with open(package_error_list_filename, 'w') as f: + f.write(str(json.dumps(package_delete_fail_list, indent=1))) - if len(pkgs_delete_fail_list): - return + if len(package_delete_fail_list): + return False - # 2.in new not in old : Software packages to be installed. - # If the package download fails, record it, and then download again when - # the update command is executed. + return True + + +def install_packages(new_package, old_package, package_filename): + """ + If the package download fails, record it, + and then download again when the update command is executed. + """ - case_download = sub_list(newpkgs, oldpkgs) - # print 'in new not in old:', casedownload - pkgs_download_fail_list = [] + bsp_root = Import('bsp_root') + pkgs_root = Import('pkgs_root') + env_root = Import('env_root') + + case_download = sub_list(new_package, old_package) + packages_download_fail_list = [] for pkg in case_download: if install_pkg(env_root, pkgs_root, bsp_root, pkg): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) else: - # If the PKG download fails, record it in the - # pkgs_download_fail_list. - pkgs_download_fail_list.append(pkg) + # if package download fails, record it in the packages_download_fail_list + packages_download_fail_list.append(pkg) print(pkg, 'download failed.') - flag = False + return False # Get the currently updated configuration. - newpkgs = sub_list(newpkgs, pkgs_download_fail_list) + new_package = sub_list(new_package, packages_download_fail_list) # Give hints based on the success of the download. - if len(pkgs_download_fail_list): + if len(packages_download_fail_list): print("\nPackage download failed list:") - for item in pkgs_download_fail_list: + for item in packages_download_fail_list: print(item) print("You need to reuse the command to download again.") # update pkgs.json and SConscript - write_storage_file(pkgs_fn, newpkgs) + with open(package_filename, 'w') as f: + f.write(str(json.dumps(new_package, indent=1))) - # handle download error packages. - get_flag = handle_download_error_packages( - pkgs_fn, bsp_packages_path) + return True - if get_flag is not None: - flag = get_flag - # Update the software packages, which the version is 'latest' +def package_update(isDeleteOld=False): + """Update env's packages. + + Compare the old and new software package list and update the package. + Remove unwanted packages and download the newly selected package.- + Check if the files in the deleted packages have been changed, and if so, + remind the user saved the modified file. + """ + + sys_value = pre_package_update() + if not sys_value: + return + + flag = True + + old_package = sys_value[0] + new_package = sys_value[1] + package_delete_error_list = sys_value[2] + package_filename = sys_value[3] + package_error_list_filename = sys_value[4] + bsp_packages_path = sys_value[5] + sqlite_pathname = sys_value[6] + + if not delete_useless_packages(package_delete_error_list, bsp_packages_path): + return + + # 1. packages in old and not in new : Software packages that need to be removed + if not remove_packages(sys_value, isDeleteOld): + return + + # 2.in new not in old : Software packages to be installed. + if not install_packages(new_package, old_package, package_filename): + flag = False + + # handle download error packages. + if not handle_download_error_packages(package_filename, bsp_packages_path): + flag = False + + # update the software packages, which the version is 'latest' try: - update_latest_packages(pkgs_fn, bsp_packages_path) - except KeyboardInterrupt: + update_latest_packages(package_filename, bsp_packages_path) + except Exception as e: + print('Error message:%s\t' % e) flag = False if flag: From 41b5943079a5957da0815dcac9dca825434ac344 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 17:39:16 +0800 Subject: [PATCH 17/18] [add new feature] The env tool adds a new feature that if the user renames the package folder, such as changing the name of the Lua-v1.0.0 folder to Lua, then env will consider the package to be self-administered by the user. At this point, command will not updated this package again. If user would like to pull other versions of this package, use the command. --- cmds/cmd_package_update.py | 85 +++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 12787605..28fb9827 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -117,13 +117,25 @@ def determine_url_valid(url_from_srv): print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) -def install_pkg(env_root, pkgs_root, bsp_root, pkg): +def is_user_mange_package(bsp_package_path, pkg): + for root, dirs, files in os.walk(bsp_package_path, topdown=True): + for name in dirs: + if name.lower() == pkg["name"].lower(): + return True + break + return False + + +def install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): """Install the required packages.""" - # default true ret = True local_pkgs_path = os.path.join(env_root, 'local_pkgs') - bsp_pkgs_path = os.path.join(bsp_root, 'packages') + bsp_package_path = os.path.join(bsp_root, 'packages') + + if not force_update: + if is_user_mange_package(bsp_package_path, pkg): + return ret # get the .config file from env env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') @@ -140,18 +152,16 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): package_url = package.get_url(pkg['ver']) pkgs_name_in_json = package.get_name() - if package_url[-4:] == '.git': - ver_sha = package.get_versha(pkg['ver']) - # print("==================================================>") - # print("packages name :"%pkgs_name_in_json.encode("utf-8")) - # print("ver :"%pkg['ver']) - # print("url :"%package_url.encode("utf-8")) - # print("url_from_json : "%url_from_json.encode("utf-8")) + # print("packages name :", pkgs_name_in_json.encode("utf-8")) + # print("ver :", pkg['ver']) + # print("url :", package_url.encode("utf-8")) + # print("url_from_json : ", url_from_json.encode("utf-8")) # print("==================================================>") - get_package_url = None - get_ver_sha = None + if package_url[-4:] == '.git': + ver_sha = package.get_versha(pkg['ver']) + upstream_change_flag = False try: @@ -172,14 +182,14 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): print('Error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - if package_url[-4:] == '.git': + if package_url.endswith('.git'): try: - repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) + repo_path = os.path.join(bsp_package_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] repo_path_full = '"' + repo_path + '"' clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(clone_cmd, cwd=bsp_pkgs_path) + execute_command(clone_cmd, cwd=bsp_package_path) git_check_cmd = 'git checkout -q ' + ver_sha execute_command(git_check_cmd, cwd=repo_path) @@ -241,9 +251,8 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): # unpack package if not os.path.exists(pkg_dir): - try: - if not package.unpack(package_path, bsp_pkgs_path, pkg, pkgs_name_in_json): + if not package.unpack(package_path, bsp_package_path, pkg, pkgs_name_in_json): ret = False except Exception as e: os.remove(package_path) @@ -288,11 +297,7 @@ def update_submodule(repo_path): def get_pkg_folder_by_orign_path(orign_path, version): - # TODO fix for old version project, will remove after new major version - # release - if os.path.exists(orign_path + '-' + version): - return orign_path + '-' + version - return orign_path + return orign_path + '-' + version def git_cmd_exec(cmd, cwd): @@ -466,10 +471,11 @@ def pre_package_update(): with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: f.write(str(Bridge_SConscript)) - return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] + return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, + dbsqlite_pathname] -def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename): +def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update): bsp_root = Import('bsp_root') env_root = Import('env_root') pkgs_root = Import('pkgs_root') @@ -480,12 +486,14 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, package_file print("\n==============================> Packages list to download : \n") for pkg in error_packages_list: print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - print("\nThe packages in the list above are accidentally deleted, env will redownload them.") - print("Warning: Packages should be deleted in command.\n") + print("\nThe packages in the list above are accidentally deleted or renamed.") + print("\nIf you manually delete the version suffix of the package folder name, ") + print("you can use the command to re-download these packages.") + print("\nIn case of accidental deletion, the ENV tool will automatically re-download these packages.") # re download the packages in error_packages_list for pkg in error_packages_list: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): + if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("==============================> %s %s is redownloaded successfully. \n" % ( pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: @@ -546,7 +554,7 @@ def get_package_remove_path(pkg, bsp_packages_path): return remove_path_ver -def handle_download_error_packages(pkgs_fn, bsp_packages_path): +def handle_download_error_packages(pkgs_fn, bsp_packages_path, force_update): """ handle download error packages. Check to see if the packages stored in the Json file list actually exist, @@ -559,16 +567,15 @@ def handle_download_error_packages(pkgs_fn, bsp_packages_path): error_packages_list = [] for pkg in read_back_pkgs_json: - removepath = get_package_remove_path(pkg, bsp_packages_path) - - if os.path.exists(removepath): + remove_path = get_package_remove_path(pkg, bsp_packages_path) + if os.path.exists(remove_path): continue else: + print("Error packages add: ", pkg) error_packages_list.append(pkg) # Handle the failed download packages - get_flag = error_packages_handle( - error_packages_list, read_back_pkgs_json, pkgs_fn) + get_flag = error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn, force_update) return get_flag @@ -644,7 +651,7 @@ def remove_packages(sys_value, isDeleteOld): return True -def install_packages(new_package, old_package, package_filename): +def install_packages(new_package, old_package, package_filename, force_update): """ If the package download fails, record it, and then download again when the update command is executed. @@ -658,7 +665,7 @@ def install_packages(new_package, old_package, package_filename): packages_download_fail_list = [] for pkg in case_download: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): + if install_pkg(env_root, pkgs_root, bsp_root, pkg, force_update): print("==============================> %s %s is downloaded successfully. \n" % ( pkg['name'], pkg['ver'])) else: @@ -685,7 +692,7 @@ def install_packages(new_package, old_package, package_filename): return True -def package_update(isDeleteOld=False): +def package_update(force_update=False): """Update env's packages. Compare the old and new software package list and update the package. @@ -712,15 +719,15 @@ def package_update(isDeleteOld=False): return # 1. packages in old and not in new : Software packages that need to be removed - if not remove_packages(sys_value, isDeleteOld): + if not remove_packages(sys_value, force_update): return # 2.in new not in old : Software packages to be installed. - if not install_packages(new_package, old_package, package_filename): + if not install_packages(new_package, old_package, package_filename, force_update): flag = False # handle download error packages. - if not handle_download_error_packages(package_filename, bsp_packages_path): + if not handle_download_error_packages(package_filename, bsp_packages_path, force_update): flag = False # update the software packages, which the version is 'latest' From a1bd958b2b3d7d5380387265af3ddaffff5defd2 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Wed, 8 Apr 2020 18:39:12 +0800 Subject: [PATCH 18/18] [fix] Solve the problem that when the user deletes the .git folder, causing the upstream path to be changed --- cmds/cmd_package_update.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_package_update.py b/cmds/cmd_package_update.py index 28fb9827..a4a3a651 100644 --- a/cmds/cmd_package_update.py +++ b/cmds/cmd_package_update.py @@ -328,6 +328,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): read_back_pkgs_json = json.load(f) for pkg in read_back_pkgs_json: + right_path_flag = True package = Package() pkg_path = pkg['path'] if pkg_path[0] == '/' or pkg_path[0] == '\\': @@ -343,8 +344,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) try: - # If mirror acceleration is enabled, get the update address from - # the mirror server. + # If mirror acceleration is enabled, get the update address from the mirror server. if (not os.path.isfile(env_config_file)) or \ (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): @@ -354,14 +354,26 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) - if mirror_url[0] is not None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) + # if git root is same as repo path, then change the upstream + get_git_root = get_git_root_path(repo_path) + if os.path.normcase(repo_path) == os.path.normcase(get_git_root): + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + else: + print("\n==============================> updating") + print("Package path: %s" % repo_path) + print("Git root: %s" % get_git_root) + print("Error: Not currently in a git root directory, cannot switch upstream.\n") + right_path_flag = False except Exception as e: print("Error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") + if not right_path_flag: + continue + # Update the package repository from upstream. git_pull_repo(repo_path) @@ -380,6 +392,18 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): print("==============================> %s update done\n" % (pkgs_name_in_json)) +def get_git_root_path(repo_path): + before = os.getcwd() + os.chdir(repo_path) + result = os.popen('git rev-parse --show-toplevel') + result = result.read() + for line in result.splitlines()[:5]: + get_git_root = line + break + os.chdir(before) + return get_git_root + + def pre_package_update(): """ Make preparations before updating the software package. """