From 4944d038030795d921fdb07c1115ee33ad1e3613 Mon Sep 17 00:00:00 2001 From: SummerGift <459994202@qq.com> Date: Wed, 30 Oct 2019 18:44:17 +0800 Subject: [PATCH 01/16] [optimize] Adaptting for python3 --- .vscode/settings.json | 20 ++++++++++++++++++++ archive.py | 6 +++--- cmds/cmd_menuconfig.py | 12 ++++++------ cmds/cmd_package.py | 33 ++++++++++++++++----------------- env.py | 2 +- init_env.py | 2 +- kconfig.py | 2 +- package.py | 4 ++-- pkgsdb.py | 6 +++--- 9 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..0ce7a27d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,20 @@ +{ + "MicroPython.executeButton": [ + { + "text": "▶", + "tooltip": "运行", + "alignment": "left", + "command": "extension.executeFile", + "priority": 3.5 + } + ], + "MicroPython.syncButton": [ + { + "text": "$(sync)", + "tooltip": "同步", + "alignment": "left", + "command": "extension.execute", + "priority": 4 + } + ] +} \ No newline at end of file diff --git a/archive.py b/archive.py index ad938bc5..76d16f8a 100644 --- a/archive.py +++ b/archive.py @@ -114,7 +114,7 @@ def packtest(path): else: ret = False print('package check error. \n') - except Exception, e: + except Exception as e: print('packtest e.message:%s\t' % e.message) # arch.close() print("The archive package is broken. \n") @@ -124,14 +124,14 @@ def packtest(path): try: if not tarfile.is_tarfile(path): ret = False - except Exception, e: + except Exception as e: ret = False if ".tar.gz" in path: try: if not tarfile.is_tarfile(path): ret = False - except Exception, e: + except Exception as e: ret = False return ret diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 7e86d677..829b16be 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -103,7 +103,7 @@ def find_macro_in_config(filename, macro_name): try: config = file(filename) except: - print 'open .config failed' + print('open .config failed') return empty_line = 1 @@ -178,7 +178,7 @@ def cmd(args): os.system('chcp 437 > nul') if args.menuconfig_fn: - print 'use', args.menuconfig_fn + print('use', args.menuconfig_fn) import shutil shutil.copy(args.menuconfig_fn, fn) elif args.menuconfig_g: @@ -236,18 +236,18 @@ def cmd(args): if find_macro_in_config(fn, 'SYS_AUTO_UPDATE_PKGS'): os.system('pkgs --update') - print "==============================>The packages have been updated completely." + print("==============================>The packages have been updated completely.") if find_macro_in_config(fn, 'SYS_CREATE_MDK_IAR_PROJECT'): if find_macro_in_config(fn, 'SYS_CREATE_MDK4'): os.system('scons --target=mdk4 -s') - print "Create mdk4 project done" + print("Create mdk4 project done") elif find_macro_in_config(fn, 'SYS_CREATE_MDK5'): os.system('scons --target=mdk5 -s') - print "Create mdk5 project done" + print("Create mdk5 project done") elif find_macro_in_config(fn, 'SYS_CREATE_IAR'): os.system('scons --target=iar -s') - print "Create iar project done" + print("Create iar project done") def add_parser(sub): diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 608f96ac..237b8054 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -36,6 +36,10 @@ import logging import archive import sys +from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file +from vars import Import, Export +from string import Template +from .cmd_menuconfig import find_macro_in_config try: import requests @@ -49,11 +53,6 @@ "* $ sudo apt-get install python-requests\n" "****************************************\n") -from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file -from vars import Import, Export -from string import Template -from cmd_menuconfig import find_macro_in_config - """package command""" def execute_command(cmdstring, cwd=None, shell=True): @@ -138,7 +137,7 @@ def modify_submod_file_to_mirror(submod_path): return replace_list - except Exception, e: + except Exception as e: print('e.message:%s\t' % e.message) @@ -188,7 +187,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None - except Exception, e: + except Exception as e: # print('e.message:%s\t' % e.message) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None @@ -213,7 +212,7 @@ def determine_url_valid(url_from_srv): return True - except Exception, e: + except Exception as e: # print('e.message:%s\t' % e.message) print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -267,7 +266,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): ver_sha = get_ver_sha upstream_change_flag = True - except Exception, e: + except Exception as e: # print('e.message:%s\t' % e.message) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") @@ -282,7 +281,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): cmd = 'git checkout -q ' + ver_sha execute_command(cmd, cwd=repo_path) - except Exception, e: + except Exception as e: print("\nFailed to download software package with git. Please check the network connection.") return False @@ -336,7 +335,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): try: if not package.unpack(pkg_fullpath.encode("gbk"), bsp_pkgs_path, pkg, pkgs_name_in_json.encode("gbk")): ret = False - except Exception, e: + except Exception as e: os.remove(pkg_fullpath) ret = False print('e.message: %s\t' % e.message) @@ -436,7 +435,7 @@ def get_pkg_folder_by_orign_path(orign_path, version): def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) - except Exception, e: + except Exception as e: print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e.message)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -489,7 +488,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): cmd = 'git remote set-url origin ' + mirror_url[0] git_cmd_exec(cmd, repo_path) - except Exception, e: + except Exception as e: print("Failed to connect to the mirror server, using non-mirror server to update.") # Update the package repository from upstream. @@ -627,7 +626,7 @@ def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) else: error_packages_redownload_error_list.append(pkg) - print pkg, 'download failed.' + print(pkg, 'download failed.') flag = False if len(error_packages_redownload_error_list): @@ -799,7 +798,7 @@ def package_update(isDeleteOld=False): if rm_package(gitdir) == False: pkgs_delete_fail_list.append(pkg) print("Error: Please delete the folder manually.") - except Exception, e: + except Exception as e: print('Error message:%s%s. error.message: %s\n\t' % ("Delete folder failed: ", gitdir.encode("utf-8"), e.message)) else: @@ -807,7 +806,7 @@ def package_update(isDeleteOld=False): print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) try: pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) - except Exception, e: + except Exception as e: pkgs_delete_fail_list.append(pkg) print('Error message:\n%s %s. %s \n\t' % ( "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e.message)) @@ -840,7 +839,7 @@ def package_update(isDeleteOld=False): # If the PKG download fails, record it in the # pkgs_download_fail_list. pkgs_download_fail_list.append(pkg) - print pkg, 'download failed.' + print(pkg, 'download failed.') flag = False # Get the currently updated configuration. diff --git a/env.py b/env.py index 02cd8f70..068678a9 100644 --- a/env.py +++ b/env.py @@ -71,7 +71,7 @@ def main(): try: bsp_root.decode("ascii") - except Exception, e: + except Exception as e: if platform.system() == "Windows": os.system('chcp 65001 > nul') diff --git a/init_env.py b/init_env.py index d19dff17..500fb0d9 100644 --- a/init_env.py +++ b/init_env.py @@ -34,7 +34,7 @@ def run_proc(name, env_root): try: os.system("python %s package --upgrade 1>%s 2>%s"%(exec_file, log_std, log_err)) - except Exception, e: + except Exception as e: print("Auto upgrade failed, please check your network.") pass diff --git a/kconfig.py b/kconfig.py index ff803216..cdad8154 100644 --- a/kconfig.py +++ b/kconfig.py @@ -53,7 +53,7 @@ def parse(filename): try: config = file(filename) except: - print 'open .config failed' + print('open .config failed') return ret for line in config: diff --git a/package.py b/package.py index cde59285..1bb3b0d9 100644 --- a/package.py +++ b/package.py @@ -229,7 +229,7 @@ def download(self, ver, path, url_from_srv): os.remove(path.encode("gbk")) ret = False break - except Exception, e: + except Exception as e: #print url_from_srv # print('e.message:%s\t' % e.message) retryCount = retryCount + 1 @@ -245,7 +245,7 @@ def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): # ignore the return value archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) return True - except Exception, e: + except Exception as e: print('unpack e.message:%s\t' % e.message) print('unpack %s failed' % os.path.basename(fullpkg_path)) os.remove(fullpkg_path) diff --git a/pkgsdb.py b/pkgsdb.py index 47a01b3f..cc37aaf4 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -139,9 +139,9 @@ def dbdump(dbpathname): c = get_cursor(conn) cursor = c.execute("SELECT pathname, package, md5 from packagefile") for row in cursor: - print "pathname = ", row[0] - print "package = ", row[1] - print "md5 = ", row[2], "\n" + print("pathname = ", row[0]) + print("package = ", row[1]) + print("md5 = ", row[2], "\n") conn.close() #delete the unchanged file From bd559eb9c9ee84f351797eab7f861d6d6dee3425 Mon Sep 17 00:00:00 2001 From: SummerGift <459994202@qq.com> Date: Thu, 31 Oct 2019 09:25:21 +0800 Subject: [PATCH 02/16] [fix] the decode error when the program running in python3 --- env.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/env.py b/env.py index 068678a9..56739d15 100644 --- a/env.py +++ b/env.py @@ -28,6 +28,7 @@ import sys import argparse import platform +from curses import ascii from cmds import * from vars import Export @@ -48,6 +49,8 @@ def init_argparse(): return parser +def isascii(s): + return all(ascii.isascii(c) for c in s) def main(): bsp_root = os.getcwd() @@ -69,12 +72,10 @@ def main(): Export('bsp_root') Export('pkgs_root') - try: - bsp_root.decode("ascii") - except Exception as e: + if not isascii(bsp_root): if platform.system() == "Windows": os.system('chcp 65001 > nul') - + print ("\n\033[1;31;40m警告:\033[0m") print ("\033[1;31;40m当前路径不支持非英文字符,请修改当前路径为纯英文路径。\033[0m") print ("\033[1;31;40mThe current path does not support non-English characters.\033[0m") From 261fd4f0f4d775efe27f0c0ed9a9413be7929f88 Mon Sep 17 00:00:00 2001 From: SummerGift <459994202@qq.com> Date: Thu, 31 Oct 2019 09:40:59 +0800 Subject: [PATCH 03/16] [optimize] change the way to open a file for python3 --- cmds/cmd_menuconfig.py | 3 ++- cmds/cmd_package.py | 18 +++++++----------- kconfig.py | 3 ++- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 829b16be..791104c3 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -101,7 +101,8 @@ def mk_rtconfig(filename): def find_macro_in_config(filename, macro_name): try: - config = file(filename) + with open(filename, "r", encoding="utf-8") as f: + config = f.read() except: print('open .config failed') return diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 237b8054..37879d5e 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -549,7 +549,7 @@ def pre_package_update(): # prepare target packages file dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') Export('dbsqlite_pathname') - dbsqlite_pathname = dbsqlite_pathname.decode('gbk') + dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') # Avoid creating tables more than one time if not os.path.isfile(dbsqlite_pathname): @@ -596,10 +596,8 @@ def pre_package_update(): # create SConscript file if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - bridge_script = file(os.path.join( - bsp_packages_path, 'SConscript'), 'w') - bridge_script.write(Bridge_SConscript) - bridge_script.close() + with open(os.path.join(bsp_packages_path, 'SConscript'),'w') as f: + f.write(str(Bridge_SConscript)) return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] @@ -720,9 +718,8 @@ def write_storage_file(pkgs_fn, newpkgs): next update. """ - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(newpkgs, indent=1)) - pkgs_file.close() + with open(pkgs_fn,'w') as f: + f.write(str(json.dumps(newpkgs, indent=1))) def package_update(isDeleteOld=False): @@ -819,9 +816,8 @@ def package_update(isDeleteOld=False): return else: # write error messages - pkgs_file = file(pkgs_error_list_fn, 'w') - pkgs_file.write(json.dumps(pkgs_delete_fail_list, indent=1)) - pkgs_file.close() + with open(pkgs_error_list_fn,'w') as f: + f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) # 2.in new not in old : Software packages to be installed. # If the package download fails, record it, and then download again when diff --git a/kconfig.py b/kconfig.py index cdad8154..835a3ec0 100644 --- a/kconfig.py +++ b/kconfig.py @@ -51,7 +51,8 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): ret = [] try: - config = file(filename) + with open(filename, "r", encoding="utf-8") as f: + config = f.read() except: print('open .config failed') return ret From 059edfb532192f997e727cb03a2fbe71b04f2b9e Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 09:51:06 +0800 Subject: [PATCH 04/16] [update] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7bbc71c0..bdbd200a 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,4 @@ ENV/ # mypy .mypy_cache/ +.vscode/settings.json From 0764147d522672198eeffeb119866b2a503f71a9 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 10:34:24 +0800 Subject: [PATCH 05/16] [fix] ascii bsp ascii path judge method --- env.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/env.py b/env.py index 56739d15..18a4efa7 100644 --- a/env.py +++ b/env.py @@ -28,7 +28,6 @@ import sys import argparse import platform -from curses import ascii from cmds import * from vars import Export @@ -49,8 +48,6 @@ def init_argparse(): return parser -def isascii(s): - return all(ascii.isascii(c) for c in s) def main(): bsp_root = os.getcwd() @@ -72,7 +69,9 @@ def main(): Export('bsp_root') Export('pkgs_root') - if not isascii(bsp_root): + try: + bsp_root.encode('utf-8').decode("ascii") + except Exception as e: if platform.system() == "Windows": os.system('chcp 65001 > nul') From 87464848accbb378af82e361889a3b60eeae1c10 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 14:14:55 +0800 Subject: [PATCH 06/16] [optimize] ready for download packaages --- .gitignore | 1 + archive.py | 31 ++++++++++++++++++++++++++----- cmds/cmd_package.py | 20 ++++++++++++++++---- kconfig.py | 22 ++++++++++++++++------ package.py | 29 ++++++++++++++++------------- pkgsdb.py | 4 ++-- 6 files changed, 77 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index bdbd200a..74996b3c 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ ENV/ # mypy .mypy_cache/ .vscode/settings.json +cmds/.config.old diff --git a/archive.py b/archive.py index 76d16f8a..78574fad 100644 --- a/archive.py +++ b/archive.py @@ -103,35 +103,56 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): def packtest(path): ret = True - if ".zip" in path: + print("type(path)", path) + print("type(path)", type(path)) + + # print(path.find(".zip")) + + # path = path.encode().decode() + + if path.find(".zip") != -1: + print("准备判断是否是 zip") try: if zipfile.is_zipfile(path): # Test zip again to make sure it's a right zip file. + + print("这是一个 zip 包,准备判断其完整性") arch = zipfile.ZipFile(path, "r") + print("117") if arch.testzip(): + print("这个 zip 不完整") ret = False arch.close() else: + print("这不是一个 zip 压缩包") + # exit(0) ret = False print('package check error. \n') except Exception as e: - print('packtest e.message:%s\t' % e.message) -# arch.close() + print('packtest e.message:%s\t' % e) + print(129) + arch.close() print("The archive package is broken. \n") ret = False - if ".tar.bz2" in path: + print("ret1", ret) + + # if ".tar.bz2" in path:. + if path.find(".tar.bz2") != -1: + print(138) try: if not tarfile.is_tarfile(path): ret = False except Exception as e: ret = False - if ".tar.gz" in path: + # if ".tar.gz" in path: + if path.find(".tar.gz") != -1: try: if not tarfile.is_tarfile(path): ret = False except Exception as e: ret = False + print("ret", ret) return ret diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 37879d5e..0f62bb73 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -318,22 +318,22 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): else: # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path.decode("gbk"), package_url): + if not package.download(pkg['ver'], local_pkgs_path, package_url): return False pkg_dir = package.get_filename(pkg['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] pkg_fullpath = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - if not archive.packtest(pkg_fullpath.encode("gbk")): + if not archive.packtest(pkg_fullpath): print("package : %s is invalid"%pkg_fullpath.encode("utf-8")) return False # unpack package - if not os.path.exists(pkg_dir.encode("gbk")): + if not os.path.exists(pkg_dir): try: - if not package.unpack(pkg_fullpath.encode("gbk"), bsp_pkgs_path, pkg, pkgs_name_in_json.encode("gbk")): + if not package.unpack(pkg_fullpath, bsp_pkgs_path, pkg, pkgs_name_in_json): ret = False except Exception as e: os.remove(pkg_fullpath) @@ -560,6 +560,9 @@ def pre_package_update(): fn = '.config' pkgs = kconfig.parse(fn) + + print("newpkgs", pkgs) + newpkgs = pkgs if not os.path.exists(bsp_packages_path): @@ -579,6 +582,8 @@ def pre_package_update(): with open(pkgs_fn, 'r') as f: oldpkgs = json.load(f) + print("oldpkgs", oldpkgs) + # regenerate file : packages/pkgs_error.json pkgs_error_list_fn = os.path.join( bsp_packages_path, 'pkgs_error.json') @@ -731,7 +736,11 @@ def package_update(isDeleteOld=False): remind the user saved the modified file. """ + print(734) + sys_value = pre_package_update() + print(737) + print(sys_value) if not sys_value: return @@ -754,6 +763,9 @@ def package_update(isDeleteOld=False): bsp_packages_path = sys_value[5] dbsqlite_pathname = sys_value[6] + print(oldpkgs) + print(newpkgs) + if len(pkgs_delete_error_list): for error_package in pkgs_delete_error_list: removepath_ver = get_package_remove_path( diff --git a/kconfig.py b/kconfig.py index 835a3ec0..7fc1688d 100644 --- a/kconfig.py +++ b/kconfig.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- + # -*- coding:utf-8 -*- # # File : kconfig.py # This file is part of RT-Thread RTOS @@ -25,8 +25,9 @@ def pkgs_path(pkgs, name, path): + print("begin to get pkgs path") for pkg in pkgs: - if pkg.has_key('name') and pkg['name'] == name: + if 'name' in pkg and pkg['name'] == name: pkg['path'] = path return @@ -35,10 +36,13 @@ def pkgs_path(pkgs, name, path): pkg['path'] = path pkgs.append(pkg) + print("-------------------------------------------") + print("pkgs_path", pkgs) + def pkgs_ver(pkgs, name, ver): for pkg in pkgs: - if pkg.has_key('name') and pkg['name'] == name: + if 'name' in pkg and pkg['name'] == name: pkg['ver'] = ver return @@ -49,15 +53,16 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): + print("filename", filename) ret = [] try: - with open(filename, "r", encoding="utf-8") as f: - config = f.read() + config = open(filename, "r") except: print('open .config failed') return ret for line in config: + print(line) line = line.lstrip(' ').replace('\n', '').replace('\r', '') if len(line) == 0: @@ -66,14 +71,17 @@ def parse(filename): if line[0] == '#': continue else: + # print(line) setting = line.split('=', 1) + # print(setting) if len(setting) >= 2: if setting[0].startswith('CONFIG_PKG_'): pkg_prefix = setting[0][11:] + print("74") if pkg_prefix.startswith('USING_'): pkg_name = pkg_prefix[6:] - # print 'enable package:', pkg_name + print('enable package:', pkg_name) else: if pkg_prefix.endswith('_PATH'): pkg_name = pkg_prefix[:-5] @@ -93,6 +101,8 @@ def parse(filename): pkg_ver = pkg_ver[:-1] pkgs_ver(ret, pkg_name, pkg_ver) + print("ret", ret) + config.close() return ret diff --git a/package.py b/package.py index 1bb3b0d9..addf93c4 100644 --- a/package.py +++ b/package.py @@ -130,8 +130,8 @@ class Package: pkg = None def parse(self, filename): - f = file(filename) - json_str = f.read() + with open(filename, "r", encoding="utf-8") as f: + json_str = f.read() if json_str: self.pkg = json.loads(json_str) @@ -171,7 +171,7 @@ def download(self, ver, path, url_from_srv): ret = True url = self.get_url(ver) site = self.get_site(ver) - if site and site.has_key('filename'): + if site and 'filename' in site: filename = site['filename'] path = os.path.join(path, filename) else: @@ -204,7 +204,7 @@ def download(self, ver, path, url_from_srv): flush_count = 0 - with open(path.encode("gbk"), 'wb') as f: + with open(path, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: f.write(chunk) @@ -214,27 +214,30 @@ def download(self, ver, path, url_from_srv): sys.stdout.flush() retryCount = retryCount + 1 - if archive.packtest(path.encode("gbk")): # make sure the file is right + # exit(0) + + if archive.packtest(path): # make sure the file is right ret = True print("\rDownloded %d KB " % flush_count) print('Start to unpack. Please wait...') break else: - if os.path.isfile(path.encode("gbk")): - os.remove(path.encode("gbk")) + print("文件校验失败了") + if os.path.isfile(path): + os.remove(path) if retryCount > 5: print( "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) - if os.path.isfile(path.encode("gbk")): - os.remove(path.encode("gbk")) + if os.path.isfile(path): + os.remove(path) ret = False break except Exception as e: - #print url_from_srv - # print('e.message:%s\t' % e.message) + print(url_from_srv) + print('e.message:%s\t' %e) retryCount = retryCount + 1 if retryCount > 5: - print('%s download fail!\n' % path.decode("gbk").encode("utf-8")) + print('%s download fail!\n' % path.encode("utf-8")) if os.path.isfile(path): os.remove(path) return False @@ -246,7 +249,7 @@ def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) return True except Exception as e: - print('unpack e.message:%s\t' % e.message) + print('unpack e.message:%s\t' % e) print('unpack %s failed' % os.path.basename(fullpkg_path)) os.remove(fullpkg_path) return False diff --git a/pkgsdb.py b/pkgsdb.py index cc37aaf4..121816f7 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -35,7 +35,7 @@ def GetFileMd5(filename): if not os.path.isfile(filename): return myhash = hashlib.md5() - f = file(filename, 'rb') + f = open(filename, 'rb') while True: b = f.read(8096) if not b: @@ -118,7 +118,7 @@ def savetodb(pathname, pkgspathname, before_change_name): bsp_root = Import('bsp_root') bsppkgs = os.path.join(bsp_root, 'packages') - conn = get_conn(dbpathname.decode("gbk")) + conn = get_conn(dbpathname) save_sql = '''insert into packagefile values (?, ?, ?)''' package = os.path.basename(pkgspathname) md5pathname = os.path.join(bsppkgs, before_change_name) From c804389ed619ea7daea8596a19d55abe838f6f55 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 14:17:14 +0800 Subject: [PATCH 07/16] [add] config file --- cmds/.config | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 cmds/.config diff --git a/cmds/.config b/cmds/.config new file mode 100644 index 00000000..29a03471 --- /dev/null +++ b/cmds/.config @@ -0,0 +1,11 @@ +# +# Automatically generated file; DO NOT EDIT. +# RootMenu +# + +# +# Env config +# +# CONFIG_SYS_AUTO_UPDATE_PKGS is not set +# CONFIG_SYS_CREATE_MDK_IAR_PROJECT is not set +CONFIG_SYS_PKGS_DOWNLOAD_ACCELERATE=y From 23703816f70604577517fa31baf4e1ec6b966563 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 16:13:42 +0800 Subject: [PATCH 08/16] [optimize] ready for download packages --- cmds/cmd_menuconfig.py | 4 +-- cmds/cmd_package.py | 82 +++++++++++++++++++++++++++++++----------- kconfig.py | 6 ---- 3 files changed, 63 insertions(+), 29 deletions(-) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index 791104c3..f8335490 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -101,8 +101,7 @@ def mk_rtconfig(filename): def find_macro_in_config(filename, macro_name): try: - with open(filename, "r", encoding="utf-8") as f: - config = f.read() + config = open(filename, "r", encoding="utf-8") except: print('open .config failed') return @@ -142,6 +141,7 @@ def find_macro_in_config(filename, macro_name): if setting[0] == macro_name and setting[1] == 'y': return True + config.close() return False diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 0f62bb73..916ffa93 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -66,7 +66,7 @@ def execute_command(cmdstring, cwd=None, shell=True): stdout_str = '' while sub.poll() is None: - stdout_str += sub.stdout.read() + stdout_str += str(sub.stdout.read()) time.sleep(0.1) return stdout_str @@ -144,7 +144,22 @@ def modify_submod_file_to_mirror(submod_path): def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): """Get the download address from the mirror server based on the package name.""" - payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") + print(pkgs_name_in_json, pkgs_ver) + print("准备对比过数据类型了") + + try: + if type(pkgs_name_in_json) != type("str"): + pkgs_name_in_json = str(pkgs_name_in_json) + except Exception as e: + print('e.message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None + + print("已经对比过数据类型了") + + print("type(pkgs_name_in_json)", type(pkgs_name_in_json)) + + payload_pkgs_name_in_json = pkgs_name_in_json payload = { "userName": "RT-Thread", "packages": [ @@ -155,16 +170,18 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } payload["packages"][0]['name'] = payload_pkgs_name_in_json + print(payload) try: - r = requests.post( - "http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) + print("准备连接查询服务器") + r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) - # print(r.status_code) + print(r.status_code) + print("连接查询服务器失败") if r.status_code == requests.codes.ok: package_info = json.loads(r.text) - # print(package_info) + print(package_info) # Can't find package,change git package SHA if it's a git # package @@ -188,7 +205,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None except Exception as e: - # print('e.message:%s\t' % e.message) + print('e.message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None @@ -254,10 +271,20 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): get_ver_sha = None upstream_change_flag = False + print("准备下载软件包") + print(env_config_file) + + if find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE'): + print("找到了加速宏定义") + try: if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) + print("将使用镜像下载") + + print("get_package_url", get_package_url) + # determine whether the package package url is valid if get_package_url != None and determine_url_valid(get_package_url): package_url = get_package_url @@ -267,22 +294,34 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = True except Exception as e: - # print('e.message:%s\t' % e.message) + print('e.message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + print(package_url) + if package_url[-4:] == '.git': try: repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] repo_path_full = '"' + repo_path + '"' + before = os.getcwd() cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(cmd, cwd=bsp_pkgs_path) + # execute_command(cmd, cwd=bsp_pkgs_path) - cmd = 'git checkout -q ' + ver_sha - execute_command(cmd, cwd=repo_path) + os.chdir(bsp_pkgs_path) + os.system(cmd) + + print("clone done") + + git_check_cmd = 'git checkout -q ' + ver_sha + os.system(cmd) + + os.chdir(before) + # execute_command(cmd, cwd=repo_path) except Exception as e: print("\nFailed to download software package with git. Please check the network connection.") + os.chdir(before) return False if upstream_change_flag: @@ -480,14 +519,20 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") + print("准备获取上游地址") + print(payload_pkgs_name_in_json) # Change repo's upstream address. mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) + print("准备更新版本为 latest 的仓库") + if mirror_url[0] != None: cmd = 'git remote set-url origin ' + mirror_url[0] git_cmd_exec(cmd, repo_path) + print("更换远端地址完毕") + except Exception as e: print("Failed to connect to the mirror server, using non-mirror server to update.") @@ -505,10 +550,9 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): git_cmd_exec(cmd, repo_path) else: print("Can't find the package : %s's url in file : %s" % - (payload_pkgs_name_in_json.encode("utf-8"), pkg_path.encode("utf-8"))) + (payload_pkgs_name_in_json, pkg_path)) - print("==============================> %s update done \n" % - (pkgs_name_in_json.encode("utf-8"))) + print("==============================> %s update done\n" %(pkgs_name_in_json)) def pre_package_update(): @@ -736,11 +780,7 @@ def package_update(isDeleteOld=False): remind the user saved the modified file. """ - print(734) - sys_value = pre_package_update() - print(737) - print(sys_value) if not sys_value: return @@ -763,8 +803,8 @@ def package_update(isDeleteOld=False): bsp_packages_path = sys_value[5] dbsqlite_pathname = sys_value[6] - print(oldpkgs) - print(newpkgs) + # print(oldpkgs) + # print(newpkgs) if len(pkgs_delete_error_list): for error_package in pkgs_delete_error_list: @@ -800,7 +840,7 @@ def package_update(isDeleteOld=False): else: print ( "The folder is managed by git. Do you want to delete this folder?\n") - rc = raw_input( + rc = input( 'Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': try: diff --git a/kconfig.py b/kconfig.py index 7fc1688d..6fc37743 100644 --- a/kconfig.py +++ b/kconfig.py @@ -62,7 +62,6 @@ def parse(filename): return ret for line in config: - print(line) line = line.lstrip(' ').replace('\n', '').replace('\r', '') if len(line) == 0: @@ -71,14 +70,10 @@ def parse(filename): if line[0] == '#': continue else: - # print(line) setting = line.split('=', 1) - # print(setting) if len(setting) >= 2: if setting[0].startswith('CONFIG_PKG_'): pkg_prefix = setting[0][11:] - print("74") - if pkg_prefix.startswith('USING_'): pkg_name = pkg_prefix[6:] print('enable package:', pkg_name) @@ -101,7 +96,6 @@ def parse(filename): pkg_ver = pkg_ver[:-1] pkgs_ver(ret, pkg_name, pkg_ver) - print("ret", ret) config.close() return ret From 820d8e690d9cefb5952bf7913d42c3e2178063ea Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 16:23:10 +0800 Subject: [PATCH 09/16] [optimize] ready for upgrade latest version packages --- cmds/cmd_package.py | 34 +++++++++++++++++----------------- pkgsdb.py | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 916ffa93..04611eb0 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -85,7 +85,7 @@ def user_input(msg, default_value): msg = '%s[%s]' % (msg, default_value) print(msg) - value = raw_input() + value = input() if value == '': value = default_value @@ -149,7 +149,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): try: if type(pkgs_name_in_json) != type("str"): - pkgs_name_in_json = str(pkgs_name_in_json) + pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] except Exception as e: print('e.message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") @@ -158,8 +158,8 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): print("已经对比过数据类型了") print("type(pkgs_name_in_json)", type(pkgs_name_in_json)) + print(pkgs_name_in_json) - payload_pkgs_name_in_json = pkgs_name_in_json payload = { "userName": "RT-Thread", "packages": [ @@ -168,9 +168,9 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } ] } - payload["packages"][0]['name'] = payload_pkgs_name_in_json + payload["packages"][0]['name'] = pkgs_name_in_json - print(payload) + # print(payload) try: print("准备连接查询服务器") r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) @@ -937,10 +937,10 @@ def package_wizard(): #first step print ('\033[5;33;40m\n1.Please input a new package name :\033[0m') - name = raw_input() + name = input() while name == '' or name.isspace() == True : print ('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = raw_input() + name = input() default_description = 'Please add description of ' + name + ' in English.' #description = user_input('menuconfig option name,default:\n',default_description) @@ -958,37 +958,37 @@ def package_wizard(): 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') print ('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') print ("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = raw_input() + classnu = input() while classnu == '' or classnu.isdigit()== False or int(classnu) < 1 or int(classnu) >9: if classnu == '' : print ('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') else : print ('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = raw_input() + classnu = input() pkgsclass = packageclass[int(classnu) - 1] #fourth step print ('\033[5;33;40m\n4.Please input author name of this package :\033[0m') - authorname = raw_input() + authorname = input() while authorname == '': print ('\033[1;31;40mError: you must input author name of this package. Try again.\033[0m') - authorname = raw_input() + authorname = input() #fifth step - authoremail = raw_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + authoremail = input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') while authoremail == '': print ('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = raw_input() + authoremail = input() #sixth step print ('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') print ("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = raw_input() + license_class = input() while license_class == '' : print ('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = raw_input() + license_class = input() if license_class.isdigit()== True and int(license_class) >= 1 and int(license_class) <= 4: license = license_index[int(license_class) - 1] @@ -999,10 +999,10 @@ def package_wizard(): print ('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') print ("\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - repository = raw_input() + repository = input() while repository == '': print ('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = raw_input() + repository = input() pkg_path = name if not os.path.exists(pkg_path): diff --git a/pkgsdb.py b/pkgsdb.py index 121816f7..15f66171 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -178,7 +178,7 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): print ('Are you sure you want to permanently delete the file: %s?' % os.path.basename(pathname)) print ('If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') - rc = raw_input( + rc = input( 'Press the Y Key to delete the file or just press Enter to keep the file.') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" From 7b14e882528f46a76f82fbca1d8ee781fdfee426 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 16:29:17 +0800 Subject: [PATCH 10/16] [optimize] code clear --- archive.py | 2 +- cmds/cmd_package.py | 28 +++++----------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/archive.py b/archive.py index 78574fad..e89d4d54 100644 --- a/archive.py +++ b/archive.py @@ -129,7 +129,7 @@ def packtest(path): ret = False print('package check error. \n') except Exception as e: - print('packtest e.message:%s\t' % e) + print('packtest error message:%s\t' % e) print(129) arch.close() print("The archive package is broken. \n") diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index 04611eb0..c0c41fa8 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -144,22 +144,14 @@ def modify_submod_file_to_mirror(submod_path): def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): """Get the download address from the mirror server based on the package name.""" - print(pkgs_name_in_json, pkgs_ver) - print("准备对比过数据类型了") - try: if type(pkgs_name_in_json) != type("str"): pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] except Exception as e: - print('e.message:%s' % e) + print('error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None - print("已经对比过数据类型了") - - print("type(pkgs_name_in_json)", type(pkgs_name_in_json)) - print(pkgs_name_in_json) - payload = { "userName": "RT-Thread", "packages": [ @@ -170,14 +162,9 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } payload["packages"][0]['name'] = pkgs_name_in_json - # print(payload) try: - print("准备连接查询服务器") r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) - print(r.status_code) - print("连接查询服务器失败") - if r.status_code == requests.codes.ok: package_info = json.loads(r.text) @@ -205,7 +192,7 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): return None, None except Exception as e: - print('e.message:%s' % e) + print('error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") return None, None @@ -230,7 +217,7 @@ def determine_url_valid(url_from_srv): return True except Exception as e: - # print('e.message:%s\t' % e.message) + print('error message:%s\t' % e) print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) @@ -294,7 +281,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): upstream_change_flag = True except Exception as e: - print('e.message:%s\t' % e) + print('error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") print(package_url) @@ -377,7 +364,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): except Exception as e: os.remove(pkg_fullpath) ret = False - print('e.message: %s\t' % e.message) + print('error message: %s\t' % e.message) else: print("The file does not exist.") return ret @@ -519,20 +506,15 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") - print("准备获取上游地址") print(payload_pkgs_name_in_json) # Change repo's upstream address. mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) - print("准备更新版本为 latest 的仓库") - if mirror_url[0] != None: cmd = 'git remote set-url origin ' + mirror_url[0] git_cmd_exec(cmd, repo_path) - print("更换远端地址完毕") - except Exception as e: print("Failed to connect to the mirror server, using non-mirror server to update.") From bfd9c91478880b3d23224fb67deeeadf8d2f61cf Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 16:55:54 +0800 Subject: [PATCH 11/16] [optimize] make code clear up --- archive.py | 20 +------------------- cmds/cmd_package.py | 46 +++++++++++---------------------------------- kconfig.py | 6 ------ package.py | 6 ++---- 4 files changed, 14 insertions(+), 64 deletions(-) diff --git a/archive.py b/archive.py index e89d4d54..24117f41 100644 --- a/archive.py +++ b/archive.py @@ -103,43 +103,26 @@ def unpack(archive_fn, path, pkg, pkgs_name_in_json): def packtest(path): ret = True - print("type(path)", path) - print("type(path)", type(path)) - - # print(path.find(".zip")) - - # path = path.encode().decode() if path.find(".zip") != -1: - print("准备判断是否是 zip") try: if zipfile.is_zipfile(path): # Test zip again to make sure it's a right zip file. - - print("这是一个 zip 包,准备判断其完整性") arch = zipfile.ZipFile(path, "r") - print("117") if arch.testzip(): - print("这个 zip 不完整") ret = False arch.close() else: - print("这不是一个 zip 压缩包") - # exit(0) ret = False print('package check error. \n') except Exception as e: print('packtest error message:%s\t' % e) - print(129) - arch.close() print("The archive package is broken. \n") + arch.close() ret = False - print("ret1", ret) - # if ".tar.bz2" in path:. if path.find(".tar.bz2") != -1: - print(138) try: if not tarfile.is_tarfile(path): ret = False @@ -154,5 +137,4 @@ def packtest(path): except Exception as e: ret = False - print("ret", ret) return ret diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index c0c41fa8..b2b22ae3 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -138,7 +138,7 @@ def modify_submod_file_to_mirror(submod_path): return replace_list except Exception as e: - print('e.message:%s\t' % e.message) + print('error message:%s\t' % e) def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): @@ -168,8 +168,6 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): if r.status_code == requests.codes.ok: package_info = json.loads(r.text) - print(package_info) - # Can't find package,change git package SHA if it's a git # package if len(package_info['packages']) == 0: @@ -188,7 +186,6 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): print("\nTips : \nThe system needs to be upgraded.") print("Please use the command to upgrade packages index.\n") - return None, None except Exception as e: @@ -258,20 +255,10 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): get_ver_sha = None upstream_change_flag = False - print("准备下载软件包") - print(env_config_file) - - if find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE'): - print("找到了加速宏定义") - try: if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - print("将使用镜像下载") - - print("get_package_url", get_package_url) - # determine whether the package package url is valid if get_package_url != None and determine_url_valid(get_package_url): package_url = get_package_url @@ -284,28 +271,18 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): print('error message:%s\t' % e) print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - print(package_url) - if package_url[-4:] == '.git': try: repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) repo_path = repo_path + '-' + pkg['ver'] repo_path_full = '"' + repo_path + '"' - before = os.getcwd() - cmd = 'git clone ' + package_url + ' ' + repo_path_full - # execute_command(cmd, cwd=bsp_pkgs_path) - - os.chdir(bsp_pkgs_path) - os.system(cmd) - - print("clone done") + clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full + execute_command(clone_cmd, cwd=bsp_pkgs_path) git_check_cmd = 'git checkout -q ' + ver_sha - os.system(cmd) + execute_command(git_check_cmd, cwd=repo_path) - os.chdir(before) - # execute_command(cmd, cwd=repo_path) except Exception as e: print("\nFailed to download software package with git. Please check the network connection.") os.chdir(before) @@ -364,7 +341,7 @@ def install_pkg(env_root, pkgs_root, bsp_root, pkg): except Exception as e: os.remove(pkg_fullpath) ret = False - print('error message: %s\t' % e.message) + print('error message: %s\t' % e) else: print("The file does not exist.") return ret @@ -462,7 +439,7 @@ def git_cmd_exec(cmd, cwd): try: execute_command(cmd, cwd=cwd) except Exception as e: - print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e.message)) + print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e)) print("You can solve this problem by manually removing old packages and re-downloading them using env.") @@ -506,7 +483,6 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") - print(payload_pkgs_name_in_json) # Change repo's upstream address. mirror_url = get_url_from_mirror_server( payload_pkgs_name_in_json, pkg['ver']) @@ -516,6 +492,7 @@ def update_latest_packages(pkgs_fn, bsp_packages_path): git_cmd_exec(cmd, repo_path) except Exception as e: + print("error message : %s" % e) print("Failed to connect to the mirror server, using non-mirror server to update.") # Update the package repository from upstream. @@ -569,7 +546,6 @@ def pre_package_update(): fp = open("pkgs_error.json", 'w') fp.write("[]") fp.close() - os.chdir(bsp_root) # prepare target packages file @@ -587,7 +563,7 @@ def pre_package_update(): fn = '.config' pkgs = kconfig.parse(fn) - print("newpkgs", pkgs) + # print("newpkgs", pkgs) newpkgs = pkgs @@ -608,7 +584,7 @@ def pre_package_update(): with open(pkgs_fn, 'r') as f: oldpkgs = json.load(f) - print("oldpkgs", oldpkgs) + # print("oldpkgs", oldpkgs) # regenerate file : packages/pkgs_error.json pkgs_error_list_fn = os.path.join( @@ -831,7 +807,7 @@ def package_update(isDeleteOld=False): print("Error: Please delete the folder manually.") except Exception as e: print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", gitdir.encode("utf-8"), e.message)) + ("Delete folder failed: ", gitdir.encode("utf-8"), e)) else: if os.path.isdir(removepath_ver): print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) @@ -840,7 +816,7 @@ def package_update(isDeleteOld=False): except Exception as e: pkgs_delete_fail_list.append(pkg) print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e.message)) + "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) if len(pkgs_delete_fail_list): # write error messages diff --git a/kconfig.py b/kconfig.py index 6fc37743..0e904203 100644 --- a/kconfig.py +++ b/kconfig.py @@ -25,7 +25,6 @@ def pkgs_path(pkgs, name, path): - print("begin to get pkgs path") for pkg in pkgs: if 'name' in pkg and pkg['name'] == name: pkg['path'] = path @@ -36,9 +35,6 @@ def pkgs_path(pkgs, name, path): pkg['path'] = path pkgs.append(pkg) - print("-------------------------------------------") - print("pkgs_path", pkgs) - def pkgs_ver(pkgs, name, ver): for pkg in pkgs: @@ -53,7 +49,6 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): - print("filename", filename) ret = [] try: config = open(filename, "r") @@ -76,7 +71,6 @@ def parse(filename): pkg_prefix = setting[0][11:] if pkg_prefix.startswith('USING_'): pkg_name = pkg_prefix[6:] - print('enable package:', pkg_name) else: if pkg_prefix.endswith('_PATH'): pkg_name = pkg_prefix[:-5] diff --git a/package.py b/package.py index addf93c4..e2e7283d 100644 --- a/package.py +++ b/package.py @@ -214,7 +214,6 @@ def download(self, ver, path, url_from_srv): sys.stdout.flush() retryCount = retryCount + 1 - # exit(0) if archive.packtest(path): # make sure the file is right ret = True @@ -222,7 +221,6 @@ def download(self, ver, path, url_from_srv): print('Start to unpack. Please wait...') break else: - print("文件校验失败了") if os.path.isfile(path): os.remove(path) if retryCount > 5: @@ -234,7 +232,7 @@ def download(self, ver, path, url_from_srv): break except Exception as e: print(url_from_srv) - print('e.message:%s\t' %e) + print('error message:%s\t' %e) retryCount = retryCount + 1 if retryCount > 5: print('%s download fail!\n' % path.encode("utf-8")) @@ -249,7 +247,7 @@ def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) return True except Exception as e: - print('unpack e.message:%s\t' % e) + print('unpack error message :%s' % e) print('unpack %s failed' % os.path.basename(fullpkg_path)) os.remove(fullpkg_path) return False From 46a524c0a511347df41b059a952ae8a788da18f0 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 16:57:44 +0800 Subject: [PATCH 12/16] [del] settings.json --- .vscode/settings.json | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 0ce7a27d..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "MicroPython.executeButton": [ - { - "text": "▶", - "tooltip": "运行", - "alignment": "left", - "command": "extension.executeFile", - "priority": 3.5 - } - ], - "MicroPython.syncButton": [ - { - "text": "$(sync)", - "tooltip": "同步", - "alignment": "left", - "command": "extension.execute", - "priority": 4 - } - ] -} \ No newline at end of file From 803478be2fecc01278adba6fcf94bcfd0ed02a56 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 18:09:46 +0800 Subject: [PATCH 13/16] [optimize] change input func for python2 --- cmds/cmd_menuconfig.py | 2 +- cmds/cmd_package.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index f8335490..a8a75d60 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -101,7 +101,7 @@ def mk_rtconfig(filename): def find_macro_in_config(filename, macro_name): try: - config = open(filename, "r", encoding="utf-8") + config = open(filename, "r") except: print('open .config failed') return diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index b2b22ae3..f53f66d8 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -85,7 +85,11 @@ def user_input(msg, default_value): msg = '%s[%s]' % (msg, default_value) print(msg) - value = input() + if sys.version_info < (3, 0): + value = raw_input() + else: + value = input() + if value == '': value = default_value @@ -796,10 +800,12 @@ def package_update(isDeleteOld=False): print("Floder delete fail: %s" % gitdir.encode("utf-8")) print("Please delete this folder manually.") else: - print ( - "The folder is managed by git. Do you want to delete this folder?\n") - rc = input( - 'Press the Y Key to delete the folder or just press Enter to keep it : ') + print ("The folder is managed by git. Do you want to delete this folder?\n") + if sys.version_info < (3, 0): + rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + else: + rc = input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': try: if rm_package(gitdir) == False: From 854e09e89b7e95dc9a246ea51172431200b6f7c4 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 31 Oct 2019 18:38:09 +0800 Subject: [PATCH 14/16] [optimize] the input api Compatibility problem --- cmds/cmd_package.py | 40 ++++++++++++++++++++++++++++------------ package.py | 2 +- pkgsdb.py | 8 ++++++-- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index f53f66d8..ff364517 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -95,6 +95,22 @@ def user_input(msg, default_value): return value +def union_input(msg = None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg != None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg != None: + value = input(msg) + else: + value = input() + + return value + def get_mirror_giturl(submod_name): """Gets the submodule's url on mirror server. @@ -901,10 +917,10 @@ def package_wizard(): #first step print ('\033[5;33;40m\n1.Please input a new package name :\033[0m') - name = input() + name = union_input() while name == '' or name.isspace() == True : print ('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = input() + name = union_input() default_description = 'Please add description of ' + name + ' in English.' #description = user_input('menuconfig option name,default:\n',default_description) @@ -922,37 +938,37 @@ def package_wizard(): 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') print ('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') print ("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = input() + classnu = union_input() while classnu == '' or classnu.isdigit()== False or int(classnu) < 1 or int(classnu) >9: if classnu == '' : print ('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') else : print ('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = input() + classnu = union_input() pkgsclass = packageclass[int(classnu) - 1] #fourth step print ('\033[5;33;40m\n4.Please input author name of this package :\033[0m') - authorname = input() + authorname = union_input() while authorname == '': print ('\033[1;31;40mError: you must input author name of this package. Try again.\033[0m') - authorname = input() + authorname = union_input() #fifth step - authoremail = input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') while authoremail == '': print ('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = input() + authoremail = union_input() #sixth step print ('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') print ("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = input() + license_class = union_input() while license_class == '' : print ('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = input() + license_class = union_input() if license_class.isdigit()== True and int(license_class) >= 1 and int(license_class) <= 4: license = license_index[int(license_class) - 1] @@ -963,10 +979,10 @@ def package_wizard(): print ('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') print ("\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - repository = input() + repository = union_input() while repository == '': print ('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = input() + repository = union_input() pkg_path = name if not os.path.exists(pkg_path): diff --git a/package.py b/package.py index e2e7283d..0d9894ab 100644 --- a/package.py +++ b/package.py @@ -130,7 +130,7 @@ class Package: pkg = None def parse(self, filename): - with open(filename, "r", encoding="utf-8") as f: + with open(filename, "r") as f: json_str = f.read() if json_str: diff --git a/pkgsdb.py b/pkgsdb.py index 15f66171..488008cb 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -26,6 +26,7 @@ import sqlite3 import os import hashlib +import sys from vars import Import SHOW_SQL = False @@ -178,8 +179,11 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): print ('Are you sure you want to permanently delete the file: %s?' % os.path.basename(pathname)) print ('If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') - rc = input( - 'Press the Y Key to delete the file or just press Enter to keep the file.') + + if sys.version_info < (3, 0): + rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') + else: + rc = input('Press the Y Key to delete the file or just press Enter to keep the file.') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" cursor = c.execute(sql) From 3b92fe150f9c187eaa31f06f1ed9fac1f6724e80 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 1 Nov 2019 09:12:21 +0800 Subject: [PATCH 15/16] [optimize] make the func get_url_from_mirror_server() compatible with py2 and py3 --- cmds/cmd_package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py index ff364517..82143bd3 100644 --- a/cmds/cmd_package.py +++ b/cmds/cmd_package.py @@ -166,7 +166,10 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): try: if type(pkgs_name_in_json) != type("str"): - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] + if sys.version_info < (3, 0): + pkgs_name_in_json = str(pkgs_name_in_json) + else: + pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] except Exception as e: print('error message:%s' % e) print("\nThe mirror server could not be contacted. Please check your network connection.") @@ -182,6 +185,8 @@ def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): } payload["packages"][0]['name'] = pkgs_name_in_json + # print(payload) + try: r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) From 14f6cefa9f538b64e8f46b3588f1a170e8a5debe Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 1 Nov 2019 11:53:17 +0800 Subject: [PATCH 16/16] [optimize] ignore the file .config --- .gitignore | 1 + cmds/.config | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 cmds/.config diff --git a/.gitignore b/.gitignore index 74996b3c..9eb72a81 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,4 @@ ENV/ .mypy_cache/ .vscode/settings.json cmds/.config.old +/cmds/.config diff --git a/cmds/.config b/cmds/.config deleted file mode 100644 index 29a03471..00000000 --- a/cmds/.config +++ /dev/null @@ -1,11 +0,0 @@ -# -# Automatically generated file; DO NOT EDIT. -# RootMenu -# - -# -# Env config -# -# CONFIG_SYS_AUTO_UPDATE_PKGS is not set -# CONFIG_SYS_CREATE_MDK_IAR_PROJECT is not set -CONFIG_SYS_PKGS_DOWNLOAD_ACCELERATE=y