diff --git a/archive.py b/archive.py index 24117f41..4dfd64b6 100644 --- a/archive.py +++ b/archive.py @@ -21,87 +21,137 @@ # Change Logs: # Date Author Notes # 2018-5-28 SummerGift Add copyright information +# 2020-4-10 SummerGift Code clear up # +import logging +import os +import shutil import tarfile import zipfile -import os import pkgsdb -import platform -import shutil - +from cmds.cmd_package.cmd_package_utils import is_windows, remove_folder -def unpack(archive_fn, path, pkg, pkgs_name_in_json): - pkg_ver = pkg['ver'] - flag = True - iswindows = False - - if platform.system() == "Windows": - iswindows = True - - if ".tar.bz2" in archive_fn: - arch = tarfile.open(archive_fn, "r:bz2") +def unpack(archive_filename, bsp_package_path, package_info, package_name): + if ".tar.bz2" in archive_filename: + arch = tarfile.open(archive_filename, "r:bz2") for tarinfo in arch: - arch.extract(tarinfo, path) + arch.extract(tarinfo, bsp_package_path) a = tarinfo.name - if not os.path.isdir(os.path.join(path, a)): - if iswindows: + if not os.path.isdir(os.path.join(bsp_package_path, a)): + if is_windows(): right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.savetodb(a, archive_fn) + + pkgsdb.save_to_database(a, archive_filename) arch.close() - if ".tar.gz" in archive_fn: - arch = tarfile.open(archive_fn, "r:gz") + if ".tar.gz" in archive_filename: + arch = tarfile.open(archive_filename, "r:gz") for tarinfo in arch: - arch.extract(tarinfo, path) + arch.extract(tarinfo, bsp_package_path) a = tarinfo.name - if not os.path.isdir(os.path.join(path, a)): - if iswindows: + if not os.path.isdir(os.path.join(bsp_package_path, a)): + if is_windows(): right_path = a.replace('/', '\\') else: right_path = a a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1]) - pkgsdb.savetodb(a, archive_fn) + pkgsdb.save_to_database(a, archive_filename) arch.close() - if ".zip" in archive_fn: - arch = zipfile.ZipFile(archive_fn, "r") + if ".zip" in archive_filename: + if not handle_zip_package(archive_filename, bsp_package_path, package_name, package_info): + return False + + return True + + +def handle_zip_package(archive_filename, bsp_package_path, package_name, package_info): + package_version = package_info['ver'] + package_temp_path = os.path.join(bsp_package_path, "package_temp") + + try: + if remove_folder(package_temp_path): + os.makedirs(package_temp_path) + except Exception as e: + logging.warning('Error message : {0}'.format(e)) + + logging.info("BSP packages path {0}".format(bsp_package_path)) + logging.info("BSP package temp path: {0}".format(package_temp_path)) + logging.info("archive filename : {0}".format(archive_filename)) + + try: + flag = True + package_folder_name = "" + package_name_with_version = "" + arch = zipfile.ZipFile(archive_filename, "r") for item in arch.namelist(): - arch.extract(item, path) - if not os.path.isdir(os.path.join(path, item)): - if iswindows: + arch.extract(item, package_temp_path) + if not os.path.isdir(os.path.join(package_temp_path, item)): + if is_windows(): right_path = item.replace('/', '\\') else: right_path = item - # Gets the folder name and change_dirname only once + # Gets the folder name and changed folder name only once if flag: - dir_name = os.path.split(right_path)[0] - change_dirname = pkgs_name_in_json + '-' + pkg_ver + package_folder_name = os.path.split(right_path)[0] + package_name_with_version = package_name + '-' + package_version flag = False - - right_name_to_db = right_path.replace(dir_name, change_dirname, 1) - pkgsdb.savetodb(right_name_to_db, archive_fn, right_path) - arch.close() - - # Change the folder name - change_dirname = pkgs_name_in_json + '-' + pkg_ver - - if os.path.isdir(os.path.join(path, change_dirname)): - if iswindows: - cmd = 'rd /s /q ' + os.path.join(path, change_dirname) - os.system(cmd) - else: - shutil.rmtree(os.path.join(path, change_dirname)) - - os.rename(os.path.join(path, dir_name),os.path.join(path, change_dirname)) + right_name_to_db = right_path.replace(package_folder_name, package_name_with_version, 1) + right_path = os.path.join("package_temp", right_path) + pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path) + arch.close() -def packtest(path): + if not move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, + bsp_package_path): + return False + except Exception as e: + logging.warning('unpack error message : {0}'.format(e)) + logging.warning('unpack {0} failed'.format(os.path.basename(archive_filename))) + # remove temp folder and archive file + remove_folder(package_temp_path) + os.remove(archive_filename) + return False + + return True + + +def move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version, + bsp_packages_path): + """move package in temp folder to bsp packages folder.""" + origin_package_folder_path = os.path.join(package_temp_path, package_folder_name) + package_name_with_version = package_name + '-' + package_version + package_folder_in_temp = os.path.join(package_temp_path, package_name_with_version) + bsp_package_path = os.path.join(bsp_packages_path, package_name_with_version) + logging.info("origin name: {0}".format(origin_package_folder_path)) + logging.info("rename name: {0}".format(package_folder_in_temp)) + + result = True + try: + # rename package folder name to package name with version + os.rename(origin_package_folder_path, package_folder_in_temp) + + # if there is no specified version package in the bsp package path, + # then move package from package_folder_in_temp to bsp_package_path + if not os.path.isdir(bsp_package_path): + shutil.move(package_folder_in_temp, bsp_package_path) + except Exception as e: + logging.warning('{0}'.format(e)) + result = False + finally: + # must remove temp folder + remove_folder(package_temp_path) + + return result + + +def package_integrity_test(path): ret = True if path.find(".zip") != -1: @@ -116,7 +166,7 @@ def packtest(path): ret = False print('package check error. \n') except Exception as e: - print('packtest error message:%s\t' % e) + print('Package test error message:%s\t' % e) print("The archive package is broken. \n") arch.close() ret = False @@ -127,6 +177,7 @@ def packtest(path): if not tarfile.is_tarfile(path): ret = False except Exception as e: + print('Error message:%s' % e) ret = False # if ".tar.gz" in path: @@ -135,6 +186,7 @@ def packtest(path): if not tarfile.is_tarfile(path): ret = False except Exception as e: + print('Error message:%s' % e) ret = False return ret diff --git a/cmds/__init__.py b/cmds/__init__.py index 1006f6a6..c5688f7c 100644 --- a/cmds/__init__.py +++ b/cmds/__init__.py @@ -24,3 +24,15 @@ # __all__ = ['cmd_package', 'cmd_system', 'cmd_menuconfig'] + +try: + import requests +except ImportError: + print("****************************************\n" + "* Import requests module error.\n" + "* Please install requests module first.\n" + "* pip install step:\n" + "* $ pip install requests\n" + "* command install step:\n" + "* $ sudo apt-get install python-requests\n" + "****************************************\n") diff --git a/cmds/cmd_menuconfig.py b/cmds/cmd_menuconfig.py index a8a75d60..d1eb2dea 100644 --- a/cmds/cmd_menuconfig.py +++ b/cmds/cmd_menuconfig.py @@ -28,13 +28,14 @@ import os import platform import re -from vars import Import, Export +from vars import Import +from .cmd_package.cmd_package_utils import find_macro_in_config def is_pkg_special_config(config_str): - ''' judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER''' + """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER""" - if type(config_str) == type('a'): + if isinstance(config_str, str): if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')): return True return False @@ -43,7 +44,8 @@ def is_pkg_special_config(config_str): def mk_rtconfig(filename): try: config = open(filename, 'r') - except: + except Exception as e: + print('Error message:%s' % e) print('open config:%s failed' % filename) return @@ -89,7 +91,7 @@ def mk_rtconfig(filename): if setting[1] == 'y': rtconfig.write('#define %s\n' % setting[0]) else: - rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$",line)[0])) + rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$", line)[0])) if os.path.isfile('rtconfig_project.h'): rtconfig.write('#include "rtconfig_project.h"\n') @@ -99,54 +101,7 @@ def mk_rtconfig(filename): rtconfig.close() -def find_macro_in_config(filename, macro_name): - try: - config = open(filename, "r") - except: - print('open .config failed') - return - - empty_line = 1 - - for line in config: - line = line.lstrip(' ').replace('\n', '').replace('\r', '') - - if len(line) == 0: - continue - - if line[0] == '#': - if len(line) == 1: - if empty_line: - continue - - empty_line = 1 - continue - - #comment_line = line[1:] - if line.startswith('# CONFIG_'): - line = ' ' + line[9:] - else: - line = line[1:] - - # print line - - empty_line = 0 - else: - empty_line = 0 - setting = line.split('=') - if len(setting) >= 2: - if setting[0].startswith('CONFIG_'): - setting[0] = setting[0][7:] - - if setting[0] == macro_name and setting[1] == 'y': - return True - - config.close() - return False - - def cmd(args): - env_root = Import('env_root') os_version = platform.platform(True)[10:13] kconfig_win7_path = os.path.join( @@ -159,9 +114,9 @@ def cmd(args): print("\n\033[1;31;40m 命令应当在某一特定 BSP 目录下执行，例如：\"rt-thread/bsp/stm32/stm32f091-st-nucleo\"\033[0m") print("\033[1;31;40m请确保当前目录为 BSP 根目录，并且该目录中有 Kconfig 文件。\033[0m\n") - print (" command should be used in a bsp root path with a Kconfig file.") - print ("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") - print ("You should check if there is a Kconfig file in your bsp root first.") + print(" command should be used in a bsp root path with a Kconfig file.") + print("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"") + print("You should check if there is a Kconfig file in your bsp root first.") if platform.system() == "Windows": os.system('chcp 437 > nul') @@ -195,7 +150,7 @@ def cmd(args): os.system('kconfig-mconf Kconfig -n') elif args.menuconfig_setting: - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') beforepath = os.getcwd() os.chdir(env_kconfig_path) @@ -229,7 +184,7 @@ def cmd(args): mk_rtconfig(fn) if platform.system() == "Windows": - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') fn = os.path.join(env_kconfig_path, '.config') if not os.path.isfile(fn): @@ -242,13 +197,13 @@ def cmd(args): if find_macro_in_config(fn, 'SYS_CREATE_MDK_IAR_PROJECT'): if find_macro_in_config(fn, 'SYS_CREATE_MDK4'): os.system('scons --target=mdk4 -s') - print("Create mdk4 project done") + print("Create mdk4 project done") elif find_macro_in_config(fn, 'SYS_CREATE_MDK5'): os.system('scons --target=mdk5 -s') - print("Create mdk5 project done") + print("Create mdk5 project done") elif find_macro_in_config(fn, 'SYS_CREATE_IAR'): os.system('scons --target=iar -s') - print("Create iar project done") + print("Create iar project done") def add_parser(sub): @@ -277,7 +232,7 @@ def add_parser(sub): dest='menuconfig_setting') parser.add_argument('--easy', - help='easy mode,place kconfig file everywhere,just modify the option env="RTT_ROOT" default "../.."', + help='easy mode, place kconfig everywhere, modify the option env="RTT_ROOT" default "../.."', action='store_true', default=False, dest='menuconfig_easy') diff --git a/cmds/cmd_package.py b/cmds/cmd_package.py deleted file mode 100644 index 2689b022..00000000 --- a/cmds/cmd_package.py +++ /dev/null @@ -1,1179 +0,0 @@ -# -*- coding:utf-8 -*- -# -# File : cmd_package.py -# This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Change Logs: -# Date Author Notes -# 2018-05-28 SummerGift Add copyright information -# 2018-12-28 Ernest Chen Add package information and enjoy package maker -# 2019-01-07 SummerGift The prompt supports utf-8 encoding -# - -import os -import json -import kconfig -import pkgsdb -import shutil -import platform -import subprocess -import time -import logging -import archive -import sys -import re -from package import Package, Bridge_SConscript, Kconfig_file, Package_json_file, Sconscript_file -from vars import Import, Export -from string import Template -from .cmd_menuconfig import find_macro_in_config - -try: - import requests -except ImportError: - print("****************************************\n" - "* Import requests module error.\n" - "* Please install requests module first.\n" - "* pip install step:\n" - "* $ pip install requests\n" - "* command install step:\n" - "* $ sudo apt-get install python-requests\n" - "****************************************\n") - -"""package command""" - -def execute_command(cmdstring, cwd=None, shell=True): - """Execute the system command at the specified address.""" - - if shell: - cmdstring_list = cmdstring - - sub = subprocess.Popen(cmdstring_list, cwd=cwd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, shell=shell, bufsize=4096) - - stdout_str = '' - while sub.poll() is None: - stdout_str += str(sub.stdout.read()) - time.sleep(0.1) - - return stdout_str - -def git_pull_repo(repo_path, repo_url=''): - if platform.system() == "Windows": - cmd = r'git config --local core.autocrlf true' - execute_command(cmd, cwd=repo_path) - cmd = r'git pull ' + repo_url - execute_command(cmd, cwd=repo_path) - -def determine_support_chinese(env_root): - get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') - if os.path.isfile(get_flag_file_path): - return True - else: - return False - -def user_input(msg, default_value): - """Gets the user's keyboard input.""" - - if default_value != '': - msg = '%s[%s]' % (msg, default_value) - - print(msg) - if sys.version_info < (3, 0): - value = raw_input() - else: - value = input() - - if value == '': - value = default_value - - return value - -def union_input(msg = None): - """Gets the union keyboard input.""" - - if sys.version_info < (3, 0): - if msg != None: - value = raw_input(msg) - else: - value = raw_input() - else: - if msg != None: - value = input(msg) - else: - value = input() - - return value - - -def get_mirror_giturl(submod_name): - """Gets the submodule's url on mirror server. - - Retrurn the download address of the submodule on the mirror server from the submod_name. - """ - - mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submod_name + '.git' - return mirror_url - - -def modify_submod_file_to_mirror(submod_path): - """Modify the.gitmodules file based on the submodule to be updated""" - - replace_list = [] - try: - with open(submod_path, 'r') as f: - for line in f: - line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') - if line.startswith('url'): - submod_git_url = line.split('=')[1] - submodule_name = submod_git_url.split('/')[-1].replace('.git', '') - replace_url = get_mirror_giturl(submodule_name) - # print(replace_url) - query_submodule_name = 'submod_' + submodule_name - # print(query_submodule_name) - get_package_url, get_ver_sha = get_url_from_mirror_server( - query_submodule_name, 'latest') - - if get_package_url != None and determine_url_valid(get_package_url): - replace_list.append( - (submod_git_url, replace_url, submodule_name)) - - with open(submod_path, 'r+') as f: - submod_file_count = f.read() - - write_content = submod_file_count - - for item in replace_list: - write_content = write_content.replace(item[0], item[1]) - - with open(submod_path, 'w') as f: - f.write(str(write_content)) - - return replace_list - - except Exception as e: - print('error message:%s\t' % e) - - -def get_url_from_mirror_server(pkgs_name_in_json, pkgs_ver): - """Get the download address from the mirror server based on the package name.""" - - try: - if type(pkgs_name_in_json) != type("str"): - if sys.version_info < (3, 0): - pkgs_name_in_json = str(pkgs_name_in_json) - else: - pkgs_name_in_json = str(pkgs_name_in_json)[2:-1] - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - - payload = { - "userName": "RT-Thread", - "packages": [ - { - "name": "NULL", - } - ] - } - payload["packages"][0]['name'] = pkgs_name_in_json - - # print(payload) - - try: - r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) - - if r.status_code == requests.codes.ok: - package_info = json.loads(r.text) - - # Can't find package,change git package SHA if it's a git - # package - if len(package_info['packages']) == 0: - print("Package was NOT found on mirror server. Using a non-mirrored address to download.") - return None, None - else: - for item in package_info['packages'][0]['packages_info']['site']: - if item['version'] == pkgs_ver: - # Change download url - download_url = item['URL'] - if download_url[-4:] == '.git': - # Change git package SHA - repo_sha = item['VER_SHA'] - return download_url, repo_sha - return download_url, None - - print("\nTips : \nThe system needs to be upgraded.") - print("Please use the command to upgrade packages index.\n") - return None, None - - except Exception as e: - print('error message:%s' % e) - print("\nThe mirror server could not be contacted. Please check your network connection.") - return None, None - -def determine_url_valid(url_from_srv): - - headers = {'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip, deflate', - 'Accept': '*/*', - 'User-Agent': 'curl/7.54.0'} - - try: - for i in range(0, 3): - r = requests.get(url_from_srv, stream=True, headers=headers) - if r.status_code == requests.codes.not_found: - if i == 2: - print("Warning : %s is invalid." % url_from_srv) - return False - time.sleep(1) - else: - break - - return True - - except Exception as e: - print('error message:%s\t' % e) - print('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) - - -def install_pkg(env_root, pkgs_root, bsp_root, pkg): - """Install the required packages.""" - - # default true - ret = True - local_pkgs_path = os.path.join(env_root, 'local_pkgs') - bsp_pkgs_path = os.path.join(bsp_root, 'packages') - - # get the .config file from env - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - url_from_json = package.get_url(pkg['ver']) - package_url = package.get_url(pkg['ver']) - pkgs_name_in_json = package.get_name() - - if package_url[-4:] == '.git': - ver_sha = package.get_versha(pkg['ver']) - - # print("==================================================>") - # print("packages name :"%pkgs_name_in_json.encode("utf-8")) - # print("ver :"%pkg['ver']) - # print("url :"%package_url.encode("utf-8")) - # print("url_from_json : "%url_from_json.encode("utf-8")) - # print("==================================================>") - - get_package_url = None - get_ver_sha = None - upstream_change_flag = False - - try: - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, pkg['ver']) - - # determine whether the package package url is valid - if get_package_url != None and determine_url_valid(get_package_url): - package_url = get_package_url - - if get_ver_sha != None: - ver_sha = get_ver_sha - - upstream_change_flag = True - except Exception as e: - print('error message:%s\t' % e) - print("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") - - if package_url[-4:] == '.git': - try: - repo_path = os.path.join(bsp_pkgs_path, pkgs_name_in_json) - repo_path = repo_path + '-' + pkg['ver'] - repo_path_full = '"' + repo_path + '"' - - clone_cmd = 'git clone ' + package_url + ' ' + repo_path_full - execute_command(clone_cmd, cwd=bsp_pkgs_path) - - git_check_cmd = 'git checkout -q ' + ver_sha - execute_command(git_check_cmd, cwd=repo_path) - - except Exception as e: - print("\nFailed to download software package with git. Please check the network connection.") - return False - - if upstream_change_flag: - cmd = 'git remote set-url origin ' + url_from_json - execute_command(cmd, cwd=repo_path) - - # If there is a .gitmodules file in the package, prepare to update submodule. - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): - print("Start to update submodule") - # print("开始更新软件包子模块") - - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - # print("开启了镜像加速，开始修改 .gitmodules 文件") - replace_list = modify_submod_file_to_mirror(submod_path) # Modify .gitmodules file - - # print("开始执行更新动作") - cmd = 'git submodule update --init --recursive' - execute_command(cmd, cwd=repo_path) - - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - if len(replace_list): - for item in replace_list: - submod_dir_path = os.path.join(repo_path, item[2]) - if os.path.isdir(submod_dir_path): - cmd = 'git remote set-url origin ' + item[0] - execute_command(cmd, cwd=submod_dir_path) - - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - if os.path.isfile(submod_path): - cmd = 'git checkout .gitmodules' - execute_command(cmd, cwd=repo_path) - - else: - # Download a package of compressed package type. - if not package.download(pkg['ver'], local_pkgs_path, package_url): - return False - - pkg_dir = package.get_filename(pkg['ver']) - pkg_dir = os.path.splitext(pkg_dir)[0] - pkg_fullpath = os.path.join(local_pkgs_path, package.get_filename(pkg['ver'])) - - if not archive.packtest(pkg_fullpath): - print("package : %s is invalid"%pkg_fullpath.encode("utf-8")) - return False - - # unpack package - if not os.path.exists(pkg_dir): - - try: - if not package.unpack(pkg_fullpath, bsp_pkgs_path, pkg, pkgs_name_in_json): - ret = False - except Exception as e: - os.remove(pkg_fullpath) - ret = False - print('error message: %s\t' % e) - else: - print("The file does not exist.") - return ret - - -def package_list(): - """Print the packages list in env. - - Read the.config file in the BSP directory, - and list the version number of the selected package. - """ - - fn = '.config' - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - if not os.path.isfile(fn): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print ('\033[1;31;40mNo system configuration file : .config.\033[0m') - print ('\033[1;31;40mYou should use < menuconfig > command to config bsp first.\033[0m') - - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return - - pkgs = kconfig.parse(fn) - - for pkg in pkgs: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - - pkgs_name_in_json = package.get_name() - print ("package name : %s, ver : %s "%(pkgs_name_in_json.encode("utf-8"), pkg['ver'].encode("utf-8"))) - - if not pkgs: - print ("Packages list is empty.") - print ('You can use < menuconfig > command to select online packages.') - print ('Then use < pkgs --update > command to install them.') - return - - -def sub_list(aList, bList): - """Return the items in aList but not in bList.""" - - tmp = [] - for a in aList: - if a not in bList: - tmp.append(a) - return tmp - - -def and_list(aList, bList): - """Return the items in aList and in bList.""" - - tmp = [] - for a in aList: - if a in bList: - tmp.append(a) - return tmp - - -def update_submodule(repo_path): - """Update the submodules in the repository.""" - - submod_path = os.path.join(repo_path, '.gitmodules') - if os.path.isfile(submod_path): - print("Please wait a few seconds in order to update the submodule.") - cmd = 'git submodule init -q' - execute_command(cmd, cwd=repo_path) - cmd = 'git submodule update' - execute_command(cmd, cwd=repo_path) - print("Submodule update successful") - - -def get_pkg_folder_by_orign_path(orign_path, version): - # TODO fix for old version project, will remove after new major version - # release - if os.path.exists(orign_path + '-' + version): - return orign_path + '-' + version - return orign_path - - -def git_cmd_exec(cmd, cwd): - try: - execute_command(cmd, cwd=cwd) - except Exception as e: - print('error message:%s%s. %s \n\t' %(cwd.encode("utf-8"), " path doesn't exist", e)) - print("You can solve this problem by manually removing old packages and re-downloading them using env.") - - -def update_latest_packages(pkgs_fn, bsp_packages_path): - """ update the packages that are latest version. - - If the selected package is the latest version, - check to see if it is the latest version after the update command, - if not, then update the latest version from the remote repository. - If the download has a conflict, you are currently using the prompt - message provided by git. - """ - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - for pkg in read_back_pkgs_json: - package = Package() - pkg_path = pkg['path'] - if pkg_path[0] == '/' or pkg_path[0] == '\\': - pkg_path = pkg_path[1:] - - pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') - package.parse(pkg_path) - pkgs_name_in_json = package.get_name() - - # Find out the packages which version is 'latest' - if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": - repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) - repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver']) - - try: - # If mirror acceleration is enabled, get the update address from - # the mirror server. - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") - - # Change repo's upstream address. - mirror_url = get_url_from_mirror_server( - payload_pkgs_name_in_json, pkg['ver']) - - if mirror_url[0] != None: - cmd = 'git remote set-url origin ' + mirror_url[0] - git_cmd_exec(cmd, repo_path) - - except Exception as e: - print("error message : %s" % e) - print("Failed to connect to the mirror server, using non-mirror server to update.") - - # Update the package repository from upstream. - git_pull_repo(repo_path) - - # If the package has submodules, update the submodules. - update_submodule(repo_path) - - # recover origin url to the path which get from packages.json file - if package.get_url(pkg['ver']): - cmd = 'git remote set-url origin ' + \ - package.get_url(pkg['ver']) - git_cmd_exec(cmd, repo_path) - else: - print("Can't find the package : %s's url in file : %s" % - (payload_pkgs_name_in_json, pkg_path)) - - print("==============================> %s update done\n" %(pkgs_name_in_json)) - - -def pre_package_update(): - """ Make preparations before updating the software package. """ - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - - if not os.path.exists('.config'): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - print ("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") - print ("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") - - print ('No system configuration file : .config.') - print ('You should use < menuconfig > command to config bsp first.') - - if platform.system() == "Windows": - os.system('chcp 437 > nul') - - return False - - bsp_packages_path = os.path.join(bsp_root, 'packages') - if not os.path.exists(bsp_packages_path): - os.mkdir("packages") - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # prepare target packages file - dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') - Export('dbsqlite_pathname') - dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') - - # Avoid creating tables more than one time - if not os.path.isfile(dbsqlite_pathname): - conn = pkgsdb.get_conn(dbsqlite_pathname) - sql = '''CREATE TABLE packagefile - (pathname TEXT ,package TEXT ,md5 TEXT );''' - pkgsdb.create_table(conn, sql) - - fn = '.config' - pkgs = kconfig.parse(fn) - - # print("newpkgs", pkgs) - - newpkgs = pkgs - - if not os.path.exists(bsp_packages_path): - os.mkdir(bsp_packages_path) - - pkgs_fn = os.path.join(bsp_packages_path, 'pkgs.json') - - # regenerate file : packages/pkgs.json - if not os.path.exists(pkgs_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs.json - with open(pkgs_fn, 'r') as f: - oldpkgs = json.load(f) - - # print("oldpkgs", oldpkgs) - - # regenerate file : packages/pkgs_error.json - pkgs_error_list_fn = os.path.join( - bsp_packages_path, 'pkgs_error.json') - - if not os.path.exists(pkgs_error_list_fn): - os.chdir(bsp_packages_path) - fp = open("pkgs_error.json", 'w') - fp.write("[]") - fp.close() - os.chdir(bsp_root) - - # Reading data back from pkgs_error.json - with open(pkgs_error_list_fn, 'r') as f: - pkgs_error = json.load(f) - - # create SConscript file - if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): - with open(os.path.join(bsp_packages_path, 'SConscript'),'w') as f: - f.write(str(Bridge_SConscript)) - - return [oldpkgs, newpkgs, pkgs_error, pkgs_fn, pkgs_error_list_fn, bsp_packages_path, dbsqlite_pathname] - - -def error_packages_handle(error_packages_list, read_back_pkgs_json, pkgs_fn): - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - - flag = None - - error_packages_redownload_error_list = [] - - if len(error_packages_list): - print("\n==============================> Packages list to download : \n") - for pkg in error_packages_list: - print("Package name : %s, Ver : %s"%(pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - print("\nThe packages in the list above are accidentally deleted, env will redownload them.") - print("Warning: Packages should be deleted in command.\n") - - for pkg in error_packages_list: # Redownloaded the packages in error_packages_list - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is redownloaded successfully. \n" % ( - pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - else: - error_packages_redownload_error_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - if len(error_packages_redownload_error_list): - print("%s" % error_packages_redownload_error_list) - print ("Packages:%s,%s redownloed error, you need to use command again to redownload them." % - (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8"))) - write_back_pkgs_json = sub_list( - read_back_pkgs_json, error_packages_redownload_error_list) - read_back_pkgs_json = write_back_pkgs_json - # print("write_back_pkgs_json:%s"%write_back_pkgs_json) - pkgs_file = file(pkgs_fn, 'w') - pkgs_file.write(json.dumps(write_back_pkgs_json, indent=1)) - pkgs_file.close() - - return flag - - -def rm_package(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rd /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - if platform.system() != "Windows": - shutil.rmtree(dir) - else: - dir = '"' + dir + '"' - cmd = 'rmdir /s /q ' + dir - os.system(cmd) - - if os.path.isdir(dir): - print ("Folder path: %s" % dir.encode("utf-8")) - return False - else: - print ("Path: %s \nSuccess: Folder has been removed. " % dir.encode("utf-8")) - return True - - -def get_package_remove_path(pkg, bsp_packages_path): - dirpath = pkg['path'] - ver = pkg['ver'] - if dirpath[0] == '/' or dirpath[0] == '\\': - dirpath = dirpath[1:] - - if platform.system() == "Windows": - dirpath = os.path.basename(dirpath.replace('/', '\\')) - else: - dirpath = os.path.basename(dirpath) - - removepath = os.path.join(bsp_packages_path, dirpath) - - # Handles the deletion of git repository folders with version Numbers - removepath_ver = get_pkg_folder_by_orign_path(removepath, ver) - return removepath_ver - - -def handle_download_error_packages(pkgs_fn, bsp_packages_path): - """ handle download error packages. - - Check to see if the packages stored in the Json file list actually exist, - and then download the packages if they don't exist. - """ - - with open(pkgs_fn, 'r') as f: - read_back_pkgs_json = json.load(f) - - error_packages_list = [] - - for pkg in read_back_pkgs_json: - removepath = get_package_remove_path(pkg, bsp_packages_path) - - if os.path.exists(removepath): - continue - else: - error_packages_list.append(pkg) - - # Handle the failed download packages - get_flag = error_packages_handle( - error_packages_list, read_back_pkgs_json, pkgs_fn) - - return get_flag - - -def write_storage_file(pkgs_fn, newpkgs): - """Writes the updated configuration to pkgs.json file. - - Packages that are not downloaded correctly will be redownloaded at the - next update. - """ - - with open(pkgs_fn,'w') as f: - f.write(str(json.dumps(newpkgs, indent=1))) - - -def package_update(isDeleteOld=False): - """Update env's packages. - - Compare the old and new software package list and update the package. - Remove unwanted packages and download the newly selected package.- - Check if the files in the deleted packages have been changed, and if so, - remind the user saved the modified file. - """ - - sys_value = pre_package_update() - - if not sys_value: - return - - bsp_root = Import('bsp_root') - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - flag = True - - # According to the env version, whether Chinese output is supported or not - if determine_support_chinese(env_root): - if platform.system() == "Windows": - os.system('chcp 65001 > nul') - - oldpkgs = sys_value[0] - newpkgs = sys_value[1] - pkgs_delete_error_list = sys_value[2] - pkgs_fn = sys_value[3] - pkgs_error_list_fn = sys_value[4] - bsp_packages_path = sys_value[5] - dbsqlite_pathname = sys_value[6] - - # print(oldpkgs) - # print(newpkgs) - - if len(pkgs_delete_error_list): - for error_package in pkgs_delete_error_list: - removepath_ver = get_package_remove_path( - error_package, bsp_packages_path) - - if os.path.isdir(removepath_ver): - print("\nError: %s package delete failed, begin to remove it."% - error_package['name'].encode("utf-8")) - - if rm_package(removepath_ver) == False: - print("Error: Delete package %s failed! Please delete the folder manually.\n"%error_package['name'].encode("utf-8")) - return - - # 1.in old ,not in new : Software packages that need to be removed. - casedelete = sub_list(oldpkgs, newpkgs) - pkgs_delete_fail_list = [] - - for pkg in casedelete: - - removepath_ver = get_package_remove_path(pkg, bsp_packages_path) - removepath_git = os.path.join(removepath_ver, '.git') - - # Delete. Git directory. - if os.path.isdir(removepath_ver) and os.path.isdir(removepath_git): - gitdir = removepath_ver - - print ("\nStart to remove %s \nplease wait..." % gitdir.encode("utf-8")) - if isDeleteOld: - if rm_package(gitdir) == False: - print("Floder delete fail: %s" % gitdir.encode("utf-8")) - print("Please delete this folder manually.") - else: - print ("The folder is managed by git. Do you want to delete this folder?\n") - if sys.version_info < (3, 0): - rc = raw_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - else: - rc = input('Press the Y Key to delete the folder or just press Enter to keep it : ') - - if rc == 'y' or rc == 'Y': - try: - if rm_package(gitdir) == False: - pkgs_delete_fail_list.append(pkg) - print("Error: Please delete the folder manually.") - except Exception as e: - print('Error message:%s%s. error.message: %s\n\t' % - ("Delete folder failed: ", gitdir.encode("utf-8"), e)) - else: - if os.path.isdir(removepath_ver): - print("Start to remove %s \nplease wait..." % removepath_ver.encode("utf-8")) - try: - pkgsdb.deletepackdir(removepath_ver, dbsqlite_pathname) - except Exception as e: - pkgs_delete_fail_list.append(pkg) - print('Error message:\n%s %s. %s \n\t' % ( - "Delete folder failed, please delete the folder manually", removepath_ver.encode("utf-8"), e)) - - if len(pkgs_delete_fail_list): - # write error messages - pkgs_file = file(pkgs_error_list_fn, 'w') - pkgs_file.write(json.dumps(pkgs_delete_fail_list, indent=1)) - pkgs_file.close() - return - else: - # write error messages - with open(pkgs_error_list_fn,'w') as f: - f.write(str(json.dumps(pkgs_delete_fail_list, indent=1))) - - # 2.in new not in old : Software packages to be installed. - # If the package download fails, record it, and then download again when - # the update command is executed. - - casedownload = sub_list(newpkgs, oldpkgs) - # print 'in new not in old:', casedownload - pkgs_download_fail_list = [] - - for pkg in casedownload: - if install_pkg(env_root, pkgs_root, bsp_root, pkg): - print("==============================> %s %s is downloaded successfully. \n" % ( - pkg['name'], pkg['ver'])) - else: - # If the PKG download fails, record it in the - # pkgs_download_fail_list. - pkgs_download_fail_list.append(pkg) - print(pkg, 'download failed.') - flag = False - - # Get the currently updated configuration. - newpkgs = sub_list(newpkgs, pkgs_download_fail_list) - - # Give hints based on the success of the download. - if len(pkgs_download_fail_list): - print("\nPackage download failed list:" ) - for item in pkgs_download_fail_list: - print(item) - - print("You need to reuse the command to download again.") - - # update pkgs.json and SConscript - write_storage_file(pkgs_fn, newpkgs) - - # handle download error packages. - get_flag = handle_download_error_packages( - pkgs_fn, bsp_packages_path) - - if get_flag != None: - flag = get_flag - - # Update the software packages, which the version is 'latest' - try: - update_latest_packages(pkgs_fn, bsp_packages_path) - except KeyboardInterrupt: - flag = False - - if flag: - print ("Operation completed successfully.") - else: - print ("Operation failed.") - -def package_wizard(): - """Packages creation wizard. - - The user enters the package name, version number, category, and automatically generates the package index file. - """ - # Welcome - print ('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') - - #Simple introduction about the wizard - print ('note :') - print (' \033[5;35;40m[ ]\033[0m means default setting or optional information.') - print (' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') - - #first step - print ('\033[5;33;40m\n1.Please input a new package name :\033[0m') - - name = union_input() - regular_obj = re.compile('\W') - while name == '' or name.isspace() == True or regular_obj.search(name.strip()): - if name == '' or name.isspace(): - print ('\033[1;31;40mError: you must input a package name. Try again.\033[0m') - name = union_input() - else: - print ('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') - name = union_input() - - default_description = 'Please add description of ' + name + ' in English.' - #description = user_input('menuconfig option name,default:\n',default_description) - description = default_description - description_zh = "请添加软件包 " + name +" 的中文描述。" - - #second step - ver = user_input('\033[5;33;40m\n2.Please input this package version, default :\033[0m', '1.0.0') - ver_standard = ver.replace('.', '') - #keyword = user_input('keyword,default:\n', name) - keyword = name - - #third step - packageclass = ('iot', 'language', 'misc', 'multimedia', - 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') - print ('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') - print ("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") - classnu = union_input() - while classnu == '' or classnu.isdigit()== False or int(classnu) < 1 or int(classnu) >9: - if classnu == '' : - print ('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') - else : - print ('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') - classnu = union_input() - - pkgsclass = packageclass[int(classnu) - 1] - - #fourth step - print ("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") - - authorname = union_input() - while authorname == '': - print ("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") - authorname = union_input() - - #fifth step - authoremail = union_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') - while authoremail == '': - print ('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') - authoremail = union_input() - - #sixth step - print ('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') - print ("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") - license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') - license_class = union_input() - while license_class == '' : - print ('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') - license_class = union_input() - - if license_class.isdigit()== True and int(license_class) >= 1 and int(license_class) <= 4: - license = license_index[int(license_class) - 1] - else : - license = license_class - - #seventh step - print ('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') - print ("\033[1;32;40mFor example, hello package's repository url is 'https://github.com/RT-Thread-packages/hello'.\033[0m") - - repository = union_input() - while repository == '': - print ('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') - repository = union_input() - - pkg_path = name - if not os.path.exists(pkg_path): - os.mkdir(pkg_path) - else: - print ("\033[1;31;40mError: the package directory is exits!\033[0m") - - s = Template(Kconfig_file) - uppername = str.upper(name) - kconfig = s.substitute(name=uppername, description=description, version=ver, - pkgs_class=pkgsclass, lowercase_name=name, version_standard=ver_standard) - f = open(os.path.join(pkg_path, 'Kconfig'), 'w') - f.write(kconfig) - f.close() - - s = Template(Package_json_file) - package = s.substitute(name=name, pkgsclass=pkgsclass,authorname=authorname,authoremail=authoremail, description=description, description_zh=description_zh,version=ver, keyword=keyword,license=license, repository=repository, pkgs_using_name=uppername) - f = open(os.path.join(pkg_path, 'package.json'), 'w') - f.write(package) - f.close() - - print ('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') - print ('Please \033[5;34;40mupdate\033[0m other information of this package based on Kconfig and package.json in directory '+name+'.') - -def upgrade_packages_index(): - """Update the package repository index.""" - - env_root = Import('env_root') - pkgs_root = Import('pkgs_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') - if get_package_url != None: - git_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' - else: - git_repo = 'https://github.com/RT-Thread/packages.git' - - packages_root = pkgs_root - pkgs_path = os.path.join(packages_root, 'packages') - - if not os.path.isdir(pkgs_path): - cmd = 'git clone ' + git_repo + ' ' + pkgs_path - os.system(cmd) - print ("upgrade from :%s" % (git_repo.encode("utf-8"))) - else: - print("Begin to upgrade env packages.") - git_pull_repo(pkgs_path, git_repo) - print("==============================> Env packages upgrade done \n") - - for filename in os.listdir(packages_root): - package_path = os.path.join(packages_root, filename) - if os.path.isdir(package_path): - - if package_path == pkgs_path: - continue - - if os.path.isdir(os.path.join(package_path, '.git')): - print("Begin to upgrade %s." % filename) - git_pull_repo(package_path) - print("==============================> Env %s update done \n" % filename) - - -def upgrade_env_script(): - """Update env function scripts.""" - - print("Begin to upgrade env scripts.") - env_root = Import('env_root') - env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds') - env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): - get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') - if get_package_url != None: - env_scripts_repo = get_package_url - else: - print("Failed to get url from mirror server. Using default url.") - env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' - else: - env_scripts_repo = 'https://github.com/RT-Thread/env.git' - - env_scripts_root = os.path.join(env_root, 'tools', 'scripts') - git_pull_repo(env_scripts_root, env_scripts_repo) - print("==============================> Env scripts upgrade done \n") - - -def package_upgrade(): - """Update the package repository directory and env function scripts.""" - - upgrade_packages_index() - upgrade_env_script() - - -def package_print_env(): - print ("Here are some environmental variables.") - print ( - "If you meet some problems,please check them. Make sure the configuration is correct.") - print ("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) - print ("RTT_CC:%s" % (os.getenv("RTT_CC"))) - print ("SCONS:%s" % (os.getenv("SCONS"))) - print ("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) - - env_root = os.getenv('ENV_ROOT') - if env_root == None: - if platform.system() != 'Windows': - env_root = os.path.join(os.getenv('HOME'), '.env') - - print ("ENV_ROOT:%s" % (env_root)) - - -def cmd(args): - """Env's pkgs command execution options.""" - - if args.package_update_y: - package_update(True) - elif args.package_update: - package_update() - elif args.package_create: - package_wizard() - elif args.package_list: - package_list() - elif args.package_upgrade: - package_upgrade() - elif args.package_print_env: - package_print_env() - else: - os.system('pkgs -h') - - -def add_parser(sub): - """The pkgs command parser for env.""" - - parser = sub.add_parser('package', help=__doc__, description=__doc__) - - parser.add_argument('--force-update', - help='force update and clean packages, install or remove the packages by your settings in menuconfig', - action='store_true', - default=False, - dest='package_update_y') - - parser.add_argument('--update', - help='update packages, install or remove the packages by your settings in menuconfig', - action='store_true', - default=False, - dest='package_update') - - parser.add_argument('--list', - help='list target packages', - action='store_true', - default=False, - dest='package_list') - - parser.add_argument('--wizard', - help='create a new package with wizard', - action='store_true', - default=False, - dest='package_create') - - parser.add_argument('--upgrade', - help='upgrade local packages list and ENV scripts from git repo', - action='store_true', - default=False, - dest='package_upgrade') - - parser.add_argument('--printenv', - help='print environmental variables to check', - action='store_true', - default=False, - dest='package_print_env') - - # parser.add_argument('--upgrade', dest='reposource', required=False, - # help='add source & update packages repo ') - - parser.set_defaults(func=cmd) diff --git a/cmds/cmd_package/__init__.py b/cmds/cmd_package/__init__.py new file mode 100644 index 00000000..68361a40 --- /dev/null +++ b/cmds/cmd_package/__init__.py @@ -0,0 +1,97 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2018-05-28 SummerGift Add copyright information +# 2018-12-28 Ernest Chen Add package information and enjoy package maker +# 2019-01-07 SummerGift The prompt supports utf-8 encoding +# 2020-04-08 SummerGift Optimize program structure +# + +from .cmd_package_printenv import package_print_env, package_print_help +from .cmd_package_list import list_packages +from .cmd_package_wizard import package_wizard +from .cmd_package_upgrade import package_upgrade +from .cmd_package_update import package_update + + +def run_env_cmd(args): + """Run packages command.""" + + if args.package_update_force: + package_update(True) + elif args.package_update: + package_update() + elif args.package_create: + package_wizard() + elif args.list_packages: + list_packages() + elif args.package_upgrade: + package_upgrade() + elif args.package_print_env: + package_print_env() + else: + package_print_help() + + +def add_parser(sub): + """The packages command parser for env.""" + + parser = sub.add_parser('package', help=__doc__, description=__doc__) + + parser.add_argument('--force-update', + help='force update and clean packages, install or remove packages by settings in menuconfig', + action='store_true', + default=False, + dest='package_update_force') + + parser.add_argument('--update', + help='update packages, install or remove the packages by your settings in menuconfig', + action='store_true', + default=False, + dest='package_update') + + parser.add_argument('--list', + help='list target packages', + action='store_true', + default=False, + dest='list_packages') + + parser.add_argument('--wizard', + help='create a new package with wizard', + action='store_true', + default=False, + dest='package_create') + + parser.add_argument('--upgrade', + help='upgrade local packages list and ENV scripts from git repo', + action='store_true', + default=False, + dest='package_upgrade') + + parser.add_argument('--printenv', + help='print environmental variables to check', + action='store_true', + default=False, + dest='package_print_env') + + parser.set_defaults(func=run_env_cmd) + diff --git a/cmds/cmd_package/cmd_package_list.py b/cmds/cmd_package/cmd_package_list.py new file mode 100644 index 00000000..1b0cbcc8 --- /dev/null +++ b/cmds/cmd_package/cmd_package_list.py @@ -0,0 +1,73 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import platform +import kconfig +from package import PackageOperation +from vars import Import + + +def list_packages(): + """Print the packages list in env. + + Read the.config file in the BSP directory, + and list the version number of the selected package. + """ + + config_file = '.config' + pkgs_root = Import('pkgs_root') + + if not os.path.isfile(config_file): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\033[1;31;40mWarning: Can't find .config.\033[0m") + print('\033[1;31;40mYou should use command to config bsp first.\033[0m') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return + + packages = kconfig.parse(config_file) + + for pkg in packages: + package = PackageOperation() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + package_name_in_json = package.get_name().encode("utf-8") + print("package name : %s, ver : %s " % (package_name_in_json, pkg['ver'].encode("utf-8"))) + + if not packages: + print("Packages list is empty.") + print('You can use < menuconfig > command to select online packages.') + print('Then use < pkgs --update > command to install them.') + return diff --git a/cmds/cmd_package/cmd_package_printenv.py b/cmds/cmd_package/cmd_package_printenv.py new file mode 100644 index 00000000..36291da5 --- /dev/null +++ b/cmds/cmd_package/cmd_package_printenv.py @@ -0,0 +1,47 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import platform + + +def package_print_env(): + print("Here are some environmental variables.") + print("If you meet some problems,please check them. Make sure the configuration is correct.") + print("RTT_EXEC_PATH:%s" % (os.getenv("RTT_EXEC_PATH"))) + print("RTT_CC:%s" % (os.getenv("RTT_CC"))) + print("SCONS:%s" % (os.getenv("SCONS"))) + print("PKGS_ROOT:%s" % (os.getenv("PKGS_ROOT"))) + + env_root = os.getenv('ENV_ROOT') + if env_root is None: + if platform.system() != 'Windows': + env_root = os.path.join(os.getenv('HOME'), '.env') + + print("ENV_ROOT:%s" % env_root) + + +def package_print_help(): + os.system('pkgs -h') diff --git a/cmds/cmd_package/cmd_package_update.py b/cmds/cmd_package/cmd_package_update.py new file mode 100644 index 00000000..25fdc9f5 --- /dev/null +++ b/cmds/cmd_package/cmd_package_update.py @@ -0,0 +1,845 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# 2020-04-13 SummerGift refactoring +# + +import json +import logging +import os +import platform +import shutil +import time + +import requests + +import archive +import kconfig +import pkgsdb +from package import PackageOperation, Bridge_SConscript +from vars import Import, Export +from .cmd_package_utils import get_url_from_mirror_server, execute_command, git_pull_repo, user_input, \ + find_macro_in_config + + +def determine_support_chinese(env_root): + get_flag_file_path = os.path.join(env_root, 'tools', 'bin', 'env_above_ver_1_1') + if os.path.isfile(get_flag_file_path): + return True + else: + return False + + +def get_mirror_giturl(submodule_name): + """Gets the submodule's url on mirror server. + + Retrurn the download address of the submodule on the mirror server from the submod_name. + """ + + mirror_url = 'https://gitee.com/RT-Thread-Mirror/submod_' + submodule_name + '.git' + return mirror_url + + +def modify_submod_file_to_mirror(submodule_path): + """Modify the.gitmodules file based on the submodule to be updated""" + + replace_list = [] + try: + with open(submodule_path, 'r') as f: + for line in f: + line = line.replace('\t', '').replace(' ', '').replace('\n', '').replace('\r', '') + if line.startswith('url'): + submodule_git_url = line.split('=')[1] + submodule_name = submodule_git_url.split('/')[-1].replace('.git', '') + replace_url = get_mirror_giturl(submodule_name) + query_submodule_name = 'submod_' + submodule_name + get_package_url, get_ver_sha = get_url_from_mirror_server( + query_submodule_name, 'latest') + + if get_package_url is not None and determine_url_valid(get_package_url): + replace_list.append( + (submodule_git_url, replace_url, submodule_name)) + + with open(submodule_path, 'r+') as f: + submod_file_count = f.read() + + write_content = submod_file_count + + for item in replace_list: + write_content = write_content.replace(item[0], item[1]) + + with open(submodule_path, 'w') as f: + f.write(str(write_content)) + + return replace_list + + except Exception as e: + logging.warning('Error message:%s\t' % e) + + +def determine_url_valid(url_from_srv): + headers = {'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + 'Accept': '*/*', + 'User-Agent': 'curl/7.54.0'} + + # noinspection PyBroadException + try: + for i in range(0, 3): + r = requests.get(url_from_srv, stream=True, headers=headers) + if r.status_code == requests.codes.not_found: + if i == 2: + print("Warning : %s is invalid." % url_from_srv) + return False + time.sleep(1) + else: + break + + return True + + except Exception as e: + # So much error message should be ignore + logging.error('Network connection error or the url : %s is invalid.\n' % url_from_srv.encode("utf-8")) + + +def is_user_mange_package(bsp_package_path, pkg): + for root, dirs, files in os.walk(bsp_package_path, topdown=True): + for name in dirs: + package_name_lower = pkg["name"].lower() + folder_name_lower = name.lower() + folder_name_common = folder_name_lower.replace("-", "_") + if folder_name_lower == package_name_lower or folder_name_common == package_name_lower: + return True + break + return False + + +def need_using_mirror_download(config_file): + """default using mirror url to download packages""" + + if not os.path.isfile(config_file): + return True + elif os.path.isfile(config_file) and find_macro_in_config(config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE'): + return True + + +def is_git_url(package_url): + return package_url.endswith('.git') + + +def install_git_package(bsp_package_path, package_name, package_info, package_url, ver_sha, upstream_changed, + url_origin, env_config_file): + try: + repo_path = os.path.join(bsp_package_path, package_name) + repo_path = repo_path + '-' + package_info['ver'] + repo_name_with_version = '"' + repo_path + '"' + + clone_cmd = 'git clone ' + package_url + ' ' + repo_name_with_version + logging.info(clone_cmd) + execute_command(clone_cmd, cwd=bsp_package_path) + + git_check_cmd = 'git checkout -q ' + ver_sha + execute_command(git_check_cmd, cwd=repo_path) + except Exception as e: + print('Error message:%s' % e) + print("\nFailed to download software package with git. Please check the network connection.") + return False + + # change upstream to origin url + if upstream_changed: + cmd = 'git remote set-url origin ' + url_origin + execute_command(cmd, cwd=repo_path) + + # If there is a .gitmodules file in the package, prepare to update submodule. + submodule_path = os.path.join(repo_path, '.gitmodules') + if os.path.isfile(submodule_path): + print("Start to update submodule") + if need_using_mirror_download(env_config_file): + replace_list = modify_submod_file_to_mirror(submodule_path) # Modify .gitmodules file + + cmd = 'git submodule update --init --recursive' + execute_command(cmd, cwd=repo_path) + + if need_using_mirror_download(env_config_file): + if len(replace_list): + for item in replace_list: + submodule_path = os.path.join(repo_path, item[2]) + if os.path.isdir(submodule_path): + cmd = 'git remote set-url origin ' + item[0] + execute_command(cmd, cwd=submodule_path) + + if need_using_mirror_download(env_config_file): + if os.path.isfile(submodule_path): + cmd = 'git checkout .gitmodules' + execute_command(cmd, cwd=repo_path) + + return True + + +def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, package_name): + result = True + # Download a package of compressed package type. + if not package.download(package_info['ver'], local_pkgs_path, package_url): + return False + + pkg_dir = package.get_filename(package_info['ver']) + pkg_dir = os.path.splitext(pkg_dir)[0] + package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) + + if not archive.package_integrity_test(package_path): + print("package : %s is invalid" % package_path.encode("utf-8")) + return False + + # unpack package + if not os.path.exists(pkg_dir): + try: + if not archive.unpack(package_path, bsp_package_path, package_info, package_name): + result = False + except Exception as e: + result = False + logging.error('Error message: {0}'.format(e)) + else: + print("The file does not exist.") + + return result + + +# noinspection PyUnboundLocalVariable +def install_package(env_root, pkgs_root, bsp_root, package_info, force_update): + """Install the required packages.""" + + result = True + local_pkgs_path = os.path.join(env_root, 'local_pkgs') + bsp_package_path = os.path.join(bsp_root, 'packages') + + if not force_update: + logging.info( + "Begin to check if it's an user managed package {0}, {1} \n".format(bsp_package_path, package_info)) + if is_user_mange_package(bsp_package_path, package_info): + logging.info("User managed package {0}, {1} no need install. \n".format(bsp_package_path, package_info)) + return result + else: + logging.info("NOT User managed package {0}, {1} need install. \n".format(bsp_package_path, package_info)) + + # get the .config file from env + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') + + package = PackageOperation() + pkg_path = package_info['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + + url_from_json = package.get_url(package_info['ver']) + + if not url_from_json: + return False + + package_url = url_from_json + + pkgs_name_in_json = package.get_name() + logging.info("begin to install packages: {0}".format(pkgs_name_in_json)) + if is_git_url(package_url): + ver_sha = package.get_versha(package_info['ver']) + + upstream_changed = False + + # noinspection PyBroadException + try: + if need_using_mirror_download(env_config_file): + get_package_url, get_ver_sha = get_url_from_mirror_server(pkgs_name_in_json, package_info['ver']) + + # Check whether the package package url is valid + if get_package_url and determine_url_valid(get_package_url): + package_url = get_package_url + + if get_ver_sha: + ver_sha = get_ver_sha + + upstream_changed = True + except Exception as e: + logging.warning("Failed to connect to the mirror server, package will be downloaded from non-mirror server.\n") + + if is_git_url(package_url): + if not install_git_package(bsp_package_path, pkgs_name_in_json, package_info, package_url, ver_sha, + upstream_changed, + url_from_json, env_config_file): + result = False + else: + if not install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, + pkgs_name_in_json): + result = False + return result + + +def sub_list(aList, bList): + """Return the items in aList but not in bList.""" + + tmp = [] + for a in aList: + if a not in bList: + tmp.append(a) + return tmp + + +def and_list(aList, bList): + """Return the items in aList and in bList.""" + + tmp = [] + for a in aList: + if a in bList: + tmp.append(a) + return tmp + + +def update_submodule(repo_path): + """Update the submodules in the repository.""" + + try: + if os.path.isfile(os.path.join(repo_path, '.gitmodules')): + print("Please wait a few seconds in order to update the submodule.") + cmd = 'git submodule init -q' + execute_command(cmd, cwd=repo_path) + cmd = 'git submodule update' + execute_command(cmd, cwd=repo_path) + print("Submodule update successful") + except Exception as e: + logging.warning('Error message:%s' % e) + + +def get_package_folder(origin_path, version): + return origin_path + '-' + version + + +def git_cmd_exec(cmd, cwd): + try: + execute_command(cmd, cwd=cwd) + except Exception as e: + logging.warning('Error message:%s%s. %s \n\t' % (cwd.encode("utf-8"), " path doesn't exist", e)) + print("You can solve this problem by manually removing old packages and re-downloading them using env.") + + +def update_latest_packages(sys_value): + """ update the packages that are latest version. + + If the selected package is the latest version, + check to see if it is the latest version after the update command, + if not, then update the latest version from the remote repository. + If the download has a conflict, you are currently using the prompt + message provided by git. + """ + + logging.info("Begin to update latest version packages") + + result = True + + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + + env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config') + + with open(package_filename, 'r') as f: + read_back_pkgs_json = json.load(f) + + for pkg in read_back_pkgs_json: + right_path_flag = True + package = PackageOperation() + pkg_path = pkg['path'] + if pkg_path[0] == '/' or pkg_path[0] == '\\': + pkg_path = pkg_path[1:] + + pkg_path = os.path.join(pkgs_root, pkg_path, 'package.json') + package.parse(pkg_path) + pkgs_name_in_json = package.get_name() + + # Find out the packages which version is 'latest' + if pkg['ver'] == "latest_version" or pkg['ver'] == "latest": + repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json) + repo_path = get_package_folder(repo_path, pkg['ver']) + + # noinspection PyBroadException + try: + # If mirror acceleration is enabled, get the update address from the mirror server. + if need_using_mirror_download(env_config_file): + payload_pkgs_name_in_json = pkgs_name_in_json.encode("utf-8") + + # Change repo's upstream address. + mirror_url = get_url_from_mirror_server( + payload_pkgs_name_in_json, pkg['ver']) + + # if git root is same as repo path, then change the upstream + get_git_root = get_git_root_path(repo_path) + if get_git_root: + if os.path.normcase(repo_path) == os.path.normcase(get_git_root): + if mirror_url[0] is not None: + cmd = 'git remote set-url origin ' + mirror_url[0] + git_cmd_exec(cmd, repo_path) + else: + print("\n==============================> updating") + print("Package path: %s" % repo_path) + print("Git root: %s" % get_git_root) + print("Error: Not currently in a git root directory, cannot switch upstream.\n") + right_path_flag = False + result = False + else: + right_path_flag = False + result = False + + except Exception as e: + logging.warning("Failed to connect to the mirror server, using non-mirror server to update.") + + if not right_path_flag: + continue + + # Update the package repository from upstream. + git_pull_repo(repo_path) + + # If the package has submodules, update the submodules. + update_submodule(repo_path) + + # recover origin url to the path which get from packages.json file + if package.get_url(pkg['ver']): + cmd = 'git remote set-url origin ' + \ + package.get_url(pkg['ver']) + git_cmd_exec(cmd, repo_path) + else: + print("Can't find the package : %s's url in file : %s" % + (payload_pkgs_name_in_json, pkg_path)) + + print("==============================> %s update done\n" % pkgs_name_in_json) + + return result + + +def get_git_root_path(repo_path): + if os.path.isdir(repo_path): + try: + before = os.getcwd() + os.chdir(repo_path) + result = os.popen("git rev-parse --show-toplevel") + result = result.read() + for line in result.splitlines()[:5]: + get_git_root = line + break + os.chdir(before) + return get_git_root + except Exception as e: + logging.warning("Error message : %s" % e) + return None + else: + logging.warning("Missing path {0}".format(repo_path)) + logging.warning("If you manage this package manually, Env tool will not update it.") + return None + + +def pre_package_update(): + """ Make preparations before updating the software package. """ + + logging.info("Begin prepare package update") + bsp_root = Import('bsp_root') + env_root = Import('env_root') + + if not os.path.exists('.config'): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + print("\n\033[1;31;40m当前路径下没有发现 .config 文件，请确保当前目录为 BSP 根目录。\033[0m") + print("\033[1;31;40m如果确定当前目录为 BSP 根目录，请先使用 命令来生成 .config 文件。\033[0m\n") + + print('No system configuration file : .config.') + print('You should use < menuconfig > command to config bsp first.') + + if platform.system() == "Windows": + os.system('chcp 437 > nul') + + return False + + # According to the env version, whether Chinese output is supported or not + if determine_support_chinese(env_root): + if platform.system() == "Windows": + os.system('chcp 65001 > nul') + + # create packages folder + bsp_packages_path = os.path.join(bsp_root, 'packages') + if not os.path.exists(bsp_packages_path): + os.mkdir("packages") + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # prepare target packages file + dbsqlite_pathname = os.path.join(bsp_packages_path, 'packages.dbsqlite') + Export('dbsqlite_pathname') + dbsqlite_pathname = dbsqlite_pathname.encode('utf-8').decode('gbk') + + # avoid creating tables more than one time + if not os.path.isfile(dbsqlite_pathname): + conn = pkgsdb.get_conn(dbsqlite_pathname) + sql = '''CREATE TABLE packagefile + (pathname TEXT ,package TEXT ,md5 TEXT );''' + pkgsdb.create_table(conn, sql) + + fn = '.config' + pkgs = kconfig.parse(fn) + newpkgs = pkgs + + if not os.path.exists(bsp_packages_path): + os.mkdir(bsp_packages_path) + + # regenerate file : packages/pkgs.json + package_json_filename = os.path.join(bsp_packages_path, 'pkgs.json') + if not os.path.exists(package_json_filename): + os.chdir(bsp_packages_path) + fp = open("pkgs.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # Reading data back from pkgs.json + with open(package_json_filename, 'r') as f: + oldpkgs = json.load(f) + + # regenerate file : packages/pkgs_error.json + pkgs_error_list_fn = os.path.join(bsp_packages_path, 'pkgs_error.json') + + if not os.path.exists(pkgs_error_list_fn): + os.chdir(bsp_packages_path) + fp = open("pkgs_error.json", 'w') + fp.write("[]") + fp.close() + os.chdir(bsp_root) + + # read data back from pkgs_error.json + with open(pkgs_error_list_fn, 'r') as f: + package_error = json.load(f) + + # create SConscript file + if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')): + with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f: + f.write(str(Bridge_SConscript)) + + return [oldpkgs, newpkgs, package_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path, + dbsqlite_pathname] + + +def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update): + bsp_root = Import('bsp_root') + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + download_error = [] + flag = True + + if len(error_packages_list): + print("\n==============================> Packages list to download : \n") + for package in error_packages_list: + print("Package name : %s, Ver : %s" % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) + print("\nThe packages in the list above are accidentally deleted or renamed.") + print("\nIf you manually delete the version suffix of the package folder name, ") + print("you can use command to re-download these packages.") + print("In case of accidental deletion, the ENV tool will automatically re-download these packages.") + + # re-download the packages in error_packages_list + for package in error_packages_list: + if install_package(env_root, pkgs_root, bsp_root, package, force_update): + print("\n==============================> %s %s update done \n" + % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) + else: + download_error.append(package) + print(package, 'download failed.') + flag = False + + if len(download_error): + print("%s" % download_error) + + for package in download_error: + print("Packages:%s, %s re-download error, you can use command to re-download them." + % (package['name'].encode("utf-8"), package['ver'].encode("utf-8"))) + + error_write_back = sub_list(read_back_pkgs_json, download_error) + with open(package_filename, 'w') as f: + f.write(json.dumps(error_write_back, indent=1)) + + return flag + + +def rm_package(dir_remove): + logging.info("remove dir: {0}".format(dir_remove)) + + if platform.system() != "Windows": + shutil.rmtree(dir_remove) + else: + dir_remove = '"' + dir_remove + '"' + cmd = 'rd /s /q ' + dir_remove + os.system(cmd) + + if os.path.isdir(dir_remove): + if platform.system() != "Windows": + shutil.rmtree(dir_remove) + else: + dir_remove = '"' + dir_remove + '"' + cmd = 'rmdir /s /q ' + dir_remove + os.system(cmd) + + if os.path.isdir(dir_remove): + print("Folder path: %s" % dir_remove.encode("utf-8")) + return False + else: + print("Path: %s \nSuccess: Folder has been removed. " % dir_remove.encode("utf-8")) + return True + + +def get_package_remove_path(pkg, bsp_packages_path): + dir_path = pkg['path'] + ver = pkg['ver'] + if dir_path[0] == '/' or dir_path[0] == '\\': + dir_path = dir_path[1:] + + if platform.system() == "Windows": + dir_path = os.path.basename(dir_path.replace('/', '\\')) + else: + dir_path = os.path.basename(dir_path) + + # Handles the deletion of git repository folders with version Numbers + remove_path = os.path.join(bsp_packages_path, dir_path) + remove_path_ver = get_package_folder(remove_path, ver) + return remove_path_ver + + +def handle_download_error_packages(sys_value, force_update): + """ handle download error packages. + + Check to see if the packages stored in the Json file list actually exist, + and then download the packages if they don't exist. + """ + + logging.info("begin to handel download error packages") + package_filename = sys_value[3] + bsp_packages_path = sys_value[5] + + with open(package_filename, 'r') as f: + package_json = json.load(f) + + error_packages_list = [] + + for pkg in package_json: + remove_path = get_package_remove_path(pkg, bsp_packages_path) + if os.path.exists(remove_path): + continue + else: + print("Error package : %s" % pkg) + error_packages_list.append(pkg) + + # Handle the failed download packages + get_flag = error_packages_handle(error_packages_list, package_json, package_filename, force_update) + + return get_flag + + +def delete_useless_packages(sys_value): + logging.info("Begin to delete useless packages") + package_delete_error_list = sys_value[2] + bsp_packages_path = sys_value[5] + + # try to delete useless packages, exit command if fails + if len(package_delete_error_list): + for error_package in package_delete_error_list: + remove_path_with_version = get_package_remove_path(error_package, bsp_packages_path) + if os.path.isdir(remove_path_with_version): + print("\nError: %s package delete failed, begin to remove it." % + error_package['name'].encode("utf-8")) + + if not rm_package(remove_path_with_version): + print("Error: Delete package %s failed! Please delete the folder manually.\n" % + error_package['name'].encode("utf-8")) + return False + return True + + +def is_git_package(pkg, bsp_packages_path): + remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) + remove_path_git = os.path.join(remove_path_with_version, '.git') + return os.path.isdir(remove_path_with_version) and os.path.isdir(remove_path_git) + + +def delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list): + git_folder_to_remove = remove_path_with_version + + print("\nStart to remove %s \nplease wait..." % git_folder_to_remove.encode("utf-8")) + if force_update: + logging.info("package force update, Begin to remove package {0}".format(git_folder_to_remove)) + if not rm_package(git_folder_to_remove): + print("Floder delete fail: %s" % git_folder_to_remove.encode("utf-8")) + print("Please delete this folder manually.") + else: + print("The folder is managed by git. Do you want to delete this folder?\n") + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + try: + if not rm_package(git_folder_to_remove): + package_delete_fail_list.append(pkg) + print("Error: Please delete the folder manually.") + except Exception as e: + logging.warning('Error message:%s%s. error.message: %s\n\t' % + ("Delete folder failed: ", git_folder_to_remove.encode("utf-8"), e)) + + +def delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname): + if os.path.isdir(remove_path_with_version): + print("Start to remove %s \nplease wait..." % remove_path_with_version.encode("utf-8")) + try: + pkgsdb.deletepackdir(remove_path_with_version, sqlite_pathname) + except Exception as e: + package_delete_fail_list.append(pkg) + logging.warning('Error message:\n%s %s. %s \n\t' % ( + "Delete folder failed, please delete the folder manually", + remove_path_with_version.encode("utf-8"), e)) + + +def remove_packages(sys_value, force_update): + logging.info("Begin to remove packages") + old_package = sys_value[0] + new_package = sys_value[1] + package_error_list_filename = sys_value[4] + bsp_packages_path = sys_value[5] + sqlite_pathname = sys_value[6] + + case_delete = sub_list(old_package, new_package) + package_delete_fail_list = [] + + for pkg in case_delete: + remove_path_with_version = get_package_remove_path(pkg, bsp_packages_path) + + # delete .git directory + if is_git_package(pkg, bsp_packages_path): + delete_git_package(pkg, remove_path_with_version, force_update, package_delete_fail_list) + else: + delete_zip_package(pkg, remove_path_with_version, package_delete_fail_list, sqlite_pathname) + + # write error messages + with open(package_error_list_filename, 'w') as f: + f.write(str(json.dumps(package_delete_fail_list, indent=1))) + + if len(package_delete_fail_list): + return False + + return True + + +def install_packages(sys_value, force_update): + """ + If the package download fails, record it, + and then download again when the update command is executed. + """ + + logging.info("Begin to install packages") + + old_package = sys_value[0] + new_package = sys_value[1] + + logging.info("old_package {0}".format(old_package)) + logging.info("new_package {0}".format(new_package)) + + package_filename = sys_value[3] + bsp_root = Import('bsp_root') + pkgs_root = Import('pkgs_root') + env_root = Import('env_root') + + case_download = sub_list(new_package, old_package) + packages_download_fail_list = [] + + for package in case_download: + if install_package(env_root, pkgs_root, bsp_root, package, force_update): + print("==============================> %s %s is downloaded successfully. \n" % ( + package['name'], package['ver'])) + else: + # if package download fails, record it in the packages_download_fail_list + packages_download_fail_list.append(package) + print(package, 'download failed.') + return False + + # Get the currently updated configuration. + new_package = sub_list(new_package, packages_download_fail_list) + + # Give hints based on the success of the download. + if len(packages_download_fail_list): + print("\nPackage download failed list:") + for item in packages_download_fail_list: + print(item) + + print("You need to reuse the command to download again.") + + # Update pkgs.json and SConscript + with open(package_filename, 'w') as f: + f.write(str(json.dumps(new_package, indent=1))) + + return True + + +def package_update(force_update=False): + """Update env's packages. + + Compare the old and new software package list and update the package. + Remove unwanted packages and download the newly selected package.- + Check if the files in the deleted packages have been changed, and if so, + remind the user saved the modified file. + """ + + sys_value = pre_package_update() + if not sys_value: + return + + flag = True + + if not delete_useless_packages(sys_value): + return + + # 1.in old and not in new : Software packages that need to be removed + if not remove_packages(sys_value, force_update): + return + + # 2.in new not in old : Software packages to be installed. + if not install_packages(sys_value, force_update): + flag = False + + # 3.handle download error packages. + if not handle_download_error_packages(sys_value, force_update): + flag = False + + # 4.update the software packages, which the version is 'latest' + if not update_latest_packages(sys_value): + flag = False + + if flag: + print("Operation completed successfully.") + else: + print("Operation failed.") diff --git a/cmds/cmd_package/cmd_package_upgrade.py b/cmds/cmd_package/cmd_package_upgrade.py new file mode 100644 index 00000000..fd46a212 --- /dev/null +++ b/cmds/cmd_package/cmd_package_upgrade.py @@ -0,0 +1,104 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +from vars import Import +from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server, find_macro_in_config + + +def upgrade_packages_index(): + """Update the package repository index.""" + + env_root = Import('env_root') + pkgs_root = Import('pkgs_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') + + if get_package_url is not None: + git_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + git_repo = 'https://gitee.com/RT-Thread-Mirror/packages.git' + else: + git_repo = 'https://github.com/RT-Thread/packages.git' + + packages_root = pkgs_root + pkgs_path = os.path.join(packages_root, 'packages') + + if not os.path.isdir(pkgs_path): + cmd = 'git clone ' + git_repo + ' ' + pkgs_path + os.system(cmd) + print("upgrade from :%s" % (git_repo.encode("utf-8"))) + else: + print("Begin to upgrade env packages.") + git_pull_repo(pkgs_path, git_repo) + print("==============================> Env packages upgrade done \n") + + for filename in os.listdir(packages_root): + package_path = os.path.join(packages_root, filename) + if os.path.isdir(package_path): + + if package_path == pkgs_path: + continue + + if os.path.isdir(os.path.join(package_path, '.git')): + print("Begin to upgrade %s." % filename) + git_pull_repo(package_path) + print("==============================> Env %s update done \n" % filename) + + +def upgrade_env_script(): + """Update env function scripts.""" + + print("Begin to upgrade env scripts.") + env_root = Import('env_root') + env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') + env_config_file = os.path.join(env_kconfig_path, '.config') + if (not os.path.isfile(env_config_file)) or \ + (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') + + if get_package_url is not None: + env_scripts_repo = get_package_url + else: + print("Failed to get url from mirror server. Using default url.") + env_scripts_repo = 'https://gitee.com/RT-Thread-Mirror/env.git' + else: + env_scripts_repo = 'https://github.com/RT-Thread/env.git' + + env_scripts_root = os.path.join(env_root, 'tools', 'scripts') + git_pull_repo(env_scripts_root, env_scripts_repo) + print("==============================> Env scripts upgrade done \n") + + +def package_upgrade(): + """Update the package repository directory and env function scripts.""" + + upgrade_packages_index() + upgrade_env_script() diff --git a/cmds/cmd_package/cmd_package_utils.py b/cmds/cmd_package/cmd_package_utils.py new file mode 100644 index 00000000..0ae95bcd --- /dev/null +++ b/cmds/cmd_package/cmd_package_utils.py @@ -0,0 +1,202 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import json +import os +import platform +import subprocess +import sys +import time +import shutil +import requests +import logging + + +def execute_command(cmd_string, cwd=None, shell=True): + """Execute the system command at the specified address.""" + + sub = subprocess.Popen(cmd_string, cwd=cwd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, shell=shell, bufsize=4096) + + stdout_str = '' + while sub.poll() is None: + stdout_str += str(sub.stdout.read()) + time.sleep(0.1) + + return stdout_str + + +def git_pull_repo(repo_path, repo_url=''): + try: + if platform.system() == "Windows": + cmd = r'git config --local core.autocrlf true' + execute_command(cmd, cwd=repo_path) + cmd = r'git pull ' + repo_url + execute_command(cmd, cwd=repo_path) + except Exception as e: + print('Error message:%s' % e) + + +def get_url_from_mirror_server(package_name, package_version): + """Get the download address from the mirror server based on the package name.""" + + try: + if isinstance(package_name, str): + if sys.version_info < (3, 0): + package_name = str(package_name) + else: + package_name = str(package_name)[2:-1] + except Exception as e: + print('Error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None + + payload = { + "userName": "RT-Thread", + "packages": [ + { + "name": "NULL", + } + ] + } + payload["packages"][0]['name'] = package_name + + try: + r = requests.post("http://packages.rt-thread.org/packages/queries", data=json.dumps(payload)) + + if r.status_code == requests.codes.ok: + package_info = json.loads(r.text) + + # Can't find package,change git package SHA if it's a git + # package + if len(package_info['packages']) == 0: + print("Package was NOT found on mirror server. Using a non-mirrored address to download.") + return None, None + else: + for item in package_info['packages'][0]['packages_info']['site']: + if item['version'] == package_version: + # Change download url + download_url = item['URL'] + if download_url[-4:] == '.git': + # Change git package SHA + repo_sha = item['VER_SHA'] + return download_url, repo_sha + return download_url, None + + print("\nTips : \nThe system needs to be upgraded.") + print("Please use the command to upgrade packages index.\n") + return None, None + + except Exception as e: + print('Error message:%s' % e) + print("\nThe mirror server could not be contacted. Please check your network connection.") + return None, None + + +def user_input(msg=None): + """Gets the union keyboard input.""" + + if sys.version_info < (3, 0): + if msg is not None: + value = raw_input(msg) + else: + value = raw_input() + else: + if msg is not None: + value = input(msg) + else: + value = input() + + return value + + +def find_macro_in_config(filename, macro_name): + try: + config = open(filename, "r") + except Exception as e: + print('Error message:%s' % e) + print('open .config failed') + return + + empty_line = 1 + + for line in config: + line = line.lstrip(' ').replace('\n', '').replace('\r', '') + + if len(line) == 0: + continue + + if line[0] == '#': + if len(line) == 1: + if empty_line: + continue + + empty_line = 1 + continue + + # comment_line = line[1:] + if line.startswith('# CONFIG_'): + line = ' ' + line[9:] + else: + line = line[1:] + + # print line + + empty_line = 0 + else: + empty_line = 0 + setting = line.split('=') + if len(setting) >= 2: + if setting[0].startswith('CONFIG_'): + setting[0] = setting[0][7:] + + if setting[0] == macro_name and setting[1] == 'y': + return True + + config.close() + return False + + +def is_windows(): + if platform.system() == "Windows": + return True + else: + return False + + +def remove_folder(folder_path): + try: + if os.path.isdir(folder_path): + if is_windows(): + cmd = 'rd /s /q ' + folder_path + os.system(cmd) + else: + shutil.rmtree(folder_path) + return True + else: + return True + except Exception as e: + logging.warning('Error message : {0}'.format(e)) + return False diff --git a/cmds/cmd_package/cmd_package_wizard.py b/cmds/cmd_package/cmd_package_wizard.py new file mode 100644 index 00000000..41b6a8e8 --- /dev/null +++ b/cmds/cmd_package/cmd_package_wizard.py @@ -0,0 +1,155 @@ +# -*- coding:utf-8 -*- +# +# File : cmd_package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2020, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2020-04-08 SummerGift Optimize program structure +# + +import os +import re +from package import Kconfig_file, Package_json_file +from string import Template +from .cmd_package_utils import user_input + + +def package_wizard(): + """Packages creation wizard. + + The user enters the package name, version number, category, and automatically generates the package index file. + """ + + # Welcome + print('\033[4;32;40mWelcome to using package wizard, please follow below steps.\033[0m\n') + + # Simple introduction about the wizard + print('note :') + print(' \033[5;35;40m[ ]\033[0m means default setting or optional information.') + print(' \033[5;35;40mEnter\033[0m means using default option or ending and proceeding to the next step.') + + # first step + print('\033[5;33;40m\n1.Please input a new package name :\033[0m') + + name = user_input() + regular_obj = re.compile('\W') + while name == '' or name.isspace() == True or regular_obj.search(name.strip()): + if name == '' or name.isspace(): + print('\033[1;31;40mError: you must input a package name. Try again.\033[0m') + name = user_input() + else: + print('\033[1;31;40mError: package name is made of alphabet, number and underline. Try again.\033[0m') + name = user_input() + + default_description = 'Please add description of ' + name + ' in English.' + description = default_description + description_zh = "请添加软件包 " + name + " 的中文描述。" + + # second step + print("\033[5;33;40m\n2.Please input this package version, default : '1.0.0' \033[0m") + ver = user_input() + if ver == '': + print("using default version 1.0.0") + ver = '1.0.0' + + ver_standard = ver.replace('.', '') + keyword = name + + # third step + package_class_list = ('iot', 'language', 'misc', 'multimedia', + 'peripherals', 'security', 'system', 'tools', 'peripherals/sensors') + print('\033[5;33;40m\n3.Please choose a package category from 1 to 9 : \033[0m') + print("\033[1;32;40m[1:iot]|[2:language]|[3:misc]|[4:multimedia]|" + "[5:peripherals]|[6:security]|[7:system]|[8:tools]|[9:sensors]\033[0m") + + class_number = user_input() + while class_number == '' or class_number.isdigit() is False or int(class_number) < 1 or int(class_number) > 9: + if class_number == '': + print('\033[1;31;40mError: You must choose a package category. Try again.\033[0m') + else: + print('\033[1;31;40mError: You must input an integer number from 1 to 9. Try again.\033[0m') + class_number = user_input() + + package_class = package_class_list[int(class_number) - 1] + + # fourth step + print("\033[5;33;40m\n4.Please input author's github ID of this package :\033[0m") + + author_name = user_input() + while author_name == '': + print("\033[1;31;40mError: you must input author's github ID of this package. Try again.\033[0m") + author_name = user_input() + + # fifth step + author_email = user_input('\033[5;33;40m\n5.Please input author email of this package :\n\033[0m') + while author_email == '': + print('\033[1;31;40mError: you must input author email of this package. Try again.\033[0m') + author_email = user_input() + + # sixth step + print('\033[5;33;40m\n6.Please choose a license of this package from 1 to 4, or input other license name :\033[0m') + print("\033[1;32;40m[1:Apache-2.0]|[2:MIT]|[3:LGPL-2.1]|[4:GPL-2.0]\033[0m") + license_index = ('Apache-2.0', 'MIT', 'LGPL-2.1', 'GPL-2.0') + license_class = user_input() + while license_class == '': + print('\033[1;31;40mError: you must choose or input a license of this package. Try again.\033[0m') + license_class = user_input() + + if license_class.isdigit() and 1 <= int(license_class) <= 4: + license_choice = license_index[int(license_class) - 1] + else: + license_choice = license_class + + # seventh step + print('\033[5;33;40m\n7.Please input the repository of this package :\033[0m') + print( + "\033[1;32;40mFor example, hello package's repository url " + "is 'https://github.com/RT-Thread-packages/hello'.\033[0m") + + repository = user_input() + while repository == '': + print('\033[1;31;40mError: you must input a repository of this package. Try again.\033[0m') + repository = user_input() + + pkg_path = name + if not os.path.exists(pkg_path): + os.mkdir(pkg_path) + else: + print("\033[1;31;40mError: the package directory is exits!\033[0m") + + s = Template(Kconfig_file) + upper_name = str.upper(name) + kconfig = s.substitute(name=upper_name, description=description, version=ver, + pkgs_class=package_class, lowercase_name=name, version_standard=ver_standard) + f = open(os.path.join(pkg_path, 'Kconfig'), 'w') + f.write(kconfig) + f.close() + + s = Template(Package_json_file) + package = s.substitute(name=name, pkgsclass=package_class, authorname=author_name, authoremail=author_email, + description=description, description_zh=description_zh, version=ver, keyword=keyword, + license=license_choice, repository=repository, pkgs_using_name=upper_name) + f = open(os.path.join(pkg_path, 'package.json'), 'w') + f.write(package) + f.close() + + print('\nThe package index has been created \033[1;32;40msuccessfully\033[0m.') + print('Please \033[5;34;40mupdate\033[0m other information of this package ' + 'based on Kconfig and package.json in directory ' + name + '.') + diff --git a/cmds/cmd_system.py b/cmds/cmd_system.py index 4b116202..4f682744 100644 --- a/cmds/cmd_system.py +++ b/cmds/cmd_system.py @@ -35,14 +35,11 @@ def cmd(args): if args.system_update: dir_list = os.listdir(packages_root) - kconfig = file(os.path.join(packages_root, 'Kconfig'), 'w') - - for item in dir_list: - if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): - kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') - kconfig.write('\n') - - kconfig.close() + with open(os.path.join(packages_root, 'Kconfig'), 'w') as kconfig: + for item in dir_list: + if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): + kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') + kconfig.write('\n') def add_parser(sub): diff --git a/env.py b/env.py index 18a4efa7..4a55b309 100644 --- a/env.py +++ b/env.py @@ -22,25 +22,26 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2019-1-16 SummerGift Add chinese detection +# 2020-4-13 SummerGift refactoring # import os import sys import argparse +import logging import platform from cmds import * from vars import Export -__version__ = 'rt-thread packages v1.1.0' +__version__ = 'rt-thread packages v1.2.0' def init_argparse(): parser = argparse.ArgumentParser(description=__doc__) subs = parser.add_subparsers() - parser.add_argument('-v', '--version', - action='version', version=__version__) + parser.add_argument('-v', '--version', action='version', version=__version__) cmd_system.add_parser(subs) cmd_menuconfig.add_parser(subs) @@ -49,41 +50,72 @@ def init_argparse(): return parser -def main(): - bsp_root = os.getcwd() - script_root = os.path.split(os.path.realpath(__file__))[0] +def init_logger(env_root): + log_format = "%(module)s %(lineno)d %(levelname)s %(message)s \n" + date_format = '%Y-%m-%d %H:%M:%S %a ' + logging.basicConfig(level=logging.WARNING, + format=log_format, + datefmt=date_format, + # filename=log_name + ) + + +def get_env_root(): env_root = os.getenv("ENV_ROOT") - if env_root == None: + if env_root is None: if platform.system() != 'Windows': env_root = os.path.join(os.getenv('HOME'), '.env') else: env_root = os.path.join(os.getenv('USERPROFILE'), '.env') - sys.path = sys.path + [os.path.join(script_root)] - - pkgs_root = os.getenv("PKGS_ROOT") - if pkgs_root == None: - pkgs_root = os.path.join(env_root, 'packages') + return env_root - Export('env_root') - Export('bsp_root') - Export('pkgs_root') +def get_package_root(env_root): + package_root = os.getenv("PKGS_ROOT") + if package_root is None: + package_root = os.path.join(env_root, 'packages') + return package_root + + +def get_bsp_root(): + bsp_root = os.getcwd() + + # noinspection PyBroadException try: bsp_root.encode('utf-8').decode("ascii") except Exception as e: if platform.system() == "Windows": os.system('chcp 65001 > nul') - print ("\n\033[1;31;40m警告：\033[0m") - print ("\033[1;31;40m当前路径不支持非英文字符，请修改当前路径为纯英文路径。\033[0m") - print ("\033[1;31;40mThe current path does not support non-English characters.\033[0m") - print ("\033[1;31;40mPlease modify the current path to a pure English path.\033[0m") + print("\n\033[1;31;40m警告：\033[0m") + print("\033[1;31;40m当前路径不支持非英文字符，请修改当前路径为纯英文路径。\033[0m") + print("\033[1;31;40mThe current path does not support non-English characters.\033[0m") + print("\033[1;31;40mPlease modify the current path to a pure English path.\033[0m") if platform.system() == "Windows": os.system('chcp 437 > nul') - return False + exit(1) + + return bsp_root + + +def export_environment_variable(): + script_root = os.path.split(os.path.realpath(__file__))[0] + sys.path = sys.path + [os.path.join(script_root)] + bsp_root = get_bsp_root() + env_root = get_env_root() + pkgs_root = get_package_root(env_root) + + Export('env_root') + Export('bsp_root') + Export('pkgs_root') + + +def main(): + export_environment_variable() + init_logger(get_env_root()) parser = init_argparse() args = parser.parse_args() @@ -92,4 +124,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/init_env.py b/init_env.py index 500fb0d9..0c024212 100644 --- a/init_env.py +++ b/init_env.py @@ -25,24 +25,27 @@ from multiprocessing import Process import os -import sys + def run_proc(name, env_root): - exec_file = os.path.join(env_root, "tools\scripts\env.py") + exec_file = os.path.join(env_root, r"tools\scripts\env.py") log_std = os.path.join(env_root, "env_log_std") log_err = os.path.join(env_root, "env_log_err") + # noinspection PyBroadException try: - os.system("python %s package --upgrade 1>%s 2>%s"%(exec_file, log_std, log_err)) + os.system("python %s package --upgrade 1>%s 2>%s" % (exec_file, log_std, log_err)) except Exception as e: print("Auto upgrade failed, please check your network.") pass + def main(): env_root = env_root = os.getenv("ENV_ROOT") p = Process(target=run_proc, args=('upgrade', env_root)) p.start() p.join() -if __name__=='__main__': + +if __name__ == '__main__': main() diff --git a/kconfig.py b/kconfig.py index 0e904203..7c7022b3 100644 --- a/kconfig.py +++ b/kconfig.py @@ -1,4 +1,4 @@ - # -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- # # File : kconfig.py # This file is part of RT-Thread RTOS @@ -50,10 +50,12 @@ def pkgs_ver(pkgs, name, ver): def parse(filename): ret = [] + + # noinspection PyBroadException try: - config = open(filename, "r") - except: - print('open .config failed') + config = open(filename, "r") + except Exception as e: + print('open .config failed') return ret for line in config: diff --git a/package.py b/package.py index 99e83383..fded16ef 100644 --- a/package.py +++ b/package.py @@ -22,14 +22,18 @@ # Date Author Notes # 2018-5-28 SummerGift Add copyright information # 2018-12-28 Ernest Chen Add package information and enjoy package maker +# 2020-4-7 SummerGift Code improvement # -import os import json -import archive +import logging +import os import sys + import requests +import archive + """Template for creating a new file""" Bridge_SConscript = '''import os @@ -128,7 +132,8 @@ Return('group') ''' -class Package: + +class PackageOperation: pkg = None def parse(self, filename): @@ -149,11 +154,15 @@ def get_filename(self, ver): return None def get_url(self, ver): + url = None for item in self.pkg['site']: if item['version'].lower() == ver.lower(): - return item['URL'] + url = item['URL'] - return None + if not url: + logging.warning("Can't find right url {0}, please check {1}".format(ver.lower(), self.pkg['site'])) + + return url def get_versha(self, ver): for item in self.pkg['site']: @@ -184,13 +193,13 @@ def download(self, ver, path, url_from_srv): if not os.path.getsize(path): os.remove(path) else: - if archive.packtest(path): - #print "The file is rigit." + if archive.package_integrity_test(path): + # print "The file is rigit." return True else: os.remove(path) - retryCount = 0 + retry_count = 0 headers = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate', @@ -200,7 +209,7 @@ def download(self, ver, path, url_from_srv): print('Start to download package : %s ' % filename.encode("utf-8")) while True: - #print("retryCount : %d"%retryCount) + # print("retry_count : %d"%retry_count) try: r = requests.get(url_from_srv, stream=True, headers=headers) @@ -215,9 +224,9 @@ def download(self, ver, path, url_from_srv): sys.stdout.write("\rDownloding %d KB" % flush_count) sys.stdout.flush() - retryCount = retryCount + 1 + retry_count = retry_count + 1 - if archive.packtest(path): # make sure the file is right + if archive.package_integrity_test(path): # make sure the file is right ret = True print("\rDownloded %d KB " % flush_count) print('Start to unpack. Please wait...') @@ -225,7 +234,7 @@ def download(self, ver, path, url_from_srv): else: if os.path.isfile(path): os.remove(path) - if retryCount > 5: + if retry_count > 5: print( "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) if os.path.isfile(path): @@ -233,23 +242,14 @@ def download(self, ver, path, url_from_srv): ret = False break except Exception as e: - print(url_from_srv) - print('error message:%s\t' %e) - retryCount = retryCount + 1 - if retryCount > 5: + print(url_from_srv) + print('error message:%s\t' % e) + retry_count = retry_count + 1 + if retry_count > 5: print('%s download fail!\n' % path.encode("utf-8")) if os.path.isfile(path): os.remove(path) return False return ret - def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): - try: - # ignore the return value - archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) - return True - except Exception as e: - print('unpack error message :%s' % e) - print('unpack %s failed' % os.path.basename(fullpkg_path)) - os.remove(fullpkg_path) - return False + diff --git a/pkgsdb.py b/pkgsdb.py index 488008cb..a6db58d1 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -21,38 +21,37 @@ # Change Logs: # Date Author Notes # 2018-5-28 SummerGift Add copyright information +# 2020-4-10 SummerGift Code clear up # import sqlite3 import os import hashlib -import sys - +from cmds.cmd_package.cmd_package_utils import user_input from vars import Import + SHOW_SQL = False -def GetFileMd5(filename): +def get_file_md5(filename): if not os.path.isfile(filename): return - myhash = hashlib.md5() + hash_value = hashlib.md5() f = open(filename, 'rb') while True: b = f.read(8096) if not b: break - myhash.update(b) + hash_value.update(b) f.close() - return myhash.hexdigest() + return hash_value.hexdigest() def get_conn(path): conn = sqlite3.connect(path) if os.path.exists(path) and os.path.isfile(path): - #print('on disk:[{}]'.format(path)) return conn else: - conn = None print('on memory:[:memory:]') return sqlite3.connect(':memory:') @@ -73,17 +72,16 @@ def create_table(conn, sql): if sql is not None and sql != '': cu = get_cursor(conn) if SHOW_SQL: - print('执行sql:[{}]'.format(sql)) + print('execute :[{}]'.format(sql)) cu.execute(sql) conn.commit() - #print('create data table successful!') close_all(conn) else: print('the [{}] is empty or equal None!'.format(sql)) def save(conn, sql, data): - '''insert data to database''' + """insert data to database""" if sql is not None and sql != '': if data is not None: cu = get_cursor(conn) @@ -99,94 +97,82 @@ def save(conn, sql, data): def isdataexist(pathname): ret = True - dbpathname = Import('dbsqlite_pathname') + dbfilename = Import('dbsqlite_pathname') - conn = get_conn(dbpathname) + conn = get_conn(dbfilename) c = get_cursor(conn) sql = 'SELECT md5 from packagefile where pathname = "' + pathname + '"' cursor = c.execute(sql) for row in cursor: dbmd5 = row[0] + if dbmd5: ret = False conn.close() return ret -#将数据添加到数据库，如果数据库中已经存在则不重复添加 -def savetodb(pathname, pkgspathname, before_change_name): - dbpathname = Import('dbsqlite_pathname') +# 将数据添加到数据库，如果数据库中已经存在则不重复添加 +def save_to_database(pathname, package_pathname, before_change_name): + db_pathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') - bsppkgs = os.path.join(bsp_root, 'packages') + bsp_packages_path = os.path.join(bsp_root, 'packages') - conn = get_conn(dbpathname) + conn = get_conn(db_pathname) save_sql = '''insert into packagefile values (?, ?, ?)''' - package = os.path.basename(pkgspathname) - md5pathname = os.path.join(bsppkgs, before_change_name) - #print("pathname to save : %s"%pathname) - #print("md5pathname : %s"%md5pathname) - + package = os.path.basename(package_pathname) + md5pathname = os.path.join(bsp_packages_path, before_change_name) + if not os.path.isfile(md5pathname): print("md5pathname is Invalid") - - md5 = GetFileMd5(md5pathname) - #print("md5 to save : %s"%md5) + + md5 = get_file_md5(md5pathname) data = [(pathname, package, md5)] save(conn, save_sql, data) -def dbdump(dbpathname): - conn = get_conn(dbpathname) +def dbdump(dbfilename): + conn = get_conn(dbfilename) c = get_cursor(conn) cursor = c.execute("SELECT pathname, package, md5 from packagefile") for row in cursor: - print("pathname = ", row[0]) - print("package = ", row[1]) - print("md5 = ", row[2], "\n") + print("pathname = ", row[0]) + print("package = ", row[1]) + print("md5 = ", row[2], "\n") conn.close() -#delete the unchanged file - -def remove_unchangedfile(pathname, dbpathname, dbsqlname): +def remove_unchanged_file(pathname, dbfilename, dbsqlname): + """delete unchanged files""" flag = True - conn = get_conn(dbpathname) + conn = get_conn(dbfilename) c = get_cursor(conn) - - #print('pathname : %s'%pathname) - #print('dbsqlname : %s'%dbsqlname) - - filemd5 = GetFileMd5(pathname) - #print("filemd5 : %s"%filemd5) + filemd5 = get_file_md5(pathname) dbmd5 = 0 sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' - #print sql + # print sql cursor = c.execute(sql) for row in cursor: - dbmd5 = row[0] # fetch md5 from databas - #print("dbmd5 : %s"%dbmd5) + # fetch md5 from database + dbmd5 = row[0] if dbmd5 == filemd5: # delete file info from database sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - cursor = c.execute(sql) conn.commit() os.remove(pathname) else: - print ("%s has been changed." % pathname) - print ('Are you sure you want to permanently delete the file: %s?' % - os.path.basename(pathname)) - print ('If you choose to keep the changed file,you should copy the file to another folder. \nbecaues it may be covered by the next update.') + print("%s has been changed." % pathname) + print('Are you sure you want to permanently delete the file: %s?' % + os.path.basename(pathname)) + print('If you choose to keep the changed file,you should copy the file to another folder. ' + '\nbecaues it may be covered by the next update.') - if sys.version_info < (3, 0): - rc = raw_inuput('Press the Y Key to delete the file or just press Enter to keep the file.') - else: - rc = input('Press the Y Key to delete the file or just press Enter to keep the file.') + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') if rc == 'y' or rc == 'Y': sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - cursor = c.execute(sql) conn.commit() os.remove(pathname) print("%s has been removed.\n" % pathname) @@ -196,12 +182,12 @@ def remove_unchangedfile(pathname, dbpathname, dbsqlname): return flag -#删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 -#如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false +# 删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 +# 如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false def deletepackdir(dirpath, dbpathname): flag = getdirdisplay(dirpath, dbpathname) - if flag == True: + if flag: if os.path.exists(dirpath): for root, dirs, files in os.walk(dirpath, topdown=False): for name in files: @@ -209,11 +195,11 @@ def deletepackdir(dirpath, dbpathname): for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(dirpath) - #print "the dir should be delete" + # print "the dir should be delete" return flag -#遍历filepath下所有文件，包括子目录 +# 遍历filepath下所有文件，包括子目录 def displaydir(filepath, basepath, length, dbpathname): flag = True if os.path.isdir(filepath): @@ -225,8 +211,7 @@ def displaydir(filepath, basepath, length, dbpathname): else: pathname = os.path.join(filepath, fi_d) dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] - #print("dbsqlname : %s"%dbsqlname) - if not remove_unchangedfile(pathname, dbpathname, dbsqlname): + if not remove_unchanged_file(pathname, dbpathname, dbsqlname): flag = False return flag @@ -235,6 +220,5 @@ def getdirdisplay(filepath, dbpathname): display = filepath length = len(display) basepath = os.path.basename(filepath) - #print "basepath:",basepath flag = displaydir(filepath, basepath, length, dbpathname) return flag diff --git a/vars.py b/vars.py index 6d443b01..b70c3dbc 100644 --- a/vars.py +++ b/vars.py @@ -27,9 +27,11 @@ env_vars = {} + def Export(var): f = sys._getframe(1).f_locals env_vars[var] = f[var] + def Import(var): return env_vars[var]