[update] refactoring and add some useful feature for user package management

archive.py

@@ -21,87 +21,137 @@
 # Change Logs:
 # Date           Author          Notes
 # 2018-5-28      SummerGift      Add copyright information
+# 2020-4-10      SummerGift      Code clear up
 #
 
+import logging
+import os
+import shutil
 import tarfile
 import zipfile
-import os
 import pkgsdb
-import platform
-import shutil
-
+from cmds.cmd_package.cmd_package_utils import is_windows, remove_folder
 
-def unpack(archive_fn, path, pkg, pkgs_name_in_json):
-    pkg_ver = pkg['ver']
-    flag = True
 
-    iswindows = False
-
-    if platform.system() == "Windows":
-        iswindows = True
-
-    if ".tar.bz2" in archive_fn:
-        arch = tarfile.open(archive_fn, "r:bz2")
+def unpack(archive_filename, bsp_package_path, package_info, package_name):
+    if ".tar.bz2" in archive_filename:
+        arch = tarfile.open(archive_filename, "r:bz2")
         for tarinfo in arch:
-            arch.extract(tarinfo, path)
+            arch.extract(tarinfo, bsp_package_path)
             a = tarinfo.name
-            if not os.path.isdir(os.path.join(path, a)):
-                if iswindows:
+            if not os.path.isdir(os.path.join(bsp_package_path, a)):
+                if is_windows():
                     right_path = a.replace('/', '\\')
                 else:
                     right_path = a
                 a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1])
-                pkgsdb.savetodb(a, archive_fn)
+
+                pkgsdb.save_to_database(a, archive_filename)
         arch.close()
 
-    if ".tar.gz" in archive_fn:
-        arch = tarfile.open(archive_fn, "r:gz")
+    if ".tar.gz" in archive_filename:
+        arch = tarfile.open(archive_filename, "r:gz")
         for tarinfo in arch:
-            arch.extract(tarinfo, path)
+            arch.extract(tarinfo, bsp_package_path)
             a = tarinfo.name
-            if not os.path.isdir(os.path.join(path, a)):
-                if iswindows:
+            if not os.path.isdir(os.path.join(bsp_package_path, a)):
+                if is_windows():
                     right_path = a.replace('/', '\\')
                 else:
                     right_path = a
                 a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1])
-                pkgsdb.savetodb(a, archive_fn)
+                pkgsdb.save_to_database(a, archive_filename)
         arch.close()
 
-    if ".zip" in archive_fn:
-        arch = zipfile.ZipFile(archive_fn, "r")
+    if ".zip" in archive_filename:
+        if not handle_zip_package(archive_filename, bsp_package_path, package_name, package_info):
+            return False
+
+    return True
+
+
+def handle_zip_package(archive_filename, bsp_package_path, package_name, package_info):
+    package_version = package_info['ver']
+    package_temp_path = os.path.join(bsp_package_path, "package_temp")
+
+    try:
+        if remove_folder(package_temp_path):
+            os.makedirs(package_temp_path)
+    except Exception as e:
+        logging.warning('Error message : {0}'.format(e))
+
+    logging.info("BSP packages path {0}".format(bsp_package_path))
+    logging.info("BSP package temp path: {0}".format(package_temp_path))
+    logging.info("archive filename : {0}".format(archive_filename))
+
+    try:
+        flag = True
+        package_folder_name = ""
+        package_name_with_version = ""
+        arch = zipfile.ZipFile(archive_filename, "r")
         for item in arch.namelist():
-            arch.extract(item, path)
-            if not os.path.isdir(os.path.join(path, item)):
-                if iswindows:
+            arch.extract(item, package_temp_path)
+            if not os.path.isdir(os.path.join(package_temp_path, item)):
+                if is_windows():
                     right_path = item.replace('/', '\\')
                 else:
                     right_path = item
 
-                # Gets the folder name and change_dirname only once
+                # Gets the folder name and changed folder name only once
                 if flag:
-                    dir_name = os.path.split(right_path)[0]
-                    change_dirname = pkgs_name_in_json + '-' + pkg_ver
+                    package_folder_name = os.path.split(right_path)[0]
+                    package_name_with_version = package_name + '-' + package_version
                     flag = False
-
-                right_name_to_db = right_path.replace(dir_name, change_dirname, 1)
-                pkgsdb.savetodb(right_name_to_db, archive_fn, right_path)
-        arch.close()
-
-    # Change the folder name
-    change_dirname = pkgs_name_in_json + '-' + pkg_ver
-
-    if os.path.isdir(os.path.join(path, change_dirname)):
-        if iswindows:
-            cmd = 'rd /s /q ' + os.path.join(path, change_dirname)
-            os.system(cmd)
-        else:
-            shutil.rmtree(os.path.join(path, change_dirname))
-
-    os.rename(os.path.join(path, dir_name),os.path.join(path, change_dirname)) 
 
+                right_name_to_db = right_path.replace(package_folder_name, package_name_with_version, 1)
+                right_path = os.path.join("package_temp", right_path)
+                pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path)
+        arch.close()
 
-def packtest(path):
+        if not move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version,
+                                            bsp_package_path):
+            return False
+    except Exception as e:
+        logging.warning('unpack error message : {0}'.format(e))
+        logging.warning('unpack {0} failed'.format(os.path.basename(archive_filename)))
+        # remove temp folder and archive file
+        remove_folder(package_temp_path)
+        os.remove(archive_filename)
+        return False
+
+    return True
+
+
+def move_package_to_bsp_packages(package_folder_name, package_name, package_temp_path, package_version,
+                                 bsp_packages_path):
+    """move package in temp folder to bsp packages folder."""
+    origin_package_folder_path = os.path.join(package_temp_path, package_folder_name)
+    package_name_with_version = package_name + '-' + package_version
+    package_folder_in_temp = os.path.join(package_temp_path, package_name_with_version)
+    bsp_package_path = os.path.join(bsp_packages_path, package_name_with_version)
+    logging.info("origin name: {0}".format(origin_package_folder_path))
+    logging.info("rename name: {0}".format(package_folder_in_temp))
+
+    result = True
+    try:
+        # rename package folder name to package name with version
+        os.rename(origin_package_folder_path, package_folder_in_temp)
+
+        # if there is no specified version package in the bsp package path,
+        # then move package from package_folder_in_temp to bsp_package_path
+        if not os.path.isdir(bsp_package_path):
+            shutil.move(package_folder_in_temp, bsp_package_path)
+    except Exception as e:
+        logging.warning('{0}'.format(e))
+        result = False
+    finally:
+        # must remove temp folder
+        remove_folder(package_temp_path)
+
+    return result
+
+
+def package_integrity_test(path):
     ret = True
 
     if path.find(".zip") != -1:
@@ -116,7 +166,7 @@ def packtest(path):
                 ret = False
                 print('package check error. \n')
         except Exception as e:
-            print('packtest error message:%s\t' % e)
+            print('Package test error message:%s\t' % e)
             print("The archive package is broken. \n")
             arch.close()
             ret = False
@@ -127,6 +177,7 @@ def packtest(path):
             if not tarfile.is_tarfile(path):
                 ret = False
         except Exception as e:
+            print('Error message:%s' % e)
             ret = False
 
     # if ".tar.gz" in path:
@@ -135,6 +186,7 @@ def packtest(path):
             if not tarfile.is_tarfile(path):
                 ret = False
         except Exception as e:
+            print('Error message:%s' % e)
             ret = False
 
     return ret

cmds/__init__.py

@@ -24,3 +24,15 @@
 #
 
 __all__ = ['cmd_package', 'cmd_system', 'cmd_menuconfig']
+
+try:
+    import requests
+except ImportError:
+    print("****************************************\n"
+          "* Import requests module error.\n"
+          "* Please install requests module first.\n"
+          "* pip install step:\n"
+          "* $ pip install requests\n"
+          "* command install step:\n"
+          "* $ sudo apt-get install python-requests\n"
+          "****************************************\n")

cmds/cmd_menuconfig.py

@@ -28,13 +28,14 @@
 import os
 import platform
 import re
-from vars import Import, Export
+from vars import Import
+from .cmd_package.cmd_package_utils import find_macro_in_config
 
 
 def is_pkg_special_config(config_str):
-    ''' judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER'''
+    """judge if it's CONFIG_PKG_XX_PATH or CONFIG_PKG_XX_VER"""
 
-    if type(config_str) == type('a'):
+    if isinstance(config_str, str):
         if config_str.startswith("PKG_") and (config_str.endswith('_PATH') or config_str.endswith('_VER')):
             return True
     return False
@@ -43,7 +44,8 @@ def is_pkg_special_config(config_str):
 def mk_rtconfig(filename):
     try:
         config = open(filename, 'r')
-    except:
+    except Exception as e:
+        print('Error message:%s' % e)
         print('open config:%s failed' % filename)
         return
 
@@ -89,7 +91,7 @@ def mk_rtconfig(filename):
                 if setting[1] == 'y':
                     rtconfig.write('#define %s\n' % setting[0])
                 else:
-                    rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$",line)[0]))
+                    rtconfig.write('#define %s %s\n' % (setting[0], re.findall(r"^.*?=(.*)$", line)[0]))
 
     if os.path.isfile('rtconfig_project.h'):
         rtconfig.write('#include "rtconfig_project.h"\n')
@@ -99,54 +101,7 @@ def mk_rtconfig(filename):
     rtconfig.close()
 
 
-def find_macro_in_config(filename, macro_name):
-    try:
-        config = open(filename, "r")
-    except:
-        print('open .config failed')
-        return
-
-    empty_line = 1
-
-    for line in config:
-        line = line.lstrip(' ').replace('\n', '').replace('\r', '')
-
-        if len(line) == 0:
-            continue
-
-        if line[0] == '#':
-            if len(line) == 1:
-                if empty_line:
-                    continue
-
-                empty_line = 1
-                continue
-
-            #comment_line = line[1:]
-            if line.startswith('# CONFIG_'):
-                line = ' ' + line[9:]
-            else:
-                line = line[1:]
-
-            # print line
-
-            empty_line = 0
-        else:
-            empty_line = 0
-            setting = line.split('=')
-            if len(setting) >= 2:
-                if setting[0].startswith('CONFIG_'):
-                    setting[0] = setting[0][7:]
-
-                    if setting[0] == macro_name and setting[1] == 'y':
-                        return True
-
-    config.close()
-    return False
-
-
 def cmd(args):
-
     env_root = Import('env_root')
     os_version = platform.platform(True)[10:13]
     kconfig_win7_path = os.path.join(
@@ -159,9 +114,9 @@ def cmd(args):
         print("\n\033[1;31;40m<menuconfig> 命令应当在某一特定 BSP 目录下执行，例如：\"rt-thread/bsp/stm32/stm32f091-st-nucleo\"\033[0m")
         print("\033[1;31;40m请确保当前目录为 BSP 根目录，并且该目录中有 Kconfig 文件。\033[0m\n")
 
-        print ("<menuconfig> command should be used in a bsp root path with a Kconfig file.")
-        print ("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"")
-        print ("You should check if there is a Kconfig file in your bsp root first.")
+        print("<menuconfig> command should be used in a bsp root path with a Kconfig file.")
+        print("Example: \"rt-thread/bsp/stm32/stm32f091-st-nucleo\"")
+        print("You should check if there is a Kconfig file in your bsp root first.")
 
         if platform.system() == "Windows":
             os.system('chcp 437  > nul')
@@ -195,7 +150,7 @@ def cmd(args):
                 os.system('kconfig-mconf Kconfig -n')
 
     elif args.menuconfig_setting:
-        env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds')
+        env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds')
         beforepath = os.getcwd()
         os.chdir(env_kconfig_path)
 
@@ -229,7 +184,7 @@ def cmd(args):
         mk_rtconfig(fn)
 
     if platform.system() == "Windows":
-        env_kconfig_path = os.path.join(env_root, 'tools\scripts\cmds')
+        env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds')
         fn = os.path.join(env_kconfig_path, '.config')
 
         if not os.path.isfile(fn):
@@ -242,13 +197,13 @@ def cmd(args):
         if find_macro_in_config(fn, 'SYS_CREATE_MDK_IAR_PROJECT'):
             if find_macro_in_config(fn, 'SYS_CREATE_MDK4'):
                 os.system('scons --target=mdk4 -s')
-                print("Create mdk4 project done") 
+                print("Create mdk4 project done")
             elif find_macro_in_config(fn, 'SYS_CREATE_MDK5'):
                 os.system('scons --target=mdk5 -s')
-                print("Create mdk5 project done") 
+                print("Create mdk5 project done")
             elif find_macro_in_config(fn, 'SYS_CREATE_IAR'):
                 os.system('scons --target=iar -s')
-                print("Create iar project done") 
+                print("Create iar project done")
 
 
 def add_parser(sub):
@@ -277,7 +232,7 @@ def add_parser(sub):
                         dest='menuconfig_setting')
 
     parser.add_argument('--easy',
-                        help='easy mode,place kconfig file everywhere,just modify the option env="RTT_ROOT" default "../.."',
+                        help='easy mode, place kconfig everywhere, modify the option env="RTT_ROOT" default "../.."',
                         action='store_true',
                         default=False,
                         dest='menuconfig_easy')