[optimize] env scripts

archive.py

@@ -33,89 +33,105 @@
 import logging
 
 
-def unpack(archive_fn, path, package_info, package_name):
-    pkg_ver = package_info['ver']
-    flag = True
+def is_windows():
+    if platform.system() == "Windows":
+        return True
+    else:
+        return False
+
+
+def remove_folder(folder_path):
+    if os.path.isdir(folder_path):
+        if is_windows():
+            cmd = 'rd /s /q ' + folder_path
+            os.system(cmd)
+        else:
+            shutil.rmtree(folder_path)
+
+
+def unpack(archive_filename, path, package_info, package_name):
+    package_version = package_info['ver']
 
     package_temp_path = os.path.join(path, "package_temp")
     os.makedirs(package_temp_path)
 
-    logging.info("archive_fn", archive_fn)
-    logging.info("path", path)
-
-    if platform.system() == "Windows":
-        is_windows = True
-    else:
-        is_windows = False
+    logging.info("BSP packages path {0}".format(path))
+    logging.info("BSP package temp path: {0}".format(package_temp_path))
+    logging.info("archive filename : {0}".format(archive_filename))
 
-    if ".tar.bz2" in archive_fn:
-        arch = tarfile.open(archive_fn, "r:bz2")
+    if ".tar.bz2" in archive_filename:
+        arch = tarfile.open(archive_filename, "r:bz2")
         for tarinfo in arch:
             arch.extract(tarinfo, path)
             a = tarinfo.name
             if not os.path.isdir(os.path.join(path, a)):
-                if is_windows:
+                if is_windows():
                     right_path = a.replace('/', '\\')
                 else:
                     right_path = a
                 a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1])
 
-                pkgsdb.save_to_database(a, archive_fn)
+                pkgsdb.save_to_database(a, archive_filename)
         arch.close()
 
-    if ".tar.gz" in archive_fn:
-        arch = tarfile.open(archive_fn, "r:gz")
+    if ".tar.gz" in archive_filename:
+        arch = tarfile.open(archive_filename, "r:gz")
         for tarinfo in arch:
             arch.extract(tarinfo, path)
             a = tarinfo.name
             if not os.path.isdir(os.path.join(path, a)):
-                if is_windows:
+                if is_windows():
                     right_path = a.replace('/', '\\')
                 else:
                     right_path = a
                 a = os.path.join(os.path.split(right_path)[0], os.path.split(right_path)[1])
-                pkgsdb.save_to_database(a, archive_fn)
+                pkgsdb.save_to_database(a, archive_filename)
         arch.close()
 
-    if ".zip" in archive_fn:
-        arch = zipfile.ZipFile(archive_fn, "r")
-        for item in arch.namelist():
-            arch.extract(item, package_temp_path)
-            if not os.path.isdir(os.path.join(package_temp_path, item)):
-                if is_windows:
-                    right_path = item.replace('/', '\\')
-                else:
-                    right_path = item
-
-                # Gets the folder name and change_dirname only once
-                if flag:
-                    dir_name = os.path.split(right_path)[0]
-                    change_dirname = package_name + '-' + pkg_ver
-                    flag = False
-
-                right_name_to_db = right_path.replace(dir_name, change_dirname, 1)
-                right_path = os.path.join("package_temp", right_path)
-                pkgsdb.save_to_database(right_name_to_db, archive_fn, right_path)
-        arch.close()
-
-    # Change the folder name
-    change_dirname = package_name + '-' + pkg_ver
-
-    if os.path.isdir(os.path.join(path, change_dirname)):
-        if is_windows:
-            cmd = 'rd /s /q ' + os.path.join(path, change_dirname)
-            os.system(cmd)
-        else:
-            shutil.rmtree(os.path.join(path, change_dirname))
-
-    rename_path = os.path.join(package_temp_path, change_dirname)
+    try:
+        if ".zip" in archive_filename:
+            flag = True
+            dir_name = ""
+            package_name_with_version = ""
+
+            arch = zipfile.ZipFile(archive_filename, "r")
+            for item in arch.namelist():
+                arch.extract(item, package_temp_path)
+                if not os.path.isdir(os.path.join(package_temp_path, item)):
+                    if is_windows():
+                        right_path = item.replace('/', '\\')
+                    else:
+                        right_path = item
+
+                    # Gets the folder name and changed folder name only once
+                    if flag:
+                        dir_name = os.path.split(right_path)[0]
+                        package_name_with_version = package_name + '-' + package_version
+                        flag = False
+
+                    right_name_to_db = right_path.replace(dir_name, package_name_with_version, 1)
+                    right_path = os.path.join("package_temp", right_path)
+                    pkgsdb.save_to_database(right_name_to_db, archive_filename, right_path)
+            arch.close()
+    except Exception as e:
+        # remove temp folder and archive file
+        logging.warning('unpack error message : {0}'.format(e))
+        logging.warning('unpack {0} failed'.format(os.path.basename(archive_filename)))
+        remove_folder(package_temp_path)
+        os.remove(archive_filename)
+        return False
+
+    # rename package folder name
+    package_name_with_version = package_name + '-' + package_version
+    rename_path = os.path.join(package_temp_path, package_name_with_version)
     os.rename(os.path.join(package_temp_path, dir_name), rename_path)
 
-    # copy to bsp packages path.
-    shutil.move(rename_path, os.path.join(path, change_dirname))
+    # copy package to bsp packages path.
+    shutil.move(rename_path, os.path.join(path, package_name_with_version))
 
-    # remove temp dir
-    shutil.rmtree(package_temp_path)
+    # remove temp folder
+    remove_folder(package_temp_path)
+    return True
 
 
 def package_integrity_test(path):

cmds/cmd_package/cmd_package_update.py

@@ -191,7 +191,7 @@ def install_git_package(bsp_package_path, package_name, package_info, package_ur
     return True
 
 
-def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, pkgs_name_in_json):
+def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, package_name):
     result = True
     # Download a package of compressed package type.
     if not package.download(package_info['ver'], local_pkgs_path, package_url):
@@ -208,10 +208,9 @@ def install_not_git_package(package, package_info, local_pkgs_path, package_url,
     # unpack package
     if not os.path.exists(pkg_dir):
         try:
-            if not package.unpack(package_path, bsp_package_path, package_info, pkgs_name_in_json):
+            if not archive.unpack(package_path, bsp_package_path, package_info, package_name):
                 result = False
         except Exception as e:
-            os.remove(package_path)
             result = False
             print('Error message: %s\t' % e)
     else:
@@ -229,8 +228,13 @@ def install_package(env_root, pkgs_root, bsp_root, package_info, force_update):
     bsp_package_path = os.path.join(bsp_root, 'packages')
 
     if not force_update:
+        logging.info(
+            "Begin to check if it's an user managed package {0}, {1} \n".format(bsp_package_path, package_info))
         if is_user_mange_package(bsp_package_path, package_info):
+            logging.info("User managed package {0}, {1} no need install. \n".format(bsp_package_path, package_info))
             return result
+        else:
+            logging.info("NOT User managed package {0}, {1} need install. \n".format(bsp_package_path, package_info))
 
     # get the .config file from env
     env_config_file = os.path.join(env_root, r'tools\scripts\cmds', '.config')
@@ -315,8 +319,8 @@ def update_submodule(repo_path):
         logging.warning('Error message:%s' % e)
 
 
-def get_pkg_folder_by_orign_path(orign_path, version):
-    return orign_path + '-' + version
+def get_package_folder(origin_path, version):
+    return origin_path + '-' + version
 
 
 def git_cmd_exec(cmd, cwd):
@@ -366,7 +370,7 @@ def update_latest_packages(sys_value):
         # Find out the packages which version is 'latest'
         if pkg['ver'] == "latest_version" or pkg['ver'] == "latest":
             repo_path = os.path.join(bsp_packages_path, pkgs_name_in_json)
-            repo_path = get_pkg_folder_by_orign_path(repo_path, pkg['ver'])
+            repo_path = get_package_folder(repo_path, pkg['ver'])
 
             # noinspection PyBroadException
             try:
@@ -380,7 +384,7 @@ def update_latest_packages(sys_value):
 
                     # if git root is same as repo path, then change the upstream
                     get_git_root = get_git_root_path(repo_path)
-                    if get_git_root is not None:
+                    if get_git_root:
                         if os.path.normcase(repo_path) == os.path.normcase(get_git_root):
                             if mirror_url[0] is not None:
                                 cmd = 'git remote set-url origin ' + mirror_url[0]
@@ -427,7 +431,7 @@ def get_git_root_path(repo_path):
         try:
             before = os.getcwd()
             os.chdir(repo_path)
-            result = os.popen('git rev-parse --show-toplevel')
+            result = os.popen("git rev-parse --show-toplevel")
             result = result.read()
             for line in result.splitlines()[:5]:
                 get_git_root = line
@@ -438,7 +442,8 @@ def get_git_root_path(repo_path):
             logging.warning("Error message : %s" % e)
             return None
     else:
-        print("Missing path %s" % repo_path)
+        logging.warning("Missing path {0}".format(repo_path))
+        logging.warning("If you manage this package manually, Env tool will not update it.")
         return None
 
 
@@ -527,51 +532,53 @@ def pre_package_update():
 
     # read data back from pkgs_error.json
     with open(pkgs_error_list_fn, 'r') as f:
-        pkgs_error = json.load(f)
+        package_error = json.load(f)
 
     # create SConscript file
     if not os.path.isfile(os.path.join(bsp_packages_path, 'SConscript')):
         with open(os.path.join(bsp_packages_path, 'SConscript'), 'w') as f:
             f.write(str(Bridge_SConscript))
 
-    return [oldpkgs, newpkgs, pkgs_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path,
+    return [oldpkgs, newpkgs, package_error, package_json_filename, pkgs_error_list_fn, bsp_packages_path,
             dbsqlite_pathname]
 
 
 def error_packages_handle(error_packages_list, read_back_pkgs_json, package_filename, force_update):
     bsp_root = Import('bsp_root')
     env_root = Import('env_root')
     pkgs_root = Import('pkgs_root')
-    error_packages_redownload_error_list = []
+    download_error = []
     flag = True
 
     if len(error_packages_list):
         print("\n==============================> Packages list to download :  \n")
-        for pkg in error_packages_list:
-            print("Package name : %s, Ver : %s" % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8")))
+        for package in error_packages_list:
+            print("Package name : %s, Ver : %s" % (package['name'].encode("utf-8"), package['ver'].encode("utf-8")))
         print("\nThe packages in the list above are accidentally deleted or renamed.")
         print("\nIf you manually delete the version suffix of the package folder name, ")
         print("you can use <pkgs --force-update> command to re-download these packages.")
         print("In case of accidental deletion, the ENV tool will automatically re-download these packages.")
 
         # re-download the packages in error_packages_list
-        for pkg in error_packages_list:
-            if install_package(env_root, pkgs_root, bsp_root, pkg, force_update):
+        for package in error_packages_list:
+            if install_package(env_root, pkgs_root, bsp_root, package, force_update):
                 print("\n==============================> %s %s update done \n"
-                      % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8")))
+                      % (package['name'].encode("utf-8"), package['ver'].encode("utf-8")))
             else:
-                error_packages_redownload_error_list.append(pkg)
-                print(pkg, 'download failed.')
+                download_error.append(package)
+                print(package, 'download failed.')
                 flag = False
 
-        if len(error_packages_redownload_error_list):
-            print("%s" % error_packages_redownload_error_list)
-            print("Packages:%s,%s re-download error, you need to use <pkgs --update> command again to re-download them."
-                  % (pkg['name'].encode("utf-8"), pkg['ver'].encode("utf-8")))
+        if len(download_error):
+            print("%s" % download_error)
+
+            for package in download_error:
+                print("Packages:%s, %s re-download error, you can use <pkgs --update> command to re-download them."
+                      % (package['name'].encode("utf-8"), package['ver'].encode("utf-8")))
 
-            write_back_package_json = sub_list(read_back_pkgs_json, error_packages_redownload_error_list)
+            error_write_back = sub_list(read_back_pkgs_json, download_error)
             with open(package_filename, 'w') as f:
-                f.write(json.dumps(write_back_package_json, indent=1))
+                f.write(json.dumps(error_write_back, indent=1))
 
     return flag
 
@@ -615,7 +622,7 @@ def get_package_remove_path(pkg, bsp_packages_path):
 
     # Handles the deletion of git repository folders with version Numbers
     remove_path = os.path.join(bsp_packages_path, dir_path)
-    remove_path_ver = get_pkg_folder_by_orign_path(remove_path, ver)
+    remove_path_ver = get_package_folder(remove_path, ver)
     return remove_path_ver
 
 
@@ -749,6 +756,10 @@ def install_packages(sys_value, force_update):
 
     old_package = sys_value[0]
     new_package = sys_value[1]
+
+    logging.info("old_package {0}".format(old_package))
+    logging.info("new_package {0}".format(new_package))
+
     package_filename = sys_value[3]
     bsp_root = Import('bsp_root')
     pkgs_root = Import('pkgs_root')

env.py

@@ -52,12 +52,7 @@ def init_argparse():
 
 
 def init_logger(env_root):
-    localtime = time.asctime(time.localtime(time.time()))
-    log_path = os.path.join(env_root, "env_logging", localtime.replace(" ", "-").replace(":", "-"))
-    os.makedirs(log_path)
-    log_name = os.path.join(log_path, "running_log.txt")
-
-    log_format = "%(filename)s %(lineno)d %(message)s "
+    log_format = "%(module)s %(lineno)d %(levelname)s %(message)s \n"
     date_format = '%Y-%m-%d  %H:%M:%S %a '
     logging.basicConfig(level=logging.WARNING,
                         format=log_format,

package.py

@@ -245,14 +245,4 @@ def download(self, ver, path, url_from_srv):
                     return False
         return ret
 
-    @staticmethod
-    def unpack(package_path, path, pkg, package_name_in_json):
-        try:
-            # ignore the return value
-            archive.unpack(package_path, path, pkg, package_name_in_json)
-            return True
-        except Exception as e:
-            print('unpack error message :%s' % e)
-            print('unpack %s failed' % os.path.basename(package_path))
-            os.remove(package_path)
-            return False
+