From ea045649ad81fd519c3e58a9c62525ffb2e6bd49 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 28 May 2020 10:54:52 +0800 Subject: [PATCH 1/7] [add] .gitattributes --- .gitattributes | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..29434262 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,30 @@ +# .gitattributes for Env Project. + +# Handle line endings automatically for files detected as text +# and leave all files detected as binary untouched. +# ============ +* text=auto + +# Source files +# ============ +*.pxd text diff=python +*.py text diff=python +*.py3 text diff=python +*.pyw text diff=python +*.pyx text diff=python +*.pyz text diff=python +*.pyi text diff=python + +# Binary files +# ============ +*.db binary +*.p binary +*.pkl binary +*.pickle binary +*.pyc binary +*.pyd binary +*.pyo binary + +# MarkDown +*.md text + From 4ca63362ba1ddb92d9e96a1d30dbede5f4dc26af Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 28 May 2020 14:13:57 +0800 Subject: [PATCH 2/7] [update] cmds --- cmds/cmd_package/cmd_package_upgrade.py | 8 +- cmds/cmd_system.py | 108 ++--- package.py | 510 ++++++++++++------------ pkgsdb.py | 448 ++++++++++----------- 4 files changed, 537 insertions(+), 537 deletions(-) diff --git a/cmds/cmd_package/cmd_package_upgrade.py b/cmds/cmd_package/cmd_package_upgrade.py index fd46a212..b9e3f413 100644 --- a/cmds/cmd_package/cmd_package_upgrade.py +++ b/cmds/cmd_package/cmd_package_upgrade.py @@ -26,6 +26,7 @@ import os from vars import Import from .cmd_package_utils import git_pull_repo, get_url_from_mirror_server, find_macro_in_config +from .cmd_package_update import need_using_mirror_download def upgrade_packages_index(): @@ -35,9 +36,8 @@ def upgrade_packages_index(): pkgs_root = Import('pkgs_root') env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server('packages', 'latest') if get_package_url is not None: @@ -80,8 +80,8 @@ def upgrade_env_script(): env_root = Import('env_root') env_kconfig_path = os.path.join(env_root, r'tools\scripts\cmds') env_config_file = os.path.join(env_kconfig_path, '.config') - if (not os.path.isfile(env_config_file)) or \ - (os.path.isfile(env_config_file) and find_macro_in_config(env_config_file, 'SYS_PKGS_DOWNLOAD_ACCELERATE')): + + if need_using_mirror_download(env_config_file): get_package_url, get_ver_sha = get_url_from_mirror_server('env', 'latest') if get_package_url is not None: diff --git a/cmds/cmd_system.py b/cmds/cmd_system.py index 4f682744..a26ad348 100644 --- a/cmds/cmd_system.py +++ b/cmds/cmd_system.py @@ -1,54 +1,54 @@ -# -*- coding:utf-8 -*- -# -# File : cmd_system.py -# This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Change Logs: -# Date Author Notes -# 2018-5-28 SummerGift Add copyright information -# - -import os -from vars import Import - -'''RT-Thread environment package system''' - - -def cmd(args): - packages_root = os.path.join(Import('env_root'), 'packages') - - if args.system_update: - dir_list = os.listdir(packages_root) - - with open(os.path.join(packages_root, 'Kconfig'), 'w') as kconfig: - for item in dir_list: - if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): - kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') - kconfig.write('\n') - - -def add_parser(sub): - parser = sub.add_parser('system', help=__doc__, description=__doc__) - - parser.add_argument('--update', - help='update system menuconfig\'s online package options ', - action='store_true', - default=False, - dest='system_update') - - parser.set_defaults(func=cmd) +# -*- coding:utf-8 -*- +# +# File : cmd_system.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2018-5-28 SummerGift Add copyright information +# + +import os +from vars import Import + +'''RT-Thread environment package system''' + + +def cmd(args): + packages_root = os.path.join(Import('env_root'), 'packages') + + if args.system_update: + dir_list = os.listdir(packages_root) + + with open(os.path.join(packages_root, 'Kconfig'), 'w') as kconfig: + for item in dir_list: + if os.path.isfile(os.path.join(packages_root, item, 'Kconfig')): + kconfig.write('source "$PKGS_DIR/' + item + '/Kconfig"') + kconfig.write('\n') + + +def add_parser(sub): + parser = sub.add_parser('system', help=__doc__, description=__doc__) + + parser.add_argument('--update', + help='update system menuconfig\'s online package options ', + action='store_true', + default=False, + dest='system_update') + + parser.set_defaults(func=cmd) diff --git a/package.py b/package.py index fded16ef..649aa093 100644 --- a/package.py +++ b/package.py @@ -1,255 +1,255 @@ -# -*- coding:utf-8 -*- -# -# File : package.py -# This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Change Logs: -# Date Author Notes -# 2018-5-28 SummerGift Add copyright information -# 2018-12-28 Ernest Chen Add package information and enjoy package maker -# 2020-4-7 SummerGift Code improvement -# - -import json -import logging -import os -import sys - -import requests - -import archive - -"""Template for creating a new file""" - -Bridge_SConscript = '''import os -from building import * - -objs = [] -cwd = GetCurrentDir() -list = os.listdir(cwd) - -for item in list: - if os.path.isfile(os.path.join(cwd, item, 'SConscript')): - objs = objs + SConscript(os.path.join(item, 'SConscript')) - -Return('objs') -''' - -Kconfig_file = ''' -# Kconfig file for package ${lowercase_name} -menuconfig PKG_USING_${name} - bool "${description}" - default n - -if PKG_USING_${name} - - config PKG_${name}_PATH - string - default "/packages/${pkgs_class}/${lowercase_name}" - - choice - prompt "Version" - default PKG_USING_${name}_LATEST_VERSION - help - Select the package version - - config PKG_USING_${name}_V${version_standard} - bool "v${version}" - - config PKG_USING_${name}_LATEST_VERSION - bool "latest" - endchoice - - config PKG_${name}_VER - string - default "v${version}" if PKG_USING_${name}_V${version_standard} - default "latest" if PKG_USING_${name}_LATEST_VERSION - -endif - -''' - -Package_json_file = '''{ - "name": "${name}", - "description": "${description}", - "description_zh": "${description_zh}", - "enable": "PKG_USING_${pkgs_using_name}", - "keywords": [ - "${keyword}" - ], - "category": "${pkgsclass}", - "author": { - "name": "${authorname}", - "email": "${authoremail}", - "github": "${authorname}" - }, - "license": "${license}", - "repository": "${repository}", - "icon": "unknown", - "homepage": "unknown", - "doc": "unknown", - "site": [ - { - "version": "v${version}", - "URL": "https://${name}-${version}.zip", - "filename": "${name}-${version}.zip", - "VER_SHA": "fill in the git version SHA value" - }, - { - "version": "latest", - "URL": "https://xxxxx.git", - "filename": "Null for git package", - "VER_SHA": "fill in latest version branch name, such as master" - } - ] -} -''' - -Sconscript_file = ''' -from building import * - -cwd = GetCurrentDir() -src = Glob('*.c') + Glob('*.cpp') -CPPPATH = [cwd] - -group = DefineGroup('${name}', src, depend = [''], CPPPATH = CPPPATH) - -Return('group') -''' - - -class PackageOperation: - pkg = None - - def parse(self, filename): - with open(filename, "r") as f: - json_str = f.read() - - if json_str: - self.pkg = json.loads(json_str) - - def get_name(self): - return self.pkg['name'] - - def get_filename(self, ver): - for item in self.pkg['site']: - if item['version'].lower() == ver.lower(): - return item['filename'] - - return None - - def get_url(self, ver): - url = None - for item in self.pkg['site']: - if item['version'].lower() == ver.lower(): - url = item['URL'] - - if not url: - logging.warning("Can't find right url {0}, please check {1}".format(ver.lower(), self.pkg['site'])) - - return url - - def get_versha(self, ver): - for item in self.pkg['site']: - if item['version'].lower() == ver.lower(): - return item['VER_SHA'] - - return None - - def get_site(self, ver): - for item in self.pkg['site']: - if item['version'].lower() == ver.lower(): - return item - - return None - - def download(self, ver, path, url_from_srv): - ret = True - url = self.get_url(ver) - site = self.get_site(ver) - if site and 'filename' in site: - filename = site['filename'] - path = os.path.join(path, filename) - else: - basename = os.path.basename(url) - path = os.path.join(path, basename) - - if os.path.isfile(path): - if not os.path.getsize(path): - os.remove(path) - else: - if archive.package_integrity_test(path): - # print "The file is rigit." - return True - else: - os.remove(path) - - retry_count = 0 - - headers = {'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip, deflate', - 'Accept': '*/*', - 'User-Agent': 'curl/7.54.0'} - - print('Start to download package : %s ' % filename.encode("utf-8")) - - while True: - # print("retry_count : %d"%retry_count) - try: - r = requests.get(url_from_srv, stream=True, headers=headers) - - flush_count = 0 - - with open(path, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: - f.write(chunk) - f.flush() - flush_count += 1 - sys.stdout.write("\rDownloding %d KB" % flush_count) - sys.stdout.flush() - - retry_count = retry_count + 1 - - if archive.package_integrity_test(path): # make sure the file is right - ret = True - print("\rDownloded %d KB " % flush_count) - print('Start to unpack. Please wait...') - break - else: - if os.path.isfile(path): - os.remove(path) - if retry_count > 5: - print( - "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) - if os.path.isfile(path): - os.remove(path) - ret = False - break - except Exception as e: - print(url_from_srv) - print('error message:%s\t' % e) - retry_count = retry_count + 1 - if retry_count > 5: - print('%s download fail!\n' % path.encode("utf-8")) - if os.path.isfile(path): - os.remove(path) - return False - return ret - - +# -*- coding:utf-8 -*- +# +# File : package.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2018-5-28 SummerGift Add copyright information +# 2018-12-28 Ernest Chen Add package information and enjoy package maker +# 2020-4-7 SummerGift Code improvement +# + +import json +import logging +import os +import sys + +import requests + +import archive + +"""Template for creating a new file""" + +Bridge_SConscript = '''import os +from building import * + +objs = [] +cwd = GetCurrentDir() +list = os.listdir(cwd) + +for item in list: + if os.path.isfile(os.path.join(cwd, item, 'SConscript')): + objs = objs + SConscript(os.path.join(item, 'SConscript')) + +Return('objs') +''' + +Kconfig_file = ''' +# Kconfig file for package ${lowercase_name} +menuconfig PKG_USING_${name} + bool "${description}" + default n + +if PKG_USING_${name} + + config PKG_${name}_PATH + string + default "/packages/${pkgs_class}/${lowercase_name}" + + choice + prompt "Version" + default PKG_USING_${name}_LATEST_VERSION + help + Select the package version + + config PKG_USING_${name}_V${version_standard} + bool "v${version}" + + config PKG_USING_${name}_LATEST_VERSION + bool "latest" + endchoice + + config PKG_${name}_VER + string + default "v${version}" if PKG_USING_${name}_V${version_standard} + default "latest" if PKG_USING_${name}_LATEST_VERSION + +endif + +''' + +Package_json_file = '''{ + "name": "${name}", + "description": "${description}", + "description_zh": "${description_zh}", + "enable": "PKG_USING_${pkgs_using_name}", + "keywords": [ + "${keyword}" + ], + "category": "${pkgsclass}", + "author": { + "name": "${authorname}", + "email": "${authoremail}", + "github": "${authorname}" + }, + "license": "${license}", + "repository": "${repository}", + "icon": "unknown", + "homepage": "unknown", + "doc": "unknown", + "site": [ + { + "version": "v${version}", + "URL": "https://${name}-${version}.zip", + "filename": "${name}-${version}.zip", + "VER_SHA": "fill in the git version SHA value" + }, + { + "version": "latest", + "URL": "https://xxxxx.git", + "filename": "Null for git package", + "VER_SHA": "fill in latest version branch name, such as master" + } + ] +} +''' + +Sconscript_file = ''' +from building import * + +cwd = GetCurrentDir() +src = Glob('*.c') + Glob('*.cpp') +CPPPATH = [cwd] + +group = DefineGroup('${name}', src, depend = [''], CPPPATH = CPPPATH) + +Return('group') +''' + + +class PackageOperation: + pkg = None + + def parse(self, filename): + with open(filename, "r") as f: + json_str = f.read() + + if json_str: + self.pkg = json.loads(json_str) + + def get_name(self): + return self.pkg['name'] + + def get_filename(self, ver): + for item in self.pkg['site']: + if item['version'].lower() == ver.lower(): + return item['filename'] + + return None + + def get_url(self, ver): + url = None + for item in self.pkg['site']: + if item['version'].lower() == ver.lower(): + url = item['URL'] + + if not url: + logging.warning("Can't find right url {0}, please check {1}".format(ver.lower(), self.pkg['site'])) + + return url + + def get_versha(self, ver): + for item in self.pkg['site']: + if item['version'].lower() == ver.lower(): + return item['VER_SHA'] + + return None + + def get_site(self, ver): + for item in self.pkg['site']: + if item['version'].lower() == ver.lower(): + return item + + return None + + def download(self, ver, path, url_from_srv): + ret = True + url = self.get_url(ver) + site = self.get_site(ver) + if site and 'filename' in site: + filename = site['filename'] + path = os.path.join(path, filename) + else: + basename = os.path.basename(url) + path = os.path.join(path, basename) + + if os.path.isfile(path): + if not os.path.getsize(path): + os.remove(path) + else: + if archive.package_integrity_test(path): + # print "The file is rigit." + return True + else: + os.remove(path) + + retry_count = 0 + + headers = {'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + 'Accept': '*/*', + 'User-Agent': 'curl/7.54.0'} + + print('Start to download package : %s ' % filename.encode("utf-8")) + + while True: + # print("retry_count : %d"%retry_count) + try: + r = requests.get(url_from_srv, stream=True, headers=headers) + + flush_count = 0 + + with open(path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + flush_count += 1 + sys.stdout.write("\rDownloding %d KB" % flush_count) + sys.stdout.flush() + + retry_count = retry_count + 1 + + if archive.package_integrity_test(path): # make sure the file is right + ret = True + print("\rDownloded %d KB " % flush_count) + print('Start to unpack. Please wait...') + break + else: + if os.path.isfile(path): + os.remove(path) + if retry_count > 5: + print( + "error: Have tried downloading 5 times.\nstop Downloading file :%s" % path) + if os.path.isfile(path): + os.remove(path) + ret = False + break + except Exception as e: + print(url_from_srv) + print('error message:%s\t' % e) + retry_count = retry_count + 1 + if retry_count > 5: + print('%s download fail!\n' % path.encode("utf-8")) + if os.path.isfile(path): + os.remove(path) + return False + return ret + + diff --git a/pkgsdb.py b/pkgsdb.py index a6db58d1..5a1a3d4d 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -1,224 +1,224 @@ -# -*- coding:utf-8 -*- -# -# File : pkgsdb.py -# This file is part of RT-Thread RTOS -# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Change Logs: -# Date Author Notes -# 2018-5-28 SummerGift Add copyright information -# 2020-4-10 SummerGift Code clear up -# - -import sqlite3 -import os -import hashlib -from cmds.cmd_package.cmd_package_utils import user_input -from vars import Import - -SHOW_SQL = False - - -def get_file_md5(filename): - if not os.path.isfile(filename): - return - hash_value = hashlib.md5() - f = open(filename, 'rb') - while True: - b = f.read(8096) - if not b: - break - hash_value.update(b) - f.close() - return hash_value.hexdigest() - - -def get_conn(path): - conn = sqlite3.connect(path) - if os.path.exists(path) and os.path.isfile(path): - return conn - else: - print('on memory:[:memory:]') - return sqlite3.connect(':memory:') - - -def close_all(conn): - if conn is not None: - conn.close() - - -def get_cursor(conn): - if conn is not None: - return conn.cursor() - else: - return get_conn('').cursor() - - -def create_table(conn, sql): - if sql is not None and sql != '': - cu = get_cursor(conn) - if SHOW_SQL: - print('execute :[{}]'.format(sql)) - cu.execute(sql) - conn.commit() - close_all(conn) - else: - print('the [{}] is empty or equal None!'.format(sql)) - - -def save(conn, sql, data): - """insert data to database""" - if sql is not None and sql != '': - if data is not None: - cu = get_cursor(conn) - for d in data: - if SHOW_SQL: - print('execute sql:[{}],arguments:[{}]'.format(sql, d)) - cu.execute(sql, d) - conn.commit() - close_all(conn) - else: - print('the [{}] is empty or equal None!'.format(sql)) - - -def isdataexist(pathname): - ret = True - dbfilename = Import('dbsqlite_pathname') - - conn = get_conn(dbfilename) - c = get_cursor(conn) - sql = 'SELECT md5 from packagefile where pathname = "' + pathname + '"' - cursor = c.execute(sql) - for row in cursor: - dbmd5 = row[0] - - if dbmd5: - ret = False - conn.close() - return ret - - -# 将数据添加到数据库，如果数据库中已经存在则不重复添加 -def save_to_database(pathname, package_pathname, before_change_name): - db_pathname = Import('dbsqlite_pathname') - bsp_root = Import('bsp_root') - bsp_packages_path = os.path.join(bsp_root, 'packages') - - conn = get_conn(db_pathname) - save_sql = '''insert into packagefile values (?, ?, ?)''' - package = os.path.basename(package_pathname) - md5pathname = os.path.join(bsp_packages_path, before_change_name) - - if not os.path.isfile(md5pathname): - print("md5pathname is Invalid") - - md5 = get_file_md5(md5pathname) - data = [(pathname, package, md5)] - save(conn, save_sql, data) - - -def dbdump(dbfilename): - conn = get_conn(dbfilename) - c = get_cursor(conn) - cursor = c.execute("SELECT pathname, package, md5 from packagefile") - for row in cursor: - print("pathname = ", row[0]) - print("package = ", row[1]) - print("md5 = ", row[2], "\n") - conn.close() - - -def remove_unchanged_file(pathname, dbfilename, dbsqlname): - """delete unchanged files""" - flag = True - - conn = get_conn(dbfilename) - c = get_cursor(conn) - filemd5 = get_file_md5(pathname) - dbmd5 = 0 - - sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' - # print sql - cursor = c.execute(sql) - for row in cursor: - # fetch md5 from database - dbmd5 = row[0] - - if dbmd5 == filemd5: - # delete file info from database - sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - conn.commit() - os.remove(pathname) - else: - print("%s has been changed." % pathname) - print('Are you sure you want to permanently delete the file: %s?' % - os.path.basename(pathname)) - print('If you choose to keep the changed file,you should copy the file to another folder. ' - '\nbecaues it may be covered by the next update.') - - rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') - if rc == 'y' or rc == 'Y': - sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" - conn.commit() - os.remove(pathname) - print("%s has been removed.\n" % pathname) - else: - flag = False - conn.close() - return flag - - -# 删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 -# 如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false -def deletepackdir(dirpath, dbpathname): - flag = getdirdisplay(dirpath, dbpathname) - - if flag: - if os.path.exists(dirpath): - for root, dirs, files in os.walk(dirpath, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(dirpath) - # print "the dir should be delete" - return flag - - -# 遍历filepath下所有文件，包括子目录 -def displaydir(filepath, basepath, length, dbpathname): - flag = True - if os.path.isdir(filepath): - files = os.listdir(filepath) - for fi in files: - fi_d = os.path.join(filepath, fi) - if os.path.isdir(fi_d): - displaydir(fi_d, basepath, length, dbpathname) - else: - pathname = os.path.join(filepath, fi_d) - dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] - if not remove_unchanged_file(pathname, dbpathname, dbsqlname): - flag = False - return flag - - -def getdirdisplay(filepath, dbpathname): - display = filepath - length = len(display) - basepath = os.path.basename(filepath) - flag = displaydir(filepath, basepath, length, dbpathname) - return flag +# -*- coding:utf-8 -*- +# +# File : pkgsdb.py +# This file is part of RT-Thread RTOS +# COPYRIGHT (C) 2006 - 2018, RT-Thread Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Change Logs: +# Date Author Notes +# 2018-5-28 SummerGift Add copyright information +# 2020-4-10 SummerGift Code clear up +# + +import sqlite3 +import os +import hashlib +from cmds.cmd_package.cmd_package_utils import user_input +from vars import Import + +SHOW_SQL = False + + +def get_file_md5(filename): + if not os.path.isfile(filename): + return + hash_value = hashlib.md5() + f = open(filename, 'rb') + while True: + b = f.read(8096) + if not b: + break + hash_value.update(b) + f.close() + return hash_value.hexdigest() + + +def get_conn(path): + conn = sqlite3.connect(path) + if os.path.exists(path) and os.path.isfile(path): + return conn + else: + print('on memory:[:memory:]') + return sqlite3.connect(':memory:') + + +def close_all(conn): + if conn is not None: + conn.close() + + +def get_cursor(conn): + if conn is not None: + return conn.cursor() + else: + return get_conn('').cursor() + + +def create_table(conn, sql): + if sql is not None and sql != '': + cu = get_cursor(conn) + if SHOW_SQL: + print('execute :[{}]'.format(sql)) + cu.execute(sql) + conn.commit() + close_all(conn) + else: + print('the [{}] is empty or equal None!'.format(sql)) + + +def save(conn, sql, data): + """insert data to database""" + if sql is not None and sql != '': + if data is not None: + cu = get_cursor(conn) + for d in data: + if SHOW_SQL: + print('execute sql:[{}],arguments:[{}]'.format(sql, d)) + cu.execute(sql, d) + conn.commit() + close_all(conn) + else: + print('the [{}] is empty or equal None!'.format(sql)) + + +def isdataexist(pathname): + ret = True + dbfilename = Import('dbsqlite_pathname') + + conn = get_conn(dbfilename) + c = get_cursor(conn) + sql = 'SELECT md5 from packagefile where pathname = "' + pathname + '"' + cursor = c.execute(sql) + for row in cursor: + dbmd5 = row[0] + + if dbmd5: + ret = False + conn.close() + return ret + + +# 将数据添加到数据库，如果数据库中已经存在则不重复添加 +def save_to_database(pathname, package_pathname, before_change_name): + db_pathname = Import('dbsqlite_pathname') + bsp_root = Import('bsp_root') + bsp_packages_path = os.path.join(bsp_root, 'packages') + + conn = get_conn(db_pathname) + save_sql = '''insert into packagefile values (?, ?, ?)''' + package = os.path.basename(package_pathname) + md5pathname = os.path.join(bsp_packages_path, before_change_name) + + if not os.path.isfile(md5pathname): + print("md5pathname is Invalid") + + md5 = get_file_md5(md5pathname) + data = [(pathname, package, md5)] + save(conn, save_sql, data) + + +def dbdump(dbfilename): + conn = get_conn(dbfilename) + c = get_cursor(conn) + cursor = c.execute("SELECT pathname, package, md5 from packagefile") + for row in cursor: + print("pathname = ", row[0]) + print("package = ", row[1]) + print("md5 = ", row[2], "\n") + conn.close() + + +def remove_unchanged_file(pathname, dbfilename, dbsqlname): + """delete unchanged files""" + flag = True + + conn = get_conn(dbfilename) + c = get_cursor(conn) + filemd5 = get_file_md5(pathname) + dbmd5 = 0 + + sql = 'SELECT md5 from packagefile where pathname = "' + dbsqlname + '"' + # print sql + cursor = c.execute(sql) + for row in cursor: + # fetch md5 from database + dbmd5 = row[0] + + if dbmd5 == filemd5: + # delete file info from database + sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" + conn.commit() + os.remove(pathname) + else: + print("%s has been changed." % pathname) + print('Are you sure you want to permanently delete the file: %s?' % + os.path.basename(pathname)) + print('If you choose to keep the changed file,you should copy the file to another folder. ' + '\nbecaues it may be covered by the next update.') + + rc = user_input('Press the Y Key to delete the folder or just press Enter to keep it : ') + if rc == 'y' or rc == 'Y': + sql = "DELETE from packagefile where pathname = '" + dbsqlname + "'" + conn.commit() + os.remove(pathname) + print("%s has been removed.\n" % pathname) + else: + flag = False + conn.close() + return flag + + +# 删除一个包，如果有文件被改动，则提示(y/n)是否要删除，输入y则删除文件，输入其他字符则保留文件。 +# 如果没有文件被改动，直接删除文件夹,包文件夹被完全删除返回true，有被修改的文件没有被删除返回false +def deletepackdir(dirpath, dbpathname): + flag = getdirdisplay(dirpath, dbpathname) + + if flag: + if os.path.exists(dirpath): + for root, dirs, files in os.walk(dirpath, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(dirpath) + # print "the dir should be delete" + return flag + + +# 遍历filepath下所有文件，包括子目录 +def displaydir(filepath, basepath, length, dbpathname): + flag = True + if os.path.isdir(filepath): + files = os.listdir(filepath) + for fi in files: + fi_d = os.path.join(filepath, fi) + if os.path.isdir(fi_d): + displaydir(fi_d, basepath, length, dbpathname) + else: + pathname = os.path.join(filepath, fi_d) + dbsqlname = basepath + os.path.join(filepath, fi_d)[length:] + if not remove_unchanged_file(pathname, dbpathname, dbsqlname): + flag = False + return flag + + +def getdirdisplay(filepath, dbpathname): + display = filepath + length = len(display) + basepath = os.path.basename(filepath) + flag = displaydir(filepath, basepath, length, dbpathname) + return flag From d2aa26630b380a1528585e1967c860bae973d702 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 28 May 2020 15:42:30 +0800 Subject: [PATCH 3/7] [update] .gitattributes --- .gitattributes | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitattributes b/.gitattributes index 29434262..f8888b79 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,13 +7,13 @@ # Source files # ============ -*.pxd text diff=python -*.py text diff=python -*.py3 text diff=python -*.pyw text diff=python -*.pyx text diff=python -*.pyz text diff=python -*.pyi text diff=python +*.pxd text eol=crlf diff=python +*.py text eol=crlf diff=python +*.py3 text eol=crlf diff=python +*.pyw text eol=crlf diff=python +*.pyx text eol=crlf diff=python +*.pyz text eol=crlf diff=python +*.pyi text eol=crlf diff=python # Binary files # ============ @@ -26,5 +26,5 @@ *.pyo binary # MarkDown -*.md text +*.md text eol=crlf From a85e11a9a47eb7368ab7a49c8271c821cdb795bf Mon Sep 17 00:00:00 2001 From: SummerGift Date: Thu, 28 May 2020 17:50:54 +0800 Subject: [PATCH 4/7] [del] .gitattrubutes --- .gitattributes | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index f8888b79..00000000 --- a/.gitattributes +++ /dev/null @@ -1,30 +0,0 @@ -# .gitattributes for Env Project. - -# Handle line endings automatically for files detected as text -# and leave all files detected as binary untouched. -# ============ -* text=auto - -# Source files -# ============ -*.pxd text eol=crlf diff=python -*.py text eol=crlf diff=python -*.py3 text eol=crlf diff=python -*.pyw text eol=crlf diff=python -*.pyx text eol=crlf diff=python -*.pyz text eol=crlf diff=python -*.pyi text eol=crlf diff=python - -# Binary files -# ============ -*.db binary -*.p binary -*.pkl binary -*.pickle binary -*.pyc binary -*.pyd binary -*.pyo binary - -# MarkDown -*.md text eol=crlf - From 6ba024e106e1996f3f5427f91a2f24f548722eca Mon Sep 17 00:00:00 2001 From: SummerGift <459994202@qq.com> Date: Thu, 28 May 2020 18:00:03 +0800 Subject: [PATCH 5/7] add .gitattributes --- .gitattributes | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f8888b79 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,30 @@ +# .gitattributes for Env Project. + +# Handle line endings automatically for files detected as text +# and leave all files detected as binary untouched. +# ============ +* text=auto + +# Source files +# ============ +*.pxd text eol=crlf diff=python +*.py text eol=crlf diff=python +*.py3 text eol=crlf diff=python +*.pyw text eol=crlf diff=python +*.pyx text eol=crlf diff=python +*.pyz text eol=crlf diff=python +*.pyi text eol=crlf diff=python + +# Binary files +# ============ +*.db binary +*.p binary +*.pkl binary +*.pickle binary +*.pyc binary +*.pyd binary +*.pyo binary + +# MarkDown +*.md text eol=crlf + From c409766bf17e4d3891c290513c482e6613e2c648 Mon Sep 17 00:00:00 2001 From: SummerGift Date: Fri, 29 May 2020 09:52:14 +0800 Subject: [PATCH 6/7] [update] pkgsdb.py --- pkgsdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgsdb.py b/pkgsdb.py index 5a1a3d4d..199a082d 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -112,7 +112,7 @@ def isdataexist(pathname): return ret -# 将数据添加到数据库，如果数据库中已经存在则不重复添加 +# Add data to the database, if the data already exists, don't add again def save_to_database(pathname, package_pathname, before_change_name): db_pathname = Import('dbsqlite_pathname') bsp_root = Import('bsp_root') From dab56ae1195287d9bde67e7911cbf1a2088aa60c Mon Sep 17 00:00:00 2001 From: SummerGift <459994202@qq.com> Date: Fri, 29 May 2020 10:05:30 +0800 Subject: [PATCH 7/7] [update] pkgsdb.py --- pkgsdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgsdb.py b/pkgsdb.py index 199a082d..b4b3fb85 100644 --- a/pkgsdb.py +++ b/pkgsdb.py @@ -199,7 +199,7 @@ def deletepackdir(dirpath, dbpathname): return flag -# 遍历filepath下所有文件，包括子目录 +# walk through all files in filepath, include subfolder def displaydir(filepath, basepath, length, dbpathname): flag = True if os.path.isdir(filepath):