Skip to content

donate-cpu-server.py: some minor improvements and cleanups #4492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
orbitcowboy merged 7 commits into from
Sep 22, 2022
Merged

donate-cpu-server.py: some minor improvements and cleanups #4492

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cfdfeb8
donate-cpu-server.py: bumped version
firewave Feb 18, 2022
ca9c477
donate-cpu-server.py: adjusted some variables names
firewave Feb 18, 2022
41dc5b0
donate-cpu-server.py: small date string parsing improvement
firewave Sep 21, 2022
04af615
donate-cpu-server.py: added timestamp to all messages
firewave Sep 21, 2022
37b39db
donate-cpu-server.py: bail out if `donated-results` folder is missing
firewave Sep 21, 2022
09b3fb2
donate-cpu-server.py: use more detailed timestamp in logging
firewave Sep 21, 2022
2512582
donate-cpu-server.py: use `with` scopes
firewave Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 66 additions & 63 deletions tools/donate-cpu-server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes)
SERVER_VERSION = "1.3.28"
SERVER_VERSION = "1.3.29"

OLD_VERSION = '2.9'

Expand All @@ -46,6 +46,10 @@
logger.addHandler(handler_file)


def print_ts(msg):
print('[' + strDateTime() + '] ' + msg)


# Set up an exception hook for all uncaught exceptions so they can be logged
def handle_uncaught_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
Expand All @@ -59,7 +63,7 @@ def handle_uncaught_exception(exc_type, exc_value, exc_traceback):


def strDateTime() -> str:
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')


def dateTimeFromStr(datestr: str) -> datetime.datetime:
Expand Down Expand Up @@ -148,13 +152,13 @@ def latestReport(latestResults: list) -> str:
package = filename[filename.rfind('/')+1:]
current_year = datetime.date.today().year

datestr = ''
datestr = None
count = ['0', '0']
lost = 0
added = 0
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
#elif line.startswith('cppcheck:'):
# cppcheck = line[9:]
Expand Down Expand Up @@ -185,8 +189,8 @@ def crashReport(results_path: str) -> str:
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
with open(filename, 'rt') as file_:
datestr = None
for line in file_:
line = line.strip()
if line.startswith('cppcheck: '):
Expand All @@ -196,21 +200,21 @@ def crashReport(results_path: str) -> str:
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
elif line.startswith('count:'):
if line.find('Crash') < 0:
break
package = filename[filename.rfind('/')+1:]
counts = line.split(' ')
c2 = ''
c_version = ''
if counts[2] == 'Crash!':
c2 = 'Crash'
c1 = ''
c_version = 'Crash'
c_head = ''
if counts[1] == 'Crash!':
c1 = 'Crash'
html += fmt(package, datestr, c2, c1) + '\n'
if c1 != 'Crash':
c_head = 'Crash'
html += fmt(package, datestr, c_version, c_head) + '\n'
if c_head != 'Crash':
break
elif line.find(' received signal ') != -1:
crash_line = next(file_, '').strip()
Expand Down Expand Up @@ -243,7 +247,7 @@ def crashReport(results_path: str) -> str:
stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
continue

print('{} - unmatched stack frame - {}'.format(package, l))
print_ts('{} - unmatched stack frame - {}'.format(package, l))
break
key = hash(' '.join(stack_trace))

Expand Down Expand Up @@ -279,8 +283,8 @@ def timeoutReport(results_path: str) -> str:
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
with open(filename, 'rt') as file_:
datestr = None
for line in file_:
line = line.strip()
if line.startswith('cppcheck: '):
Expand All @@ -290,7 +294,7 @@ def timeoutReport(results_path: str) -> str:
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
elif line.startswith('count:'):
if line.find('TO!') < 0:
Expand Down Expand Up @@ -680,7 +684,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
for filename in glob.glob(resultPath + '/*'):
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
datestr = None
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith('cppcheck: '):
Expand All @@ -690,7 +694,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
continue
if not line.startswith('elapsed-time:'):
Expand Down Expand Up @@ -765,7 +769,7 @@ def timeReportSlow(resultPath: str) -> str:
for filename in glob.glob(resultPath + '/*'):
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
datestr = None
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith('cppcheck: '):
Expand All @@ -775,7 +779,7 @@ def timeReportSlow(resultPath: str) -> str:
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
continue
elif line.startswith('count:'):
Expand Down Expand Up @@ -815,7 +819,7 @@ def timeReportSlow(resultPath: str) -> str:
def check_library_report(result_path: str, message_id: str) -> str:
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore', 'checkLibraryCheckType'):
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
print(error_message)
print_ts(error_message)
return error_message

if message_id == 'checkLibraryCheckType':
Expand Down Expand Up @@ -941,7 +945,7 @@ def __init__(self, connection: socket.socket, cmd: str, resultPath: str, latestR
def run(self):
try:
cmd = self.cmd
print('[' + strDateTime() + '] ' + cmd)
print_ts(cmd)
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
if res is None:
self.connection.close()
Expand Down Expand Up @@ -1012,16 +1016,15 @@ def run(self):
else:
filename = resultPath + '/' + url
if not os.path.isfile(filename):
print('HTTP/1.1 404 Not Found')
print_ts('HTTP/1.1 404 Not Found')
self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
else:
f = open(filename, 'rt')
data = f.read()
f.close()
with open(filename, 'rt') as f:
data = f.read()
httpGetResponse(self.connection, data, 'text/plain')
except:
tb = "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
print(tb)
print_ts(tb)
httpGetResponse(self.connection, tb, 'text/plain')
finally:
time.sleep(1)
Expand All @@ -1042,12 +1045,12 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
with open('latest.txt', 'rt') as f:
latestResults = f.read().strip().split(' ')

print('[' + strDateTime() + '] version ' + SERVER_VERSION)
print('[' + strDateTime() + '] listening on port ' + str(server_address_port))
print_ts('version ' + SERVER_VERSION)
print_ts('listening on port ' + str(server_address_port))

while True:
# wait for a connection
print('[' + strDateTime() + '] waiting for a connection')
print_ts('waiting for a connection')
connection, client_address = sock.accept()
try:
bytes_received = connection.recv(128)
Expand All @@ -1057,7 +1060,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
continue
except UnicodeDecodeError as e:
connection.close()
print('Error: Decoding failed: ' + str(e))
print_ts('Error: Decoding failed: ' + str(e))
continue
if cmd.find('\n') < 1:
continue
Expand All @@ -1070,7 +1073,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
newThread.start()
elif cmd == 'GetCppcheckVersions\n':
reply = 'head ' + OLD_VERSION
print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply)
print_ts('GetCppcheckVersions: ' + reply)
connection.send(reply.encode('utf-8', 'ignore'))
connection.close()
elif cmd == 'get\n':
Expand All @@ -1079,11 +1082,10 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
if packageIndex >= len(packages):
packageIndex = 0

f = open('package-index.txt', 'wt')
f.write(str(packageIndex) + '\n')
f.close()
with open('package-index.txt', 'wt') as f:
f.write(str(packageIndex) + '\n')

print('[' + strDateTime() + '] get:' + pkg)
print_ts('get:' + pkg)
connection.send(pkg.encode('utf-8', 'ignore'))
connection.close()
elif cmd.startswith('write\nftp://') or cmd.startswith('write\nhttp://'):
Expand All @@ -1098,7 +1100,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
try:
text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed (write): ' + str(e))
print_ts('Error: Decoding failed (write): ' + str(e))
data = ''
break
t = 0.0
Expand All @@ -1114,17 +1116,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
if pos < 10:
continue
url = data[:pos]
print('[' + strDateTime() + '] write:' + url)
print_ts('write:' + url)

# save data
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
if res is None:
res = re.match(r'https?://cppcheck\.sf\.net/([a-z]+).tgz', url)
if res is None:
print('results not written. res is None.')
print_ts('results not written. res is None.')
continue
if url not in packages:
print('results not written. url is not in packages.')
print_ts('results not written. url is not in packages.')
continue
# Verify that head was compared to correct OLD_VERSION
versions_found = False
Expand All @@ -1133,17 +1135,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
if line.startswith('cppcheck: '):
versions_found = True
if OLD_VERSION not in line.split():
print('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' +
print_ts('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' +
line)
print('Ignoring data.')
print_ts('Ignoring data.')
old_version_wrong = True
break
if not versions_found:
print('Cppcheck versions missing in result data. Ignoring data.')
print_ts('Cppcheck versions missing in result data. Ignoring data.')
continue
if old_version_wrong:
continue
print('results added for package ' + res.group(1))
print_ts('results added for package ' + res.group(1))
filename = os.path.join(resultPath, res.group(1))
with open(filename, 'wt') as f:
f.write(strDateTime() + '\n' + data)
Expand All @@ -1167,7 +1169,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
try:
text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed (write_info): ' + str(e))
print_ts('Error: Decoding failed (write_info): ' + str(e))
data = ''
break
t = 0.0
Expand All @@ -1183,19 +1185,19 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
if pos < 10:
continue
url = data[:pos]
print('[' + strDateTime() + '] write_info:' + url)
print_ts('write_info:' + url)

# save data
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
if res is None:
res = re.match(r'https://cppcheck\.sf\.net/([a-z]+).tgz', url)
if res is None:
print('info output not written. res is None.')
print_ts('info output not written. res is None.')
continue
if url not in packages:
print('info output not written. url is not in packages.')
print_ts('info output not written. url is not in packages.')
continue
print('adding info output for package ' + res.group(1))
print_ts('adding info output for package ' + res.group(1))
info_path = resultPath + '/' + 'info_output'
if not os.path.exists(info_path):
os.mkdir(info_path)
Expand All @@ -1206,28 +1208,28 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
packages_count = str(len(packages))
connection.send(packages_count.encode('utf-8', 'ignore'))
connection.close()
print('[' + strDateTime() + '] getPackagesCount: ' + packages_count)
print_ts('getPackagesCount: ' + packages_count)
continue
elif cmd.startswith('getPackageIdx'):
request_idx = abs(int(cmd[len('getPackageIdx:'):]))
if request_idx < len(packages):
pkg = packages[request_idx]
connection.send(pkg.encode('utf-8', 'ignore'))
connection.close()
print('[' + strDateTime() + '] getPackageIdx: ' + pkg)
print_ts('getPackageIdx: ' + pkg)
else:
connection.close()
print('[' + strDateTime() + '] getPackageIdx: index is out of range')
print_ts('getPackageIdx: index is out of range')
continue
else:
if cmd.find('\n') < 0:
print('[' + strDateTime() + '] invalid command: "' + firstLine + '"')
print_ts('invalid command: "' + firstLine + '"')
else:
lines = cmd.split('\n')
s = '\\n'.join(lines[:2])
if len(lines) > 2:
s += '...'
print('[' + strDateTime() + '] invalid command: "' + s + '"')
print_ts('invalid command: "' + s + '"')
connection.close()


Expand All @@ -1236,26 +1238,27 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
if not os.path.isdir(workPath):
workPath = os.path.expanduser('~/daca@home')
os.chdir(workPath)
print('work path: ' + workPath)
print_ts('work path: ' + workPath)
resultPath = workPath + '/donated-results'
if not os.path.isdir(resultPath):
print_ts("fatal: result path '{}' is missing".format(resultPath))
sys.exit(1)

f = open('packages.txt', 'rt')
packages = [val.strip() for val in f.readlines()]
f.close()
with open('packages.txt', 'rt') as f:
packages = [val.strip() for val in f.readlines()]

print('packages: ' + str(len(packages)))
print_ts('packages: ' + str(len(packages)))

if len(packages) == 0:
print('fatal: there are no packages')
print_ts('fatal: there are no packages')
sys.exit(1)

packageIndex = 0
if os.path.isfile('package-index.txt'):
f = open('package-index.txt', 'rt')
packageIndex = int(f.read())
with open('package-index.txt', 'rt') as f:
packageIndex = int(f.read())
if packageIndex < 0 or packageIndex >= len(packages):
packageIndex = 0
f.close()

server_address_port = 8000
if '--test' in sys.argv[1:]:
Expand All @@ -1264,4 +1267,4 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
try:
server(server_address_port, packages, packageIndex, resultPath)
except socket.timeout:
print('Timeout!')
print_ts('Timeout!')