fixed #11663 (daca: fetch basic analysis flags from server) [skip ci]

Author: firewave

tools/donate-cpu-server.py

@@ -26,7 +26,7 @@
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-SERVER_VERSION = "1.3.42"
+SERVER_VERSION = "1.3.43"
 
 OLD_VERSION = '2.11'
 
@@ -1235,10 +1235,68 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
             newThread = HttpClientThread(connection, cmd, resultPath, latestResults)
             newThread.start()
         elif cmd == 'GetCppcheckVersions\n':
+            # TODO: deprecate
             reply = 'head ' + OLD_VERSION
             print_ts('GetCppcheckVersions: ' + reply)
             connection.send(reply.encode('utf-8', 'ignore'))
             connection.close()
+        elif cmd == 'GetCppcheckData\n':
+            data = read_data(connection, cmd, pos_nl, max_data_size=8 * 1024, check_done=False, cmd_name='write_nodata')
+            if data is None:
+                continue
+
+            reply_obj = {
+                'v': 1, # version of the response object
+                'err': None, # provide an error to show within the client
+                'note': None # provide a note to show within the client
+            }
+
+            # TODO
+            try:
+                from packaging.version import Version
+                Version(data)
+            except Exception as e:
+                reply_obj.err = 'invalid client version: {}'.format(e)
+                pass
+
+            if reply_obj.err is None:
+                cppcheck_opts = [
+                    '--enable=style,information',
+                    '--inconclusive',
+                    '--template=daca2',
+                    '--showtime=top5',
+                    '--check-library',
+                    '--platform=unix64',
+                    # suppressions
+                    '--inline-suppr',
+                    '--suppress=unmatchedSuppression',
+                    # user provided defines
+                    '-D__GNUC__',
+                    # debug warnings and suppressions
+                    '--debug-warnings',
+                    '--suppress=autoNoType',
+                    '--suppress=bailoutUninitVar',
+                    '--suppress=symbolDatabaseWarning',
+                    '--suppress=valueFlowBailout',
+                    # TODO: remove missingInclude disabling when it no longer is implied by --enable=information
+                    '--disable=missingInclude'
+                ]
+
+                reply_data = {
+                    'versions': ['head', OLD_VERSION], # the versions to check with
+                    'build_opts': { # the build flags
+                        'make': ['MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g -w'],
+                        'mingw32-make': ['MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g -w'],
+                        'msbuild.exe': ['/property:Configuration=Release;Platform=x64']
+                    },
+                    'cppcheck_opts': ' '.join(cppcheck_opts) # the check options
+                }
+                reply_obj.update(reply_data)
+
+            reply = json.dumps(reply_obj)
+            print_ts('GetCppcheckData: ' + reply)
+            connection.send(reply.encode('utf-8', 'ignore'))
+            connection.close()
         elif cmd == 'get\n':
             while True:
                 pkg = packages[packageIndex]

tools/donate-cpu.py

@@ -174,7 +174,7 @@
             print('Stopping. Thank you!')
             sys.exit(0)
     try:
-        cppcheck_versions = lib.try_retry(lib.get_cppcheck_versions, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
+        cppcheck_versions, build_opts, cppcheck_opts = lib.try_retry(lib.get_cppcheck_options, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
     except Exception as e:
         print('Failed to get cppcheck versions from server ({}), retry later'.format(e))
         sys.exit(1)
@@ -192,11 +192,11 @@
             print('Failed to update Cppcheck ({}), retry later'.format(e))
             sys.exit(1)
         if ver == 'main':
-            if (has_changes or not lib.has_binary(current_cppcheck_dir)) and not lib.compile_cppcheck(current_cppcheck_dir):
+            if (has_changes or not lib.has_binary(current_cppcheck_dir)) and not lib.compile_cppcheck(current_cppcheck_dir, build_opts):
                 print('Failed to compile Cppcheck-{}, retry later'.format(ver))
                 sys.exit(1)
         else:
-            if not lib.compile_version(current_cppcheck_dir):
+            if not lib.compile_version(current_cppcheck_dir, build_opts):
                 print('Failed to compile Cppcheck-{}, retry later'.format(ver))
                 sys.exit(1)
     if package_urls:
@@ -254,7 +254,7 @@ def get_client_version_head():
                     return None
 
             client_version_head = get_client_version_head()
-        c, errout, info, t, cppcheck_options, timing_info = lib.scan_package(tree_path, source_path, libraries, capture_callstack)
+        c, errout, info, t, cppcheck_options, timing_info = lib.scan_package(tree_path, source_path, libraries, cppcheck_opts, capture_callstack)
         if c < 0:
             if c == -101 and 'error: could not find or open any of the paths given.' in errout:
                 # No sourcefile found (for example only headers present)

tools/donate_cpu_lib.py

@@ -16,7 +16,7 @@
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-CLIENT_VERSION = "1.3.48"
+CLIENT_VERSION = "1.3.49"
 
 # Timeout for analysis with Cppcheck in seconds
 CPPCHECK_TIMEOUT = 30 * 60
@@ -179,11 +179,11 @@ def has_binary(cppcheck_path):
     return False
 
 
-def compile_version(cppcheck_path):
+def compile_version(cppcheck_path, build_opts):
     if has_binary(cppcheck_path):
         return True
     # Build
-    ret = compile_cppcheck(cppcheck_path)
+    ret = compile_cppcheck(cppcheck_path, build_opts)
     # Clean intermediate build files
     if __make_cmd == "msbuild.exe":
         exclude_bin = 'bin'
@@ -196,7 +196,7 @@ def compile_version(cppcheck_path):
     return ret
 
 
-def compile_cppcheck(cppcheck_path):
+def compile_cppcheck(cppcheck_path, build_opts):
     print('Compiling {}'.format(os.path.basename(cppcheck_path)))
 
     cppcheck_bin = __get_cppcheck_binary(cppcheck_path)
@@ -211,9 +211,10 @@ def compile_cppcheck(cppcheck_path):
             # append to cl.exe options - need to omit dash or slash since a dash is being prepended
             build_env["_CL_"] = __jobs.replace('j', 'MP', 1)
             # TODO: processes still exhaust all threads of the system
-            subprocess.check_call([__make_cmd, '-t:cli', os.path.join(cppcheck_path, 'cppcheck.sln'), '/property:Configuration=Release;Platform=x64'], cwd=cppcheck_path, env=build_env)
+            build_cmd = [__make_cmd, '-t:cli', os.path.join(cppcheck_path, 'cppcheck.sln'), build_opts[__make_cmd]]
+            subprocess.check_call(build_cmd, cwd=cppcheck_path, env=build_env)
         else:
-            build_cmd = [__make_cmd, __jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g -w']
+            build_cmd = [__make_cmd, __jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g -w', build_opts[__make_cmd]]
             build_env = os.environ
             if __make_cmd == 'mingw32-make':
                 # TODO: MinGW will always link even if no changes are present
@@ -241,6 +242,7 @@ def compile_cppcheck(cppcheck_path):
     return True
 
 
+# TODO: deprecate
 def get_cppcheck_versions():
     print('Connecting to server to get Cppcheck versions..')
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
@@ -253,6 +255,27 @@ def get_cppcheck_versions():
     return versions.decode('utf-8').split()
 
 
+def get_cppcheck_options():
+    print('Connecting to server to get Cppcheck options..')
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.connect(__server_address)
+        sock.send(b'GetCppcheckData\n') # TODO: send version
+        options = sock.recv(1024) # TODO: increase
+    # TODO: sock.recv() sometimes hangs and returns b'' afterwards
+    if not options:
+        raise Exception('received empty response')
+    import json
+    options_obj = json.loads(options.decode('utf-8'))
+    if not options_obj.v or options_obj.v != 1:
+        raise Exception("received invalid version '{}'".format(options_obj.v))
+    # v1
+    if options_obj.err is not None:
+        raise Exception('server provided error: {}'.format(options_obj.err))
+    if options_obj.note is not None:
+        print('server provided notice: {}'.format(options_obj.note))
+    return options_obj.versions, options_obj.build_opts, options_obj.cppcheck_opts
+
+
 def get_packages_count():
     def __get_packages_count():
         print('Connecting to server to get count of packages..')
@@ -431,7 +454,7 @@ def __run_command(cmd, print_cmd=True):
     return return_code, stdout, stderr, elapsed_time
 
 
-def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
+def scan_package(cppcheck_path, source_path, libraries, cppcheck_opts, capture_callstack=True):
     print('Analyze..')
     libs = ''
     for library in libraries:
@@ -440,11 +463,8 @@ def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
 
     dir_to_scan = source_path
 
-    # TODO: remove missingInclude disabling when it no longer is implied by --enable=information
     # Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
-    options = libs + ' --showtime=top5 --check-library --inconclusive --enable=style,information --inline-suppr --disable=missingInclude --suppress=unmatchedSuppression --template=daca2'
-    options += ' --debug-warnings --suppress=autoNoType --suppress=valueFlowBailout --suppress=bailoutUninitVar --suppress=symbolDatabaseWarning'
-    options += ' -D__GNUC__ --platform=unix64'
+    options = libs + cppcheck_opts
     options_rp = options + ' -rp={}'.format(dir_to_scan)
     if __make_cmd == 'msbuild.exe':
         cppcheck_cmd = os.path.join(cppcheck_path, 'bin', 'cppcheck.exe') + ' ' + options_rp