From 87e4ece74f94ac94ec4c0f77869953c617348c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Csord=C3=A1s?= Date: Fri, 8 Dec 2017 14:09:57 +0100 Subject: [PATCH 1/2] Server configuration file * Rename session_config.json to server_config.json. --- ...session_config.json => server_config.json} | 0 docs/authentication.md | 4 +- libcodechecker/libauth/cc_ldap.py | 2 +- libcodechecker/libhandlers/server.py | 2 +- libcodechecker/session_manager.py | 63 +++++++++---------- libcodechecker/util.py | 21 +++++++ tests/libtest/env.py | 14 ++--- 7 files changed, 61 insertions(+), 45 deletions(-) rename config/{session_config.json => server_config.json} (100%) diff --git a/config/session_config.json b/config/server_config.json similarity index 100% rename from config/session_config.json rename to config/server_config.json diff --git a/docs/authentication.md b/docs/authentication.md index 9215a9fe1f..a7f78d3a3c 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -25,8 +25,8 @@ Table of Contents ## Server-side configuration The server's configuration is stored in the server's *workspace* folder, in -`session_config.json`. This file is created, at the first start of the server, -using the package's installed `config/session_config.json` as a template. +`server_config.json`. This file is created, at the first start of the server, +using the package's installed `config/server_config.json` as a template. The `authentication` section of the config file controls how authentication is handled. diff --git a/libcodechecker/libauth/cc_ldap.py b/libcodechecker/libauth/cc_ldap.py index d1c8426cf3..b3379ed199 100644 --- a/libcodechecker/libauth/cc_ldap.py +++ b/libcodechecker/libauth/cc_ldap.py @@ -7,7 +7,7 @@ """ LDAP authentication module for CodeChecker. -Authenticate user based on the session_config.json LDAP part. +Authenticate user based on the server_config.json LDAP part. In the configuration `null` means it is not configured. diff --git a/libcodechecker/libhandlers/server.py b/libcodechecker/libhandlers/server.py index 0983086530..a6233748dd 100644 --- a/libcodechecker/libhandlers/server.py +++ b/libcodechecker/libhandlers/server.py @@ -220,7 +220,7 @@ def add_arguments_to_parser(parser): help="Force the server to run in " "authentication requiring mode, despite " "the configuration value in " - "'session_config.json'. This is needed " + "'server_config.json'. This is needed " "if you need to edit the product " "configuration of a server that would not " "require authentication otherwise.") diff --git a/libcodechecker/session_manager.py b/libcodechecker/session_manager.py index cda794e8c1..a0d04a316b 100644 --- a/libcodechecker/session_manager.py +++ b/libcodechecker/session_manager.py @@ -20,6 +20,7 @@ import portalocker from libcodechecker.logger import get_logger +from libcodechecker import util unsupported_methods = [] @@ -130,29 +131,14 @@ def check_file_owner_rw(file_to_check): return True -def load_session_cfg(session_cfg_file): +def load_server_cfg(server_cfg_file): """ Tries to load the session config file which should be a valid json file, if loading fails returns an empty dict. """ - scfg_dict = {} - try: - with open(session_cfg_file, 'r') as scfg: - scfg_dict = json.loads(scfg.read()) - check_file_owner_rw(session_cfg_file) - - except IOError: - LOG.debug('Failed to open user authentication file: ' + - session_cfg_file) - raise - except ValueError as verr: - LOG.warning(verr) - LOG.warning('Not valid user authentication file: ' + - session_cfg_file) - raise - - return scfg_dict + check_file_owner_rw(server_cfg_file) + return util.load_json_or_empty(server_cfg_file, {}) class SessionManager: @@ -165,21 +151,34 @@ def __init__(self, root_sha, force_auth=False): LOG.debug('Loading session config') # Check whether workspace's configuration exists. - session_cfg_file = os.path.join(SessionManager.CodeChecker_Workspace, - "session_config.json") - - if not os.path.exists(session_cfg_file): - LOG.info("CodeChecker server's authentication example " - "configuration file created at " + session_cfg_file) - shutil.copyfile(os.path.join(os.environ['CC_PACKAGE_ROOT'], - "config", "session_config.json"), - session_cfg_file) + server_cfg_file = os.path.join(SessionManager.CodeChecker_Workspace, + "server_config.json") + + if not os.path.exists(server_cfg_file): + # For backward compatibility reason if the session_config.json file + # exists we rename it to server_config.json. + session_cfg_file = os.path.join( + SessionManager.CodeChecker_Workspace, + "session_config.json") + example_cfg_file = os.path.join(os.environ['CC_PACKAGE_ROOT'], + "config", "server_config.json") + if os.path.exists(session_cfg_file): + LOG.info("Renaming {0} to {1}. Please check the example " + "configuration file ({2}) or the user guide for " + "more information.".format(session_cfg_file, + server_cfg_file, + example_cfg_file)) + os.rename(session_cfg_file, server_cfg_file) + else: + LOG.info("CodeChecker server's example configuration file " + "created at " + server_cfg_file) + shutil.copyfile(example_cfg_file, server_cfg_file) - LOG.debug(session_cfg_file) + LOG.debug(server_cfg_file) # Create the default settings and then load the file from the disk. scfg_dict = {'authentication': {'enabled': False}} - scfg_dict.update(load_session_cfg(session_cfg_file)) + scfg_dict.update(load_server_cfg(server_cfg_file)) self.__auth_config = scfg_dict["authentication"] @@ -444,11 +443,7 @@ def __init__(self): ".codechecker.passwords.json") LOG.debug(session_cfg_file) - try: - scfg_dict = load_session_cfg(session_cfg_file) - except (IOError, ValueError): - # On the client's side, continue execution with defaults. - scfg_dict = {} + scfg_dict = load_server_cfg(session_cfg_file) if not scfg_dict.get("credentials"): scfg_dict["credentials"] = {} diff --git a/libcodechecker/util.py b/libcodechecker/util.py index 6d58c470ce..116201e9fb 100644 --- a/libcodechecker/util.py +++ b/libcodechecker/util.py @@ -9,6 +9,7 @@ import datetime import hashlib +import json import os import re import shutil @@ -433,3 +434,23 @@ def get_new_line_col_without_whitespace(line_content, old_col): return ''.join(line_content.split()), \ old_col - line_strip_len + + +def load_json_or_empty(path, default=None, kind=None): + """ + Load the contents of the given file as a JSON and return it's value, + or default if the file can't be loaded. + """ + + ret = default + try: + with open(path, 'r') as handle: + ret = json.loads(handle.read()) + except IOError: + LOG.warning("Failed to open {0} file: {1}" + .format(kind if kind else 'json', path)) + except ValueError: + LOG.warning("'{1}' is not a valid {0} file." + .format(kind if kind else 'json', path)) + + return ret diff --git a/tests/libtest/env.py b/tests/libtest/env.py index 57242f524b..ac52bdbe90 100644 --- a/tests/libtest/env.py +++ b/tests/libtest/env.py @@ -267,29 +267,29 @@ def enable_auth(workspace): an auth-enabled server. Running the tests only work if the initial value (in package - session_config.json) is FALSE for authentication.enabled. + server_config.json) is FALSE for authentication.enabled. """ - session_config_filename = "session_config.json" + server_config_filename = "server_config.json" cc_package = codechecker_package() original_auth_cfg = os.path.join(cc_package, 'config', - session_config_filename) + server_config_filename) shutil.copy(original_auth_cfg, workspace) - session_cfg_file = os.path.join(workspace, - session_config_filename) + server_cfg_file = os.path.join(workspace, + server_config_filename) - with open(session_cfg_file, 'r+') as scfg: + with open(server_cfg_file, 'r+') as scfg: __scfg_original = scfg.read() scfg.seek(0) try: scfg_dict = json.loads(__scfg_original) except ValueError as verr: print(verr) - print('Malformed session config json.') + print('Malformed server config json.') sys.exit(1) scfg_dict["authentication"]["enabled"] = True From a5119f04ea21dd6dda927d962fa1bc892131db9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Csord=C3=A1s?= Date: Mon, 11 Dec 2017 14:18:52 +0100 Subject: [PATCH 2/2] Run storage limit --- config/server_config.json | 1 + docs/server_config.md | 24 ++++++++++++++++++++++ libcodechecker/server/api/report_server.py | 23 +++++++++++++++++++++ libcodechecker/server/server.py | 1 + libcodechecker/session_manager.py | 17 +++++++++++++-- tests/libtest/env.py | 23 ++++++++------------- 6 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 docs/server_config.md diff --git a/config/server_config.json b/config/server_config.json index 4a08a4c78a..9a3e9d5e71 100644 --- a/config/server_config.json +++ b/config/server_config.json @@ -1,4 +1,5 @@ { + "max_run_count": 200, "authentication": { "enabled" : false, "realm_name" : "CodeChecker Privileged server", diff --git a/docs/server_config.md b/docs/server_config.md new file mode 100644 index 0000000000..22fe2a908a --- /dev/null +++ b/docs/server_config.md @@ -0,0 +1,24 @@ +CodeChecker server configuration +==================================== + +The server's configuration is stored in the server's *workspace* folder, in +`server_config.json`. This file is created, at the first start of the server, +using the package's installed `config/server_config.json` as a template. + +> **NOTICE!** `session_config.json` file has been deprecated. + +Table of Contents +================= +* [Run limitation](#run-limitations) +* [Authentication](#authentication) + +## Run limitation +The `max_run_count` section of the config file controls how many runs can be +stored on the server for a product. + +If this field is not present in the config file or the value of this field is a +negative value, run storage becomes unlimited. + +## Authentication +For authentication configuration options see the +[Authentication](authentication.md) documentation. diff --git a/libcodechecker/server/api/report_server.py b/libcodechecker/server/api/report_server.py index ec5bd40397..b10d5cc394 100644 --- a/libcodechecker/server/api/report_server.py +++ b/libcodechecker/server/api/report_server.py @@ -429,6 +429,7 @@ class ThriftRequestHandler(object): """ def __init__(self, + manager, Session, product, auth_session, @@ -442,6 +443,7 @@ def __init__(self, raise ValueError("Cannot initialize request handler without " "a product to serve.") + self.__manager = manager self.__product = product self.__auth_session = auth_session self.__config_database = config_database @@ -1916,6 +1918,27 @@ def __store_reports(self, session, report_dir, source_root, run_id, def massStoreRun(self, name, tag, version, b64zip, force): self.__require_store() + with DBSession(self.__Session) as session: + run = session.query(Run).filter(Run.name == name).one_or_none() + max_run_count = self.__manager.get_max_run_count() + + # If max_run_count is not set in the config file, it will allow + # the user to upload unlimited runs. + if max_run_count: + run_count = session.query(Run.id).count() + + # If we are not updating a run or the run count is reached the + # limit it will throw an exception. + if not run and run_count >= max_run_count: + remove_run_count = run_count - max_run_count + 1 + raise shared.ttypes.RequestFailed( + shared.ttypes.ErrorCode.GENERAL, + 'You reached the maximum number of allowed runs ' + '({0}/{1})! Please remove at least {2} run(s) before ' + 'you try it again.'.format(run_count, + max_run_count, + remove_run_count)) + with util.TemporaryDirectory() as zip_dir: # Unzip sent data. unzip(b64zip, zip_dir) diff --git a/libcodechecker/server/server.py b/libcodechecker/server/server.py index a6aa00b982..f8865c826b 100644 --- a/libcodechecker/server/server.py +++ b/libcodechecker/server/server.py @@ -404,6 +404,7 @@ def do_POST(self): self.__check_prod_db(product) acc_handler = ReportHandler_v6( + self.server.manager, product.session_factory, product, auth_session, diff --git a/libcodechecker/session_manager.py b/libcodechecker/session_manager.py index a0d04a316b..0ba614bab9 100644 --- a/libcodechecker/session_manager.py +++ b/libcodechecker/session_manager.py @@ -137,8 +137,11 @@ def load_server_cfg(server_cfg_file): valid json file, if loading fails returns an empty dict. """ - check_file_owner_rw(server_cfg_file) - return util.load_json_or_empty(server_cfg_file, {}) + if os.path.exists(server_cfg_file): + check_file_owner_rw(server_cfg_file) + return util.load_json_or_empty(server_cfg_file, {}) + + return {} class SessionManager: @@ -180,6 +183,9 @@ def __init__(self, root_sha, force_auth=False): scfg_dict = {'authentication': {'enabled': False}} scfg_dict.update(load_server_cfg(server_cfg_file)) + self.__max_run_count = scfg_dict["max_run_count"] \ + if "max_run_count" in scfg_dict else None + self.__auth_config = scfg_dict["authentication"] if force_auth: @@ -406,6 +412,13 @@ def is_valid(self, token, access=False): return any(_sess.token == token and _sess.still_valid(access) for _sess in SessionManager.__valid_sessions) + def get_max_run_count(self): + """ + Returns the maximum storable run count. If the value is None it means + we can upload unlimited number of runs. + """ + return self.__max_run_count + def get_session(self, token, access=False): """Gets the privileged session object based based on the token. diff --git a/tests/libtest/env.py b/tests/libtest/env.py index ac52bdbe90..d45b43f9f7 100644 --- a/tests/libtest/env.py +++ b/tests/libtest/env.py @@ -21,6 +21,8 @@ from functional import PKG_ROOT from functional import REPO_ROOT +from libcodechecker import util + def get_free_port(): """ @@ -282,23 +284,14 @@ def enable_auth(workspace): server_cfg_file = os.path.join(workspace, server_config_filename) - with open(server_cfg_file, 'r+') as scfg: - __scfg_original = scfg.read() - scfg.seek(0) - try: - scfg_dict = json.loads(__scfg_original) - except ValueError as verr: - print(verr) - print('Malformed server config json.') - sys.exit(1) - - scfg_dict["authentication"]["enabled"] = True - scfg_dict["authentication"]["method_dictionary"]["enabled"] = True - scfg_dict["authentication"]["method_dictionary"]["auths"] =\ - ["cc:test", "john:doe"] + scfg_dict = util.load_json_or_empty(server_cfg_file, {}) + scfg_dict["authentication"]["enabled"] = True + scfg_dict["authentication"]["method_dictionary"]["enabled"] = True + scfg_dict["authentication"]["method_dictionary"]["auths"] = \ + ["cc:test", "john:doe"] + with open(server_cfg_file, 'w') as scfg: json.dump(scfg_dict, scfg, indent=2, sort_keys=True) - scfg.truncate() # Create a root user. root_file = os.path.join(workspace, 'root.user')