From 6fb981677c2c222a4575e398386c2ce061f0a338 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 30 Nov 2020 14:25:57 +0200 Subject: [PATCH 001/140] SKALE-3580 Cleanup unused dependencies --- setup.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index f507cab3..e9d2af6d 100644 --- a/setup.py +++ b/setup.py @@ -24,14 +24,11 @@ def find_version(*file_paths): "isort>=4.2.15,<5.4.3", ], 'dev': [ - "boto3==1.13.19", "bumpversion==0.6.0", - "skale.py==3.10dev1", "pytest==5.4.3", "pytest-cov==2.9.0", "twine==2.0.0", - "mock==4.0.2", - "when-changed" + "mock==4.0.2" ] } @@ -53,7 +50,6 @@ def find_version(*file_paths): url='https://github.com/skalenetwork/node-cli', install_requires=[ "click==7.1.2", - "confuse", "docker==4.2.2", "readsettings==3.4.5", "PyInstaller==3.6", @@ -61,7 +57,6 @@ def find_version(*file_paths): "python-dateutil==2.8.1", "Jinja2==2.11.2", "psutil==5.7.0", - "pycryptodome==3.9.7", "python-dotenv==0.13.0", "terminaltables==3.1.0", "requests==2.23.0" From e6220361d055342cf9f234c2aaab209e7f7ca6d2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 30 Nov 2020 20:14:04 +0200 Subject: [PATCH 002/140] SKALE-3580 Move to the new API structure, rm old modules --- cli/containers.py | 53 -------------------- cli/health.py | 59 ++++++++++++++++++++++ cli/schains.py | 12 ----- configs/__init__.py | 1 - configs/routes.py | 95 +++++++++++++++++++++--------------- cli/sgx.py => core/health.py | 56 +++++++++++++++------ core/helper.py | 16 +++--- core/schains.py | 44 ++++++++--------- core/validators.py | 33 ------------- core/wallet.py | 8 +-- main.py | 6 +-- tests/routes_test.py | 44 +++++++++++++++++ text.yml | 18 +++++-- 13 files changed, 251 insertions(+), 194 deletions(-) delete mode 100644 cli/containers.py create mode 100644 cli/health.py rename cli/sgx.py => core/health.py (51%) delete mode 100644 core/validators.py create mode 100644 tests/routes_test.py diff --git a/cli/containers.py b/cli/containers.py deleted file mode 100644 index 700888d6..00000000 --- a/cli/containers.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import click -from core.helper import get_request -from core.print_formatters import (print_containers, - print_err_response) - - -@click.group() -def containers_cli(): - pass - - -@containers_cli.group('containers', help="Node containers commands") -def containers(): - pass - - -@containers.command(help="List of sChain containers running on connected node") -@click.option('--all', '-a', is_flag=True) -def schains(all): - status, payload = get_request('schains_containers', {'all': all}) - if status == 'ok': - print_containers(payload) - else: - print_err_response(payload) - - -@containers.command(help="List of SKALE containers running on connected node") -@click.option('--all', '-a', is_flag=True) -def ls(all): - status, payload = get_request('skale_containers', {'all': all}) - if status == 'ok': - print_containers(payload.get('containers', [])) - else: - print_err_response(payload) diff --git a/cli/health.py b/cli/health.py new file mode 100644 index 00000000..86152273 --- /dev/null +++ b/cli/health.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +from tools.texts import Texts + +from core.health import get_containers, get_schains_checks, get_sgx_info + + +G_TEXTS = Texts() +TEXTS = G_TEXTS['health'] + + +@click.group() +def health_cli(): + pass + + +@health_cli.group('health', help=TEXTS['help']) +def health(): + pass + + +@health.command(help=TEXTS['containers']['help']) +@click.option('--all', '-a', is_flag=True) +def containers(all): + get_containers(_all=all) + + +@health.command(help=TEXTS['schains_checks']['help']) +@click.option( + '--json', + 'json_format', + help=G_TEXTS['common']['json']['help'], + is_flag=True +) +def schains_checks(json_format: bool) -> None: + get_schains_checks(json_format) + + +@health.command(help=TEXTS['sgx']['help']) +def sgx(): + get_sgx_info() diff --git a/cli/schains.py b/cli/schains.py index 5fd2d737..15ba95ae 100644 --- a/cli/schains.py +++ b/cli/schains.py @@ -23,7 +23,6 @@ from core.schains import ( describe, get_schain_firewall_rules, - show_checks, show_config, show_dkg_info, show_schains, @@ -68,17 +67,6 @@ def show_rules(schain_name: str) -> None: get_schain_firewall_rules(schain_name) -@schains.command(help="List of healthchecks for sChains served by connected node") -@click.option( - '--json', - 'json_format', - help='Show data in JSON format', - is_flag=True -) -def checks(json_format: bool) -> None: - show_checks(json_format) - - @schains.command('repair', help='Toggle schain repair mode') @click.argument('schain_name') @click.option('--yes', is_flag=True, callback=abort_if_false, diff --git a/configs/__init__.py b/configs/__init__.py index 1702618a..ce73ed48 100644 --- a/configs/__init__.py +++ b/configs/__init__.py @@ -20,7 +20,6 @@ import os import sys from pathlib import Path -from configs.routes import ROUTES # noqa: F401 HOME_DIR = os.getenv('HOME_DIR') or str(Path.home()) SKALE_DIR = os.path.join(HOME_DIR, '.skale') diff --git a/configs/routes.py b/configs/routes.py index bed219b3..e2db3dee 100644 --- a/configs/routes.py +++ b/configs/routes.py @@ -1,41 +1,58 @@ +# -*- coding: utf-8 -*- +# +# This file is part of SKALE Admin +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + + +CURRENT_API_VERSION = 'v1' +API_PREFIX = '/api' + ROUTES = { - 'login': '/login', - 'logout': '/logout', - 'register': '/join', - 'node_info': '/node-info', - 'node_about': '/about-node', - 'create_node': '/create-node', - 'node_signature': '/node-signature', - 'test_host': '/test-host', - - 'wallet_info': '/load-wallet', - 'validators_info': '/validators-info', - 'send_eth': '/api/send-eth', - - 'schains_containers': '/containers/schains/list', - 'schains_healthchecks': '/api/schains/healthchecks', - 'node_schains': '/schains/list', - 'schain_config': '/schain-config', - 'skale_containers': '/containers/list', - - 'logs_dump': '/logs/dump', - - 'ssl_status': '/api/ssl/status', - 'ssl_upload': '/api/ssl/upload', - - 'dkg_statuses': '/api/dkg/statuses', - - 'sgx_info': '/api/sgx/info', - - 'start_exit': '/api/exit/start', - 'exit_status': '/api/exit/status', - 'finalize_exit': '/api/exit/finalize', - 'get_schain_firewall_rules': '/api/schains/firewall/show', - 'turn_on_schain_firewall_rules': '/api/schains/firewall/on', - 'turn_off_schain_firewall_rules': '/api/schains/firewall/off', - - 'maintenance_on': '/api/node/maintenance-on', - 'maintenance_off': '/api/node/maintenance-off', - 'repair_schain': '/api/schains/repair', - 'describe_schain': '/api/schains/get' + 'v1': { + 'logs': ['dump'], + 'exit': ['start', 'status'], + 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'send-tg-notification'], + 'health': ['containers', 'schains-checks', 'sgx'], + 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], + 'security': ['status', 'upload'], + 'wallet': ['info', 'send-eth'] + } } + + +class RouteNotFoundException(Exception): + """Raised when requested route is not found in provided API version""" + + +def route_exists(blueprint, method, api_version): + return ROUTES.get(api_version) and ROUTES[api_version].get(blueprint) and \ + method in ROUTES[api_version][blueprint] + + +def get_route(blueprint, method, api_version=CURRENT_API_VERSION, check=True): + route = os.path.join(API_PREFIX, api_version, blueprint, method) + if check and not route_exists(blueprint, method, api_version): + raise RouteNotFoundException(route) + return route + + +def get_all_available_routes(api_version=CURRENT_API_VERSION): + routes = ROUTES[api_version] + return [get_route(blueprint, method, api_version) for blueprint in routes + for method in routes[blueprint]] diff --git a/cli/sgx.py b/core/health.py similarity index 51% rename from cli/sgx.py rename to core/health.py index 32c81af3..3e194c20 100644 --- a/cli/sgx.py +++ b/core/health.py @@ -2,7 +2,7 @@ # # This file is part of node-cli # -# Copyright (C) 2019 SKALE Labs +# Copyright (C) 2020 SKALE Labs # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -12,34 +12,58 @@ # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import click +import json from terminaltables import SingleTable -from core.helper import get_request, safe_load_texts -from core.print_formatters import print_err_response +from core.helper import get_request +from core.print_formatters import ( + print_containers, + print_err_response, + print_schains_healthchecks +) +BLUEPRINT_NAME = 'health' -TEXTS = safe_load_texts() - -@click.group() -def sgx_cli(): - pass +def get_containers(_all): + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='containers', + params={'all': _all} + ) + if status == 'ok': + print_containers(payload.get('containers', [])) + else: + print_err_response(payload) -@sgx_cli.group('sgx', help="SGX commands") -def sgx(): - pass +def get_schains_checks(json_format: bool = False) -> None: + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='containers' + ) + if status == 'ok': + if not payload: + print('No sChains found') + return + if json_format: + print(json.dumps(payload)) + else: + print_schains_healthchecks(payload) + else: + print_err_response(payload) -@sgx.command(help="Info about connected SGX server") -def info(): - status, payload = get_request('sgx_info') +def get_sgx_info(): + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='sgx' + ) if status == 'ok': data = payload table_data = [ diff --git a/core/helper.py b/core/helper.py index 87db45b3..e83ba5e3 100644 --- a/core/helper.py +++ b/core/helper.py @@ -31,7 +31,8 @@ import requests import yaml -from configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT, ROUTES +from configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT +from configs.routes import get_route from configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, LOG_FILEPATH, DEBUG_LOG_FILEPATH) @@ -94,8 +95,9 @@ def abort_if_false(ctx, param, value): ctx.abort() -def post_request(url_name, json=None, files=None): - url = construct_url(ROUTES[url_name]) +def post_request(blueprint, method, json=None, files=None): + route = get_route(blueprint, method) + url = construct_url(route) try: response = requests.post(url, json=json, files=files) data = response.json() @@ -107,8 +109,9 @@ def post_request(url_name, json=None, files=None): return status, payload -def get_request(url_name, params=None): - url = construct_url(ROUTES[url_name]) +def get_request(blueprint, method, params=None): + route = get_route(blueprint, method) + url = construct_url(route) try: response = requests.get(url, params=params) data = response.json() @@ -122,7 +125,8 @@ def get_request(url_name, params=None): def download_dump(path, container_name=None): - url = construct_url(ROUTES['logs_dump']) + route = get_route('logs', 'dump') + url = construct_url(route) params = {} if container_name: params['container_name'] = container_name diff --git a/core/schains.py b/core/schains.py index d119e40d..650bd7b4 100644 --- a/core/schains.py +++ b/core/schains.py @@ -1,4 +1,3 @@ -import json import logging import pprint @@ -8,17 +7,21 @@ print_err_response, print_firewall_rules, print_schain_info, - print_schains, - print_schains_healthchecks + print_schains ) logger = logging.getLogger(__name__) +BLUEPRINT_NAME = 'schains' + def get_schain_firewall_rules(schain: str) -> None: - status, payload = get_request('get_schain_firewall_rules', - {'schain': schain}) + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='firewall-rules', + params={'schain_name': schain} + ) if status == 'ok': print_firewall_rules(payload['endpoints']) else: @@ -26,7 +29,10 @@ def get_schain_firewall_rules(schain: str) -> None: def show_schains() -> None: - status, payload = get_request('node_schains') + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='list' + ) if status == 'ok': schains = payload if not schains: @@ -39,7 +45,11 @@ def show_schains() -> None: def show_dkg_info(all_: bool = False) -> None: params = {'all': all_} - status, payload = get_request('dkg_statuses', params=params) + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='dkg-statuses', + params=params + ) if status == 'ok': print_dkg_statuses(payload) else: @@ -47,27 +57,17 @@ def show_dkg_info(all_: bool = False) -> None: def show_config(name: str) -> None: - status, payload = get_request('schain_config', {'schain-name': name}) + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='config', + params={'schain_name': name} + ) if status == 'ok': pprint.pprint(payload) else: print_err_response(payload) -def show_checks(json_format: bool = False) -> None: - status, payload = get_request('schains_healthchecks') - if status == 'ok': - if not payload: - print('No sChains found') - return - if json_format: - print(json.dumps(payload)) - else: - print_schains_healthchecks(payload) - else: - print_err_response(payload) - - def toggle_schain_repair_mode(schain: str) -> None: status, payload = post_request('repair_schain', {'schain': schain}) if status == 'ok': diff --git a/core/validators.py b/core/validators.py deleted file mode 100644 index 66f1feb9..00000000 --- a/core/validators.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -from core.helper import get_request -from core.print_formatters import print_err_response - - -def get_validators_info(config, format): - status, payload = get_request('validators_info') - if status == 'ok': - if format == 'json': - print(json.dumps({'validators_info': payload})) - else: - print(payload) - else: - print_err_response(payload) diff --git a/core/wallet.py b/core/wallet.py index e55badd8..1594b042 100644 --- a/core/wallet.py +++ b/core/wallet.py @@ -23,8 +23,11 @@ from core.print_formatters import print_err_response, print_wallet_info, TEXTS +API_BLUEPRINT_NAME = 'wallet' + + def get_wallet_info(_format): - status, payload = get_request('wallet_info') + status, payload = get_request('wallet', 'info') if status == 'ok': if _format == 'json': print(json.dumps(payload)) @@ -41,8 +44,7 @@ def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): 'gas_limit': gas_limit, 'gas_price': gas_price } - status, payload = post_request('send_eth', - json=json_data) + status, payload = post_request('wallet', 'send-eth', json=json_data) if status == 'ok': msg = TEXTS['wallet']['successful_transfer'] logger.info(msg) diff --git a/main.py b/main.py index bb1b101c..660caf0a 100644 --- a/main.py +++ b/main.py @@ -24,14 +24,13 @@ import traceback from cli import __version__ -from cli.containers import containers_cli +from cli.health import health_cli from cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION from cli.logs import logs_cli from cli.node import node_cli from cli.schains import schains_cli from cli.wallet import wallet_cli from cli.ssl import ssl_cli -from cli.sgx import sgx_cli from cli.exit import exit_cli from cli.validate import validate_cli from cli.resources_allocation import resources_allocation_cli @@ -93,14 +92,13 @@ def handle_exception(exc_type, exc_value, exc_traceback): cmd_collection = click.CommandCollection( sources=[ cli, + health_cli, schains_cli, - containers_cli, logs_cli, resources_allocation_cli, node_cli, wallet_cli, ssl_cli, - sgx_cli, exit_cli, validate_cli ]) diff --git a/tests/routes_test.py b/tests/routes_test.py new file mode 100644 index 00000000..8d814fb8 --- /dev/null +++ b/tests/routes_test.py @@ -0,0 +1,44 @@ +import pytest +from configs.routes import (route_exists, get_route, get_all_available_routes, + RouteNotFoundException) + + +ALL_V1_ROUTES = [ + '/api/v1/logs/dump', + '/api/v1/exit/start', + '/api/v1/exit/status', + '/api/v1/node/info', + '/api/v1/node/register', + '/api/v1/node/maintenance-on', + '/api/v1/node/maintenance-off', + '/api/v1/node/send-tg-notification', + '/api/v1/health/containers', + '/api/v1/health/schains-checks', + '/api/v1/health/sgx', + '/api/v1/schains/config', + '/api/v1/schains/list', + '/api/v1/schains/dkg-statuses', + '/api/v1/schains/firewall-rules', + '/api/v1/schains/repair', '/api/v1/schains/get', + '/api/v1/security/status', + '/api/v1/security/upload', + '/api/v1/wallet/info', + '/api/v1/wallet/upload' +] + + +def test_route_exists(): + assert route_exists('logs', 'dump', 'v1') + assert not route_exists('logshms', 'dumb', 'v1') + + +def test_get_route(): + repair_route = get_route('schains', 'repair') + assert repair_route == '/api/v1/schains/repair' + + with pytest.raises(RouteNotFoundException): + get_route('schains', 'refair') + + +def test_get_all_available_routes(): + assert get_all_available_routes() == ALL_V1_ROUTES diff --git a/text.yml b/text.yml index 47b82d37..b8f11bd4 100644 --- a/text.yml +++ b/text.yml @@ -1,4 +1,16 @@ -login: "Login user in a SKALE node" +health: + help: Node health commands + containers: + help: List of SKALE containers running on connected node + schains_checks: + help: List of health checks for sChains served by the node + sgx: + help: Info about connected SGX server + +common: + json: + help: Show data in JSON format + node: base: "SKALE node commands" info: "Info about SKALE node" @@ -22,11 +34,7 @@ wallet: successful_transfer: "Funds were successfully transferred" service: - unauthorized: 'You should login first: skale user login' - no_node_host: 'You should set host first: skale attach [HOST]' - node_host_not_valid: Provided SKALE node host is not valid node_not_registered: This SKALE node is not registered on SKALE Manager yet - empty_response: Your request returned nothing. Something went wrong. ssl: no_cert: |- From a7daf57161586bdbe02d50a662a53cd5c94f391c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 30 Nov 2020 20:22:30 +0200 Subject: [PATCH 003/140] SKALE-3580 Update README with health commands --- README.md | 60 ++++++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 2e826d89..1e836b6a 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,14 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 1. [Installation](#installation) 2. [CLI usage](#cli-usage) 2.1 [Top level commands](#top-level-commands) - 2.2 [User](#user-commands) - 2.3 [Node](#node-commands) - 2.4 [Wallet](#wallet-commands) - 2.5 [sChains](#schain-commands) - 2.6 [Containers](#containers-commands) - 2.7 [SGX](#sgx-commands) - 2.8 [SSL](#ssl-commands) - 2.9 [Logs](#logs-commands) - 2.10 [Resources allocation](#resources-allocation-commands) - 2.11 [Validate](#validate-commands) + 2.2 [Node](#node-commands) + 2.3 [Wallet](#wallet-commands) + 2.4 [sChains](#schain-commands) + 2.5 [Health](#health-commands) + 2.6 [SSL](#ssl-commands) + 2.7 [Logs](#logs-commands) + 2.8 [Resources allocation](#resources-allocation-commands) + 2.9 [Validate](#validate-commands) 3. [Development](#development) @@ -293,7 +291,7 @@ Optional arguments: `--yes` - Send without additional confirmation -### SKALE Chain commands +### sChain commands > Prefix: `skale schains` @@ -339,58 +337,40 @@ Turn on repair mode for SKALE Chain skale schains repair SCHAIN_NAME ``` -#### SKALE Chain healthcheck +### Health commands -Show healthcheck results for all SKALE Chains on the node +> Prefix: `skale health` -```shell -skale schains checks -``` - -Options: - -- `--json` - Show data in JSON format - -### Container commands - -Node container commands - -> Prefix: `skale containers` - -#### List containers +#### SKALE containers List all SKALE containers running on the connected node ```shell -skale containers ls +skale health containers ``` Options: - `-a/--all` - list all containers (by default - only running) -#### SKALE Chain containers +#### sChains healthchecks -List of SKALE chain containers running on the connected node +Show health check results for all SKALE Chains on the node ```shell -skale containers schains +skale health schains-checks ``` Options: -- `-a/--all` - list all SKALE chain containers (by default - only running) - -### SGX commands - -> Prefix: `skale sgx` +- `--json` - Show data in JSON format -#### Status +#### SGX Status of the SGX server. Returns the SGX server URL and connection status. ```shell -$ skale sgx status +$ skale health sgx SGX server status: ┌────────────────┬────────────────────────────┐ @@ -400,8 +380,6 @@ SGX server status: └────────────────┴────────────────────────────┘ ``` -Admin API URL: \[GET] `/api/ssl/sgx` - ### SSL commands > Prefix: `skale ssl` From 33a17a4634e823a14b1f1450ffc5e12b073c0951 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 3 Dec 2020 14:08:36 +0200 Subject: [PATCH 004/140] SKALE-3580 checks url fix --- core/health.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/health.py b/core/health.py index 3e194c20..fce69ed3 100644 --- a/core/health.py +++ b/core/health.py @@ -37,7 +37,7 @@ def get_containers(_all): params={'all': _all} ) if status == 'ok': - print_containers(payload.get('containers', [])) + print_containers(payload) else: print_err_response(payload) @@ -45,7 +45,7 @@ def get_containers(_all): def get_schains_checks(json_format: bool = False) -> None: status, payload = get_request( blueprint=BLUEPRINT_NAME, - method='containers' + method='schains-checks' ) if status == 'ok': if not payload: From accb4af5b19ee99328b1fca522b22827b0cdbfd7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 3 Dec 2020 18:32:33 +0200 Subject: [PATCH 005/140] SKALE-3580 Update for api 2.0 --- cli/__init__.py | 2 +- cli/exit.py | 11 ++++-- cli/node.py | 12 ++----- cli/ssl.py | 6 +++- configs/routes.py | 5 +-- core/core.py | 77 ---------------------------------------- core/helper.py | 6 +++- core/node.py | 59 ++++++++++++++++++++++++++---- core/print_formatters.py | 13 +++++++ core/schains.py | 12 +++++-- core/wallet.py | 6 ++-- tests/routes_test.py | 4 +-- 12 files changed, 106 insertions(+), 107 deletions(-) delete mode 100644 core/core.py diff --git a/cli/__init__.py b/cli/__init__.py index 42043458..4f404329 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.1.0' +__version__ = '2.0.0' if __name__ == "__main__": print(__version__) diff --git a/cli/exit.py b/cli/exit.py index 7f0ac612..82316967 100644 --- a/cli/exit.py +++ b/cli/exit.py @@ -6,6 +6,7 @@ logger = logging.getLogger(__name__) TEXTS = Texts() +BLUEPRINT_NAME = 'exit' @click.group() @@ -23,7 +24,10 @@ def node_exit(): expose_value=False, prompt='Are you sure you want to destroy your SKALE node?') def start(): - status, payload = post_request('start_exit') + status, payload = post_request( + blueprint=BLUEPRINT_NAME, + method='start' + ) if status == 'ok': msg = TEXTS['exit']['start'] logger.info(msg) @@ -35,7 +39,10 @@ def start(): @node_exit.command('status', help="Get exit process status") @click.option('--format', '-f', type=click.Choice(['json', 'text'])) def status(format): - status, payload = get_request('exit_status') + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='status' + ) if status == 'ok': exit_status = payload if format == 'json': diff --git a/cli/node.py b/cli/node.py index da365df7..adb60ca1 100644 --- a/cli/node.py +++ b/cli/node.py @@ -24,10 +24,9 @@ from skale.utils.random_names.generator import generate_random_node_name -from core.core import get_node_info, get_node_about from core.node import (get_node_signature, init, restore, - register_node as register, update, backup, - set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on) + register_node as register, update, backup, set_maintenance_mode_on, + set_maintenance_mode_off, turn_off, turn_on, get_node_info) from core.helper import abort_if_false, safe_load_texts from configs import DEFAULT_NODE_BASE_PORT from tools.helper import session_config @@ -84,13 +83,6 @@ def node_info(format): get_node_info(config, format) -@node.command('about', help="Get service info about SKALE node") -@click.option('--format', '-f', type=click.Choice(['json', 'text'])) -def node_about(format): - config = session_config() - get_node_about(config, format) - - @node.command('register', help="Register current node in the SKALE Manager") @click.option( '--name', '-n', diff --git a/cli/ssl.py b/cli/ssl.py index e8fc51ce..ccbe4763 100644 --- a/cli/ssl.py +++ b/cli/ssl.py @@ -25,6 +25,7 @@ TEXTS = safe_load_texts() +BLUEPRINT_NAME = 'ssl' @click.group() @@ -39,7 +40,10 @@ def ssl(): @ssl.command(help="Status of the SSL certificates on the node") def status(): - status, payload = get_request('ssl_status') + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='status' + ) if status == 'ok': if payload.get('is_empty'): print(TEXTS['ssl']['no_cert']) diff --git a/configs/routes.py b/configs/routes.py index e2db3dee..16485e87 100644 --- a/configs/routes.py +++ b/configs/routes.py @@ -27,10 +27,11 @@ 'v1': { 'logs': ['dump'], 'exit': ['start', 'status'], - 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'send-tg-notification'], + 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'signature', + 'send-tg-notification'], 'health': ['containers', 'schains-checks', 'sgx'], 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], - 'security': ['status', 'upload'], + 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'] } } diff --git a/core/core.py b/core/core.py deleted file mode 100644 index fce47d01..00000000 --- a/core/core.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import inspect -from enum import Enum -from configs import LONG_LINE -from core.helper import get_request, safe_load_texts -from core.print_formatters import print_err_response - - -class NodeStatuses(Enum): - """This class contains possible node statuses""" - ACTIVE = 0 - LEAVING = 1 - FROZEN = 2 - IN_MAINTENANCE = 3 - LEFT = 4 - NOT_CREATED = 5 - - -TEXTS = safe_load_texts() - - -def get_node_info(config, format): - status, payload = get_request('node_info') - if status == 'ok': - node_info = payload['node_info'] - if format == 'json': - print(node_info) - elif node_info['status'] == NodeStatuses.NOT_CREATED.value: - print(TEXTS['service']['node_not_registered']) - else: - print_node_info(node_info) - else: - print_err_response(payload) - - -def get_node_about(config, format): - status, payload = get_request('node_about') - if status == 'ok': - print(payload) - else: - print_err_response(payload) - - -def get_node_status(status): - node_status = NodeStatuses(status).name - return TEXTS['node']['status'][node_status] - - -def print_node_info(node): - print(inspect.cleandoc(f''' - {LONG_LINE} - Node info - Name: {node['name']} - IP: {node['ip']} - Public IP: {node['publicIP']} - Port: {node['port']} - Status: {get_node_status(int(node['status']))} - {LONG_LINE} - ''')) diff --git a/core/helper.py b/core/helper.py index e83ba5e3..41a7db09 100644 --- a/core/helper.py +++ b/core/helper.py @@ -190,7 +190,11 @@ def upload_certs(key_path, cert_path, force): None, json.dumps({'force': force}), 'application/json' ) - return post_request('ssl_upload', files=files_data) + return post_request( + blueprint='ssl', + method='upload', + files=files_data + ) def to_camel_case(snake_str): diff --git a/core/node.py b/core/node.py index e720028c..54be76af 100644 --- a/core/node.py +++ b/core/node.py @@ -23,6 +23,7 @@ import shlex import subprocess import time +from enum import Enum import docker @@ -38,7 +39,7 @@ from core.mysql_backup import create_mysql_backup, restore_mysql_backup from core.host import (is_node_inited, prepare_host, save_env_params, get_flask_secret_key) -from core.print_formatters import print_err_response, print_node_cmd_error +from core.print_formatters import print_err_response, print_node_cmd_error, print_node_info from core.resources import update_resource_allocation from tools.meta import update_meta from tools.helper import run_cmd, extract_env_params @@ -48,6 +49,17 @@ TEXTS = Texts() BASE_CONTAINERS_AMOUNT = 5 +BLUEPRINT_NAME = 'node' + + +class NodeStatuses(Enum): + """This class contains possible node statuses""" + ACTIVE = 0 + LEAVING = 1 + FROZEN = 2 + IN_MAINTENANCE = 3 + LEFT = 4 + NOT_CREATED = 5 def register_node(config, name, p2p_ip, @@ -64,8 +76,11 @@ def register_node(config, name, p2p_ip, 'gas_price': gas_price, 'skip_dry_run': skip_dry_run } - status, payload = post_request('create_node', - json=json_data) + status, payload = post_request( + blueprint=BLUEPRINT_NAME, + method='register', + json=json_data + ) if status == 'ok': msg = TEXTS['node']['registered'] logger.info(msg) @@ -198,7 +213,11 @@ def update(env_filepath, sync_schains): def get_node_signature(validator_id): params = {'validator_id': validator_id} - status, payload = get_request('node_signature', params=params) + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='signature', + params=params + ) if status == 'ok': return payload['signature'] else: @@ -239,7 +258,10 @@ def create_backup_archive(backup_filepath): def set_maintenance_mode_on(): print('Setting maintenance mode on...') - status, payload = post_request('maintenance_on') + status, payload = post_request( + blueprint=BLUEPRINT_NAME, + method='maintenance-on' + ) if status == 'ok': msg = TEXTS['node']['maintenance_on'] logger.info(msg) @@ -252,7 +274,10 @@ def set_maintenance_mode_on(): def set_maintenance_mode_off(): print('Setting maintenance mode off...') - status, payload = post_request('maintenance_off') + status, payload = post_request( + blueprint=BLUEPRINT_NAME, + method='maintenance-off' + ) if status == 'ok': msg = TEXTS['node']['maintenance_off'] logger.info(msg) @@ -318,3 +343,25 @@ def is_base_containers_alive(): lambda c: c.name.startswith('skale_'), containers )) return len(skale_containers) >= BASE_CONTAINERS_AMOUNT + + +def get_node_info(config, format): + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='info' + ) + if status == 'ok': + node_info = payload['node_info'] + if format == 'json': + print(node_info) + elif node_info['status'] == NodeStatuses.NOT_CREATED.value: + print(TEXTS['service']['node_not_registered']) + else: + print_node_info(node_info, get_node_status(int(node_info['status']))) + else: + print_err_response(payload) + + +def get_node_status(status): + node_status = NodeStatuses(status).name + return TEXTS['node']['status'][node_status] diff --git a/core/print_formatters.py b/core/print_formatters.py index 05b0fc69..4f0d95f0 100644 --- a/core/print_formatters.py +++ b/core/print_formatters.py @@ -272,3 +272,16 @@ def print_abi_validation_errors(info: list, raw: bool = False) -> None: def print_node_cmd_error(): print(TEXTS['node']['cmd_failed'].format(DEBUG_LOG_FILEPATH)) + + +def print_node_info(node, node_status): + print(inspect.cleandoc(f''' + {LONG_LINE} + Node info + Name: {node['name']} + IP: {node['ip']} + Public IP: {node['publicIP']} + Port: {node['port']} + Status: {node_status} + {LONG_LINE} + ''')) diff --git a/core/schains.py b/core/schains.py index 650bd7b4..907758d7 100644 --- a/core/schains.py +++ b/core/schains.py @@ -69,7 +69,11 @@ def show_config(name: str) -> None: def toggle_schain_repair_mode(schain: str) -> None: - status, payload = post_request('repair_schain', {'schain': schain}) + status, payload = post_request( + blueprint=BLUEPRINT_NAME, + method='repair', + json={'schain_name': schain} + ) if status == 'ok': print('Schain has been set for repair') else: @@ -77,7 +81,11 @@ def toggle_schain_repair_mode(schain: str) -> None: def describe(schain: str, raw=False) -> None: - status, payload = get_request('describe_schain', {'schain': schain}) + status, payload = get_request( + blueprint=BLUEPRINT_NAME, + method='get', + params={'schain_name': schain} + ) if status == 'ok': print_schain_info(payload, raw=raw) else: diff --git a/core/wallet.py b/core/wallet.py index 1594b042..81d899c4 100644 --- a/core/wallet.py +++ b/core/wallet.py @@ -23,11 +23,11 @@ from core.print_formatters import print_err_response, print_wallet_info, TEXTS -API_BLUEPRINT_NAME = 'wallet' +BLUEPRINT_NAME = 'wallet' def get_wallet_info(_format): - status, payload = get_request('wallet', 'info') + status, payload = get_request(BLUEPRINT_NAME, 'info') if status == 'ok': if _format == 'json': print(json.dumps(payload)) @@ -44,7 +44,7 @@ def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): 'gas_limit': gas_limit, 'gas_price': gas_price } - status, payload = post_request('wallet', 'send-eth', json=json_data) + status, payload = post_request(BLUEPRINT_NAME, 'send-eth', json=json_data) if status == 'ok': msg = TEXTS['wallet']['successful_transfer'] logger.info(msg) diff --git a/tests/routes_test.py b/tests/routes_test.py index 8d814fb8..bb6deaef 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -20,8 +20,8 @@ '/api/v1/schains/dkg-statuses', '/api/v1/schains/firewall-rules', '/api/v1/schains/repair', '/api/v1/schains/get', - '/api/v1/security/status', - '/api/v1/security/upload', + '/api/v1/ssl/status', + '/api/v1/ssl/upload', '/api/v1/wallet/info', '/api/v1/wallet/upload' ] From 3a07845a8648f3db0b2923348d2efc3c40113e2e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 3 Dec 2020 18:45:23 +0200 Subject: [PATCH 006/140] SKALE-3580 Fix tests --- tests/.skale/node_data/meta.json | 1 + tests/cli/containers_test.py | 139 ------------------------------- tests/cli/health_test.py | 103 +++++++++++++++++++++++ tests/cli/node_test.py | 31 +------ tests/cli/schains_test.py | 39 +-------- tests/cli/sgx_test.py | 41 --------- 6 files changed, 106 insertions(+), 248 deletions(-) create mode 100644 tests/.skale/node_data/meta.json delete mode 100644 tests/cli/containers_test.py create mode 100644 tests/cli/health_test.py delete mode 100644 tests/cli/sgx_test.py diff --git a/tests/.skale/node_data/meta.json b/tests/.skale/node_data/meta.json new file mode 100644 index 00000000..492fcaa9 --- /dev/null +++ b/tests/.skale/node_data/meta.json @@ -0,0 +1 @@ +{"version": "0.0.0", "config_stream": "master"} \ No newline at end of file diff --git a/tests/cli/containers_test.py b/tests/cli/containers_test.py deleted file mode 100644 index ce056bd3..00000000 --- a/tests/cli/containers_test.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import requests - -from tests.helper import response_mock, run_command_mock -from cli.containers import ls, schains - - -OK_RESPONSE_DATA = { - 'status': 'ok', - 'payload': [{ - 'image': 'image-skale', 'name': 'skale_schain_test', - 'state': { - 'Status': 'running', 'Running': True, 'Paused': False, - 'Restarting': False, 'OOMKilled': False, 'Dead': False, - 'Pid': 123, 'ExitCode': 0, 'Error': '', - 'StartedAt': '2019-10-08T13:59:54.52368097Z', - 'FinishedAt': '0001-01-01T00:00:00Z' - } - }, { - 'image': 'image-skale', 'name': 'skale_schain_test2', - 'state': { - 'Status': 'running', 'Running': False, 'Paused': True, - 'Restarting': False, 'OOMKilled': False, 'Dead': False, - 'Pid': 124, 'ExitCode': 0, 'Error': '', - 'StartedAt': '2019-10-08T13:59:54.52368099Z', - 'FinishedAt': '0001-01-01T00:00:00Z' - } - } - ]} - - -OK_LS_RESPONSE_DATA = { - 'status': 'ok', - 'payload': - {'containers': [{'image': 'skalenetwork/schain:1.46-develop.21', - 'name': 'skale_schain_shapely-alfecca-meridiana', - 'state': { - 'Status': 'running', 'Running': True, - 'Paused': False, 'Restarting': False, - 'OOMKilled': False, 'Dead': False, - 'Pid': 232, 'ExitCode': 0, - 'Error': '', - 'StartedAt': '2020-07-31T11:56:35.732888232Z', - 'FinishedAt': '0001-01-01T00:00:00Z'} - }, - {'image': 'skale-admin:latest', 'name': 'skale_api', - 'state': { - 'Status': 'running', - 'Running': True, 'Paused': False, - 'Restarting': False, 'OOMKilled': False, - 'Dead': False, 'Pid': 6710, 'ExitCode': 0, - 'Error': '', - 'StartedAt': '2020-07-31T11:55:17.28700307Z', - 'FinishedAt': '0001-01-01T00:00:00Z'}}] - }} - - -def test_schains_ok_response(config): - resp_mock = response_mock( - requests.codes.ok, - json_data=OK_RESPONSE_DATA - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, schains) - assert result.exit_code == 0 - - output_list = result.output.splitlines() - assert output_list[0] == ' Name Status Started At Image ' # noqa - assert output_list[1] == '-------------------------------------------------------------------------------------' # noqa - assert output_list[2] == 'skale_schain_test Running Oct 08 2019 13:59:54 image-skale' # noqa - assert output_list[3] == 'skale_schain_test2 Running (Jan 01 1 00:00:00) Oct 08 2019 13:59:54 image-skale' # noqa - - -def test_schain_error_response(config): - resp_mock = response_mock( - requests.codes.bad_request, - json_data={'status': 'error', 'payload': 'Operation failed'} - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, schains) - assert result.exit_code == 0 - print(repr(result.output)) - assert result.output == ('Command failed with following errors:\n' - '-----------------------------------------' - '---------\nOperation failed\n--------------------' - '------------------------------\n' - 'You can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n') # noqa - - -def test_schain_empty_response(config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, schains) - assert result.exit_code == 1 - assert result.output == '' - - -def test_schain_multi_error_response(config): - resp_mock = response_mock( - -1, - {'payload': ['Error test', 'Error test2'], 'status': 'error'} - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, schains) - assert result.exit_code == 0 - print(repr(result.output)) - assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nError test\nError test2\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa - - -def test_ls(): - resp_mock = response_mock( - requests.codes.ok, - json_data=OK_LS_RESPONSE_DATA - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, ls) - assert result.exit_code == 0 - assert result.output == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py new file mode 100644 index 00000000..60b1443d --- /dev/null +++ b/tests/cli/health_test.py @@ -0,0 +1,103 @@ +import requests + +from tests.helper import response_mock, run_command_mock +from cli.health import containers, schains_checks, sgx + + +OK_LS_RESPONSE_DATA = { + 'status': 'ok', + 'payload': + [ + { + 'image': 'skalenetwork/schain:1.46-develop.21', + 'name': 'skale_schain_shapely-alfecca-meridiana', + 'state': { + 'Status': 'running', 'Running': True, + 'Paused': False, 'Restarting': False, + 'OOMKilled': False, 'Dead': False, + 'Pid': 232, 'ExitCode': 0, + 'Error': '', + 'StartedAt': '2020-07-31T11:56:35.732888232Z', + 'FinishedAt': '0001-01-01T00:00:00Z' + } + }, + { + 'image': 'skale-admin:latest', 'name': 'skale_api', + 'state': { + 'Status': 'running', + 'Running': True, 'Paused': False, + 'Restarting': False, 'OOMKilled': False, + 'Dead': False, 'Pid': 6710, 'ExitCode': 0, + 'Error': '', + 'StartedAt': '2020-07-31T11:55:17.28700307Z', + 'FinishedAt': '0001-01-01T00:00:00Z' + } + } + ] +} + + +def test_containers(): + resp_mock = response_mock( + requests.codes.ok, + json_data=OK_LS_RESPONSE_DATA + ) + result = run_command_mock('core.helper.requests.get', + resp_mock, containers) + assert result.exit_code == 0 + assert result.output == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + + +def test_checks(): + payload = [ + { + "name": "test_schain", + "healthchecks": { + "data_dir": True, + "dkg": False, + "config": False, + "volume": False, + "container": False, + "ima_container": False, + "firewall_rules": False, + "rpc": False, + "blocks": False + } + } + ] + resp_mock = response_mock( + requests.codes.ok, + json_data={'payload': payload, 'status': 'ok'} + ) + result = run_command_mock('core.helper.requests.get', + resp_mock, schains_checks) + + print(result) + print(result.output) + + assert result.exit_code == 0 + assert result.output == 'sChain Name Data directory DKG Config file Volume Container IMA Firewall RPC Blocks\n-----------------------------------------------------------------------------------------------------------\ntest_schain True False False False False False False False False \n' # noqa + + result = run_command_mock('core.helper.requests.get', + resp_mock, schains_checks, ['--json']) + + assert result.exit_code == 0 + assert result.output == '[{"name": "test_schain", "healthchecks": {"data_dir": true, "dkg": false, "config": false, "volume": false, "container": false, "ima_container": false, "firewall_rules": false, "rpc": false, "blocks": false}}]\n' # noqa + + +def test_sgx_status(): + payload = { + 'sgx_server_url': 'https://127.0.0.1:1026', + 'sgx_wallet_version': '1.50.1-stable.0', + 'sgx_keyname': 'test_keyname', + 'status_name': 'CONNECTED' + } + resp_mock = response_mock( + requests.codes.ok, + json_data={'payload': payload, 'status': 'ok'} + ) + result = run_command_mock( + 'core.helper.requests.get', resp_mock, sgx) + + assert result.exit_code == 0 + assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status \x1b(0x\x1b(B CONNECTED \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 68969f17..8c143e6c 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -25,8 +25,7 @@ from configs import NODE_DATA_PATH, SKALE_DIR from core.resources import ResourceAlloc -from cli.node import (init_node, - node_about, node_info, register_node, signature, +from cli.node import (init_node, node_info, register_node, signature, update_node, backup_node, restore_node, set_node_in_maintenance, remove_node_from_maintenance, _turn_off, _turn_on) @@ -172,34 +171,6 @@ def test_update_node_without_init(config): assert result.output == "Node hasn't been inited before.\nYou should run < skale node init >\n" # noqa -def test_node_info_node_about(config): - payload = { - 'libraries': { - 'javascript': 'N/A', 'python': '0.89.0'}, - 'contracts': { - 'token': '0x3', - 'manager': '0x23' - }, - 'network': { - 'endpoint': 'ws://0.0.0.0:8080' - }, - 'local_wallet': { - 'address': '0xf', - 'eth_balance_wei': '15', - 'skale_balance_wei': '84312304', - 'eth_balance': '2.424', - 'skale_balance': '323.123' - } - } - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': payload} - ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_about) - assert result.exit_code == 0 - assert result.output == "{'libraries': {'javascript': 'N/A', 'python': '0.89.0'}, 'contracts': {'token': '0x3', 'manager': '0x23'}, 'network': {'endpoint': 'ws://0.0.0.0:8080'}, 'local_wallet': {'address': '0xf', 'eth_balance_wei': '15', 'skale_balance_wei': '84312304', 'eth_balance': '2.424', 'skale_balance': '323.123'}}\n" # noqa - - def test_node_info_node_info(config): payload = { 'node_info': { diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index b177345f..3fe2b823 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -23,7 +23,7 @@ import requests from tests.helper import response_mock, run_command_mock -from cli.schains import (get_schain_config, ls, dkg, checks, show_rules, +from cli.schains import (get_schain_config, ls, dkg, show_rules, repair, info_) @@ -151,43 +151,6 @@ def test_schain_rules(): assert result.output == 'Port Ip \n-----------------\n10000 127.0.0.1\n10001 127.0.0.1\n10002 None \n10003 None \n10004 127.0.0.1\n10005 127.0.0.1\n10007 None \n10008 None \n' # noqa -def test_checks(): - payload = [ - { - "name": "test_schain", - "healthchecks": { - "data_dir": True, - "dkg": False, - "config": False, - "volume": False, - "container": False, - "ima_container": False, - "firewall_rules": False, - "rpc": False, - "blocks": False - } - } - ] - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) - result = run_command_mock('core.helper.requests.get', - resp_mock, checks) - - print(result) - print(result.output) - - assert result.exit_code == 0 - assert result.output == 'sChain Name Data directory DKG Config file Volume Container IMA Firewall RPC Blocks\n-----------------------------------------------------------------------------------------------------------\ntest_schain True False False False False False False False False \n' # noqa - - result = run_command_mock('core.helper.requests.get', - resp_mock, checks, ['--json']) - - assert result.exit_code == 0 - assert result.output == '[{"name": "test_schain", "healthchecks": {"data_dir": true, "dkg": false, "config": false, "volume": false, "container": false, "ima_container": false, "firewall_rules": false, "rpc": false, "blocks": false}}]\n' # noqa - - def test_repair(): os.environ['TZ'] = 'Europe/London' time.tzset() diff --git a/tests/cli/sgx_test.py b/tests/cli/sgx_test.py deleted file mode 100644 index 06050bd3..00000000 --- a/tests/cli/sgx_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import requests - -from tests.helper import response_mock, run_command_mock -from cli.sgx import info - - -def test_sgx_status(): - payload = { - 'sgx_server_url': 'https://127.0.0.1:1026', - 'sgx_wallet_version': '1.50.1-stable.0', - 'sgx_keyname': 'test_keyname', - 'status_name': 'CONNECTED' - } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) - result = run_command_mock( - 'core.helper.requests.get', resp_mock, info) - - assert result.exit_code == 0 - assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status \x1b(0x\x1b(B CONNECTED \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa From d782a727a7af354995a75eb8cff5ba1acc9e50cc Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 3 Dec 2020 18:48:40 +0200 Subject: [PATCH 007/140] SKALE-3580 Remove skale.py --- cli/node.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/node.py b/cli/node.py index adb60ca1..d507d619 100644 --- a/cli/node.py +++ b/cli/node.py @@ -22,8 +22,6 @@ import click -from skale.utils.random_names.generator import generate_random_node_name - from core.node import (get_node_signature, init, restore, register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info) @@ -86,7 +84,7 @@ def node_info(format): @node.command('register', help="Register current node in the SKALE Manager") @click.option( '--name', '-n', - default=generate_random_node_name(), + prompt="Enter node name", help='SKALE node name' ) @click.option( From 7ef9a468dbc9b3b76384a097eb1a9c868c65b4db Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 14 Dec 2020 20:01:50 +0200 Subject: [PATCH 008/140] SKALE-3580 Rename routes --- README.md | 2 +- cli/exit.py | 6 +++--- cli/health.py | 2 +- configs/routes.py | 5 ++--- core/health.py | 2 +- tests/routes_test.py | 6 +++--- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1e836b6a..f427f7ab 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ Options: Show health check results for all SKALE Chains on the node ```shell -skale health schains-checks +skale health schains ``` Options: diff --git a/cli/exit.py b/cli/exit.py index 82316967..ad7a6b2f 100644 --- a/cli/exit.py +++ b/cli/exit.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) TEXTS = Texts() -BLUEPRINT_NAME = 'exit' +BLUEPRINT_NAME = 'node' @click.group() @@ -26,7 +26,7 @@ def node_exit(): def start(): status, payload = post_request( blueprint=BLUEPRINT_NAME, - method='start' + method='exit/start' ) if status == 'ok': msg = TEXTS['exit']['start'] @@ -41,7 +41,7 @@ def start(): def status(format): status, payload = get_request( blueprint=BLUEPRINT_NAME, - method='status' + method='exit/status' ) if status == 'ok': exit_status = payload diff --git a/cli/health.py b/cli/health.py index 86152273..884a20e3 100644 --- a/cli/health.py +++ b/cli/health.py @@ -50,7 +50,7 @@ def containers(all): help=G_TEXTS['common']['json']['help'], is_flag=True ) -def schains_checks(json_format: bool) -> None: +def schains(json_format: bool) -> None: get_schains_checks(json_format) diff --git a/configs/routes.py b/configs/routes.py index 16485e87..3c65396c 100644 --- a/configs/routes.py +++ b/configs/routes.py @@ -26,10 +26,9 @@ ROUTES = { 'v1': { 'logs': ['dump'], - 'exit': ['start', 'status'], 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'signature', - 'send-tg-notification'], - 'health': ['containers', 'schains-checks', 'sgx'], + 'send-tg-notification', 'exit/start', 'exit/status'], + 'health': ['containers', 'schains', 'sgx'], 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'] diff --git a/core/health.py b/core/health.py index fce69ed3..c15f3263 100644 --- a/core/health.py +++ b/core/health.py @@ -45,7 +45,7 @@ def get_containers(_all): def get_schains_checks(json_format: bool = False) -> None: status, payload = get_request( blueprint=BLUEPRINT_NAME, - method='schains-checks' + method='schains' ) if status == 'ok': if not payload: diff --git a/tests/routes_test.py b/tests/routes_test.py index bb6deaef..fa2b184f 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -5,15 +5,15 @@ ALL_V1_ROUTES = [ '/api/v1/logs/dump', - '/api/v1/exit/start', - '/api/v1/exit/status', + '/api/v1/node/exit/start', + '/api/v1/node/exit/status', '/api/v1/node/info', '/api/v1/node/register', '/api/v1/node/maintenance-on', '/api/v1/node/maintenance-off', '/api/v1/node/send-tg-notification', '/api/v1/health/containers', - '/api/v1/health/schains-checks', + '/api/v1/health/schains', '/api/v1/health/sgx', '/api/v1/schains/config', '/api/v1/schains/list', From 1f2fab82d39a46f6cec4f760554137cdd80211c0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 14 Dec 2020 20:02:33 +0200 Subject: [PATCH 009/140] SKALE-3580 Fix print_node_info --- core/print_formatters.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/print_formatters.py b/core/print_formatters.py index 165529c2..b89204fa 100644 --- a/core/print_formatters.py +++ b/core/print_formatters.py @@ -273,3 +273,16 @@ def print_abi_validation_errors(info: list, raw: bool = False) -> None: def print_node_cmd_error(): print(TEXTS['node']['cmd_failed'].format(DEBUG_LOG_FILEPATH)) + + +def print_node_info(node, node_status): + print(inspect.cleandoc(f''' + {LONG_LINE} + Node info + Name: {node['name']} + IP: {node['ip']} + Public IP: {node['publicIP']} + Port: {node['port']} + Status: {node_status} + {LONG_LINE} + ''')) From 372f3d3f7f78bf46e67d4bacfeb953f14c5ebd34 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 14 Dec 2020 20:07:38 +0200 Subject: [PATCH 010/140] SKALE-3580 Fix naming --- tests/cli/health_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 60b1443d..cb5843bd 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -1,7 +1,7 @@ import requests from tests.helper import response_mock, run_command_mock -from cli.health import containers, schains_checks, sgx +from cli.health import containers, schains, sgx OK_LS_RESPONSE_DATA = { @@ -70,7 +70,7 @@ def test_checks(): json_data={'payload': payload, 'status': 'ok'} ) result = run_command_mock('core.helper.requests.get', - resp_mock, schains_checks) + resp_mock, schains) print(result) print(result.output) @@ -79,7 +79,7 @@ def test_checks(): assert result.output == 'sChain Name Data directory DKG Config file Volume Container IMA Firewall RPC Blocks\n-----------------------------------------------------------------------------------------------------------\ntest_schain True False False False False False False False False \n' # noqa result = run_command_mock('core.helper.requests.get', - resp_mock, schains_checks, ['--json']) + resp_mock, schains, ['--json']) assert result.exit_code == 0 assert result.output == '[{"name": "test_schain", "healthchecks": {"data_dir": true, "dkg": false, "config": false, "volume": false, "container": false, "ima_container": false, "firewall_rules": false, "rpc": false, "blocks": false}}]\n' # noqa From f231a43284395e8d7a7a69b201fb501569677118 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 14 Dec 2020 20:17:49 +0200 Subject: [PATCH 011/140] SKALE-3580 Fix routes test --- tests/routes_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/routes_test.py b/tests/routes_test.py index fa2b184f..00e8a60b 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -5,25 +5,32 @@ ALL_V1_ROUTES = [ '/api/v1/logs/dump', - '/api/v1/node/exit/start', - '/api/v1/node/exit/status', + '/api/v1/node/info', '/api/v1/node/register', '/api/v1/node/maintenance-on', '/api/v1/node/maintenance-off', + '/api/v1/node/signature', '/api/v1/node/send-tg-notification', + '/api/v1/node/exit/start', + '/api/v1/node/exit/status', + '/api/v1/health/containers', '/api/v1/health/schains', '/api/v1/health/sgx', + '/api/v1/schains/config', '/api/v1/schains/list', '/api/v1/schains/dkg-statuses', '/api/v1/schains/firewall-rules', - '/api/v1/schains/repair', '/api/v1/schains/get', + '/api/v1/schains/repair', + '/api/v1/schains/get', + '/api/v1/ssl/status', '/api/v1/ssl/upload', + '/api/v1/wallet/info', - '/api/v1/wallet/upload' + '/api/v1/wallet/send-eth' ] From 816ebb53f0a61d4a857cf856314e8b0795b30a68 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 23 Dec 2020 18:07:59 +0200 Subject: [PATCH 012/140] SKALE-3333 Add exit codes to cli commands --- README.md | 13 ++- cli/exit.py | 32 +++++- cli/logs.py | 7 +- cli/node.py | 3 +- cli/resources_allocation.py | 3 +- cli/schains.py | 2 +- cli/ssl.py | 8 +- cli/wallet.py | 2 +- core/health.py | 11 +- core/helper.py | 216 ------------------------------------ core/host.py | 3 +- core/node.py | 19 ++-- core/print_formatters.py | 28 ++--- core/resources.py | 3 +- core/schains.py | 16 +-- core/wallet.py | 8 +- main.py | 3 +- tests/resources_test.py | 3 +- tools/exit_codes.py | 28 +++++ tools/helper.py | 192 +++++++++++++++++++++++++++++++- 20 files changed, 316 insertions(+), 284 deletions(-) delete mode 100644 core/helper.py create mode 100644 tools/exit_codes.py diff --git a/README.md b/README.md index f427f7ab..cb4f4304 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 2.7 [Logs](#logs-commands) 2.8 [Resources allocation](#resources-allocation-commands) 2.9 [Validate](#validate-commands) - -3. [Development](#development) +3. [Exit codes](#exit-codes) +4. [Development](#development) ## Installation @@ -479,6 +479,15 @@ Options: - `--json` - show validation result in json format +## Exit codes + +Exit codes conventions for SKALE CLI tools + +- `0` - Everything is OK +- `1` - General error exit code +- `3` - Bad API response +- `4` - Script execution error + ## Development ### Setup repo diff --git a/cli/exit.py b/cli/exit.py index ad7a6b2f..0d0599a2 100644 --- a/cli/exit.py +++ b/cli/exit.py @@ -1,7 +1,29 @@ -import click +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import logging -from core.helper import get_request, post_request, abort_if_false -from core.print_formatters import print_err_response, print_exit_status + +import click + +from core.print_formatters import print_exit_status +from tools.helper import error_exit, get_request, post_request, abort_if_false +from tools.exit_codes import CLIExitCodes from tools.texts import Texts logger = logging.getLogger(__name__) @@ -33,7 +55,7 @@ def start(): logger.info(msg) print(msg) else: - print_err_response(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) @node_exit.command('status', help="Get exit process status") @@ -50,7 +72,7 @@ def status(format): else: print_exit_status(exit_status) else: - print_err_response(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) @node_exit.command('finalize', help="Finalize exit process") diff --git a/cli/logs.py b/cli/logs.py index 3d345e89..e0a5e338 100644 --- a/cli/logs.py +++ b/cli/logs.py @@ -17,9 +17,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import sys + import click -from core.helper import download_dump +from tools.helper import download_dump from configs.cli_logger import LOG_FILEPATH, DEBUG_LOG_FILEPATH +from tools.exit_codes import CLIExitCodes @click.group() @@ -52,3 +55,5 @@ def dump(container, path): res = download_dump(path, container) if res: print(f'File {res} downloaded') + else: + sys.exit(CLIExitCodes.BAD_API_RESPONSE) diff --git a/cli/node.py b/cli/node.py index 6a276989..510030d8 100644 --- a/cli/node.py +++ b/cli/node.py @@ -25,9 +25,8 @@ from core.node import (get_node_signature, init, restore, register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info) -from core.helper import abort_if_false, safe_load_texts from configs import DEFAULT_NODE_BASE_PORT -from tools.helper import session_config +from tools.helper import session_config, abort_if_false, safe_load_texts config = session_config() diff --git a/cli/resources_allocation.py b/cli/resources_allocation.py index 83307a89..fa9875d2 100644 --- a/cli/resources_allocation.py +++ b/cli/resources_allocation.py @@ -22,8 +22,7 @@ from core.resources import (get_resource_allocation_info, generate_resource_allocation_config) -from core.helper import abort_if_false, safe_load_texts -from tools.helper import session_config +from tools.helper import session_config, abort_if_false, safe_load_texts config = session_config() TEXTS = safe_load_texts() diff --git a/cli/schains.py b/cli/schains.py index 15ba95ae..1fb34e5e 100644 --- a/cli/schains.py +++ b/cli/schains.py @@ -19,7 +19,7 @@ import click -from core.helper import abort_if_false +from tools.helper import abort_if_false from core.schains import ( describe, get_schain_firewall_rules, diff --git a/cli/ssl.py b/cli/ssl.py index ccbe4763..c5322f7c 100644 --- a/cli/ssl.py +++ b/cli/ssl.py @@ -20,8 +20,8 @@ import click from terminaltables import SingleTable -from core.helper import (get_request, safe_load_texts, upload_certs, - print_err_response) +from tools.helper import (get_request, safe_load_texts, upload_certs, + error_exit) TEXTS = safe_load_texts() @@ -56,7 +56,7 @@ def status(): print('SSL certificates status:') print(table.table) else: - print_err_response(payload) + error_exit(payload) @ssl.command(help="Upload new SSL certificates") @@ -76,4 +76,4 @@ def upload(key_path, cert_path, force): if status == 'ok': print(TEXTS['ssl']['uploaded']) else: - print_err_response(payload) + error_exit(payload) diff --git a/cli/wallet.py b/cli/wallet.py index deed279f..365853ca 100644 --- a/cli/wallet.py +++ b/cli/wallet.py @@ -21,7 +21,7 @@ import click -from core.helper import abort_if_false +from tools.helper import abort_if_false from core.wallet import get_wallet_info, send_eth logger = logging.getLogger(__name__) diff --git a/core/health.py b/core/health.py index c15f3263..f0cbb586 100644 --- a/core/health.py +++ b/core/health.py @@ -20,12 +20,13 @@ import json from terminaltables import SingleTable -from core.helper import get_request from core.print_formatters import ( print_containers, - print_err_response, print_schains_healthchecks ) +from tools.helper import error_exit, get_request +from tools.exit_codes import CLIExitCodes + BLUEPRINT_NAME = 'health' @@ -39,7 +40,7 @@ def get_containers(_all): if status == 'ok': print_containers(payload) else: - print_err_response(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def get_schains_checks(json_format: bool = False) -> None: @@ -56,7 +57,7 @@ def get_schains_checks(json_format: bool = False) -> None: else: print_schains_healthchecks(payload) else: - print_err_response(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def get_sgx_info(): @@ -76,4 +77,4 @@ def get_sgx_info(): table = SingleTable(table_data) print(table.table) else: - print_err_response(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/core/helper.py b/core/helper.py deleted file mode 100644 index 41a7db09..00000000 --- a/core/helper.py +++ /dev/null @@ -1,216 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import os -import re -import shutil -from functools import wraps -import urllib.parse - -import logging -import logging.handlers as py_handlers -from logging import Formatter - -import requests -import yaml - -from configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT -from configs.routes import get_route -from configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, - LOG_FILE_SIZE_BYTES, - LOG_FILEPATH, DEBUG_LOG_FILEPATH) -from core.print_formatters import print_err_response -from tools.helper import session_config - - -config = session_config() -logger = logging.getLogger(__name__) - - -HOST = f'http://{ADMIN_HOST}:{ADMIN_PORT}' - -DEFAULT_ERROR_DATA = { - 'status': 'error', - 'payload': 'Request failed. Check skale_api container logs' -} - - -def safe_get_config(config, key): - try: - return config[key] - except KeyError as e: - logger.error(e) - # print(f'No such key in config: {key}') - return None - - -def no_node(f): - @wraps(f) - def inner(*args, **kwargs): - # todo: check that node is not installed yet! - return f(*args, **kwargs) - - return inner - - -def safe_load_texts(): - with open(TEXT_FILE, 'r') as stream: - try: - return yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - -def safe_load_yml(filepath): - with open(filepath, 'r') as stream: - try: - return yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - -def construct_url(route): - return urllib.parse.urljoin(HOST, route) - - -def abort_if_false(ctx, param, value): - if not value: - ctx.abort() - - -def post_request(blueprint, method, json=None, files=None): - route = get_route(blueprint, method) - url = construct_url(route) - try: - response = requests.post(url, json=json, files=files) - data = response.json() - except Exception as err: - logger.error('Request failed', exc_info=err) - data = DEFAULT_ERROR_DATA - status = data['status'] - payload = data['payload'] - return status, payload - - -def get_request(blueprint, method, params=None): - route = get_route(blueprint, method) - url = construct_url(route) - try: - response = requests.get(url, params=params) - data = response.json() - except Exception as err: - logger.error('Request failed', exc_info=err) - data = DEFAULT_ERROR_DATA - - status = data['status'] - payload = data['payload'] - return status, payload - - -def download_dump(path, container_name=None): - route = get_route('logs', 'dump') - url = construct_url(route) - params = {} - if container_name: - params['container_name'] = container_name - with requests.get(url, params=params, stream=True) as r: - if r is None: - return None - if r.status_code != requests.codes.ok: # pylint: disable=no-member - print('Request failed, status code:', r.status_code) - print_err_response(r.json()) - return None - d = r.headers['Content-Disposition'] - fname_q = re.findall("filename=(.+)", d)[0] - fname = fname_q.replace('"', '') - filepath = os.path.join(path, fname) - with open(filepath, 'wb') as f: - shutil.copyfileobj(r.raw, f) - return fname - - -def init_default_logger(): - f_handler = get_file_handler(LOG_FILEPATH, logging.INFO) - debug_f_handler = get_file_handler(DEBUG_LOG_FILEPATH, logging.DEBUG) - logging.basicConfig(level=logging.DEBUG, handlers=[ - f_handler, debug_f_handler]) - - -def get_file_handler(log_filepath, log_level): - formatter = Formatter(LOG_FORMAT) - f_handler = py_handlers.RotatingFileHandler( - log_filepath, maxBytes=LOG_FILE_SIZE_BYTES, - backupCount=LOG_BACKUP_COUNT) - f_handler.setFormatter(formatter) - f_handler.setLevel(log_level) - - return f_handler - - -def load_ssl_files(key_path, cert_path): - return { - 'ssl_key': (os.path.basename(key_path), - read_file(key_path), 'application/octet-stream'), - 'ssl_cert': (os.path.basename(cert_path), - read_file(cert_path), 'application/octet-stream') - } - - -def read_file(path, mode='rb'): - with open(path, mode) as f: - return f - - -def upload_certs(key_path, cert_path, force): - with open(key_path, 'rb') as key_file, open(cert_path, 'rb') as cert_file: - files_data = { - 'ssl_key': (os.path.basename(key_path), key_file, - 'application/octet-stream'), - 'ssl_cert': (os.path.basename(cert_path), cert_file, - 'application/octet-stream') - } - files_data['json'] = ( - None, json.dumps({'force': force}), - 'application/json' - ) - return post_request( - blueprint='ssl', - method='upload', - files=files_data - ) - - -def to_camel_case(snake_str): - components = snake_str.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) - - -def validate_abi(abi_filepath: str) -> dict: - if not os.path.isfile(abi_filepath): - return {'filepath': abi_filepath, - 'status': 'error', - 'msg': 'No such file'} - try: - with open(abi_filepath) as abi_file: - json.load(abi_file) - except Exception: - return {'filepath': abi_filepath, 'status': 'error', - 'msg': 'Failed to load abi file as json'} - return {'filepath': abi_filepath, 'status': 'ok', 'msg': ''} diff --git a/core/host.py b/core/host.py index fedbb49e..88f529ae 100644 --- a/core/host.py +++ b/core/host.py @@ -23,7 +23,6 @@ from shutil import copyfile from urllib.parse import urlparse -from core.helper import validate_abi from core.resources import update_resource_allocation from configs import (ADMIN_PORT, @@ -37,7 +36,7 @@ from configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, SGX_SERVER_URL_FILEPATH) -from core.helper import safe_load_texts +from tools.helper import safe_load_texts, validate_abi TEXTS = safe_load_texts() diff --git a/core/node.py b/core/node.py index 3102b6f9..3200692c 100644 --- a/core/node.py +++ b/core/node.py @@ -35,15 +35,16 @@ TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) from configs.cli_logger import LOG_DIRNAME -from core.helper import get_request, post_request from core.mysql_backup import create_mysql_backup, restore_mysql_backup from core.host import (is_node_inited, prepare_host, save_env_params, get_flask_secret_key) -from core.print_formatters import print_err_response, print_node_cmd_error, print_node_info +from core.print_formatters import print_node_cmd_error, print_node_info +from tools.helper import error_exit, get_request, post_request from core.resources import update_resource_allocation from tools.meta import update_meta from tools.helper import run_cmd, extract_env_params from tools.texts import Texts +from tools.exit_codes import CLIExitCodes logger = logging.getLogger(__name__) TEXTS = Texts() @@ -88,7 +89,7 @@ def register_node(config, name, p2p_ip, else: error_msg = payload logger.error(f'Registration error {error_msg}') - print_err_response(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) def init(env_filepath, dry_run=False): @@ -114,9 +115,9 @@ def init(env_filepath, dry_run=False): try: run_cmd(['bash', INSTALL_SCRIPT], env=env) except Exception: - logger.exception('Install script process errored') - print_node_cmd_error() - return + error_msg = 'Install script process errored' + logger.exception(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) print('Waiting for transaction manager initialization ...') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): @@ -273,7 +274,7 @@ def set_maintenance_mode_on(): else: error_msg = payload logger.error(f'Set maintenance mode error {error_msg}') - print_err_response(error_msg) + error_exit(error_msg) def set_maintenance_mode_off(): @@ -289,7 +290,7 @@ def set_maintenance_mode_off(): else: error_msg = payload logger.error(f'Remove from maintenance mode error {error_msg}') - print_err_response(error_msg) + error_exit(error_msg) def run_turn_off_script(): @@ -363,7 +364,7 @@ def get_node_info(config, format): else: print_node_info(node_info, get_node_status(int(node_info['status']))) else: - print_err_response(payload) + error_exit(payload) def get_node_status(status): diff --git a/core/print_formatters.py b/core/print_formatters.py index b89204fa..03ee2638 100644 --- a/core/print_formatters.py +++ b/core/print_formatters.py @@ -17,8 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import os +import json import datetime import texttable from dateutil import parser @@ -32,19 +32,6 @@ TEXTS = Texts() -def print_err_response(error_payload): - if isinstance(error_payload, list): - error_msg = '\n'.join(error_payload) - else: - error_msg = error_payload - - print('Command failed with following errors:') - print(LONG_LINE) - print(error_msg) - print(LONG_LINE) - print(f'You can find more info in {DEBUG_LOG_FILEPATH}') - - def print_wallet_info(wallet): print(inspect.cleandoc(f''' {LONG_LINE} @@ -286,3 +273,16 @@ def print_node_info(node, node_status): Status: {node_status} {LONG_LINE} ''')) + + +def print_err_response(error_payload): + if isinstance(error_payload, list): + error_msg = '\n'.join(error_payload) + else: + error_msg = error_payload + + print('Command failed with following errors:') + print(LONG_LINE) + print(error_msg) + print(LONG_LINE) + print(f'You can find more info in {DEBUG_LOG_FILEPATH}') diff --git a/core/resources.py b/core/resources.py index 3cab5ce4..f2121ac4 100644 --- a/core/resources.py +++ b/core/resources.py @@ -25,8 +25,7 @@ import psutil from tools.schain_types import SchainTypes -from tools.helper import write_json, read_json, run_cmd, format_output -from core.helper import safe_load_yml +from tools.helper import write_json, read_json, run_cmd, format_output, safe_load_yml from configs import ALLOCATION_FILEPATH from configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, diff --git a/core/schains.py b/core/schains.py index 907758d7..0b018d44 100644 --- a/core/schains.py +++ b/core/schains.py @@ -1,10 +1,10 @@ import logging import pprint -from core.helper import get_request, post_request +from tools.helper import get_request, post_request from core.print_formatters import ( print_dkg_statuses, - print_err_response, + error_exit, print_firewall_rules, print_schain_info, print_schains @@ -25,7 +25,7 @@ def get_schain_firewall_rules(schain: str) -> None: if status == 'ok': print_firewall_rules(payload['endpoints']) else: - print_err_response(payload) + error_exit(payload) def show_schains() -> None: @@ -40,7 +40,7 @@ def show_schains() -> None: return print_schains(schains) else: - print_err_response(payload) + error_exit(payload) def show_dkg_info(all_: bool = False) -> None: @@ -53,7 +53,7 @@ def show_dkg_info(all_: bool = False) -> None: if status == 'ok': print_dkg_statuses(payload) else: - print_err_response(payload) + error_exit(payload) def show_config(name: str) -> None: @@ -65,7 +65,7 @@ def show_config(name: str) -> None: if status == 'ok': pprint.pprint(payload) else: - print_err_response(payload) + error_exit(payload) def toggle_schain_repair_mode(schain: str) -> None: @@ -77,7 +77,7 @@ def toggle_schain_repair_mode(schain: str) -> None: if status == 'ok': print('Schain has been set for repair') else: - print_err_response(payload) + error_exit(payload) def describe(schain: str, raw=False) -> None: @@ -89,4 +89,4 @@ def describe(schain: str, raw=False) -> None: if status == 'ok': print_schain_info(payload, raw=raw) else: - print_err_response(payload) + error_exit(payload) diff --git a/core/wallet.py b/core/wallet.py index 81d899c4..f3a3579d 100644 --- a/core/wallet.py +++ b/core/wallet.py @@ -19,8 +19,8 @@ import json -from core.helper import get_request, post_request, logger -from core.print_formatters import print_err_response, print_wallet_info, TEXTS +from core.print_formatters import print_wallet_info, TEXTS +from tools.helper import error_exit, get_request, post_request, logger BLUEPRINT_NAME = 'wallet' @@ -34,7 +34,7 @@ def get_wallet_info(_format): else: print_wallet_info(payload) else: - print_err_response(payload) + error_exit(payload) def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): @@ -52,4 +52,4 @@ def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): else: error_msg = payload logger.error(f'Sending error {error_msg}') - print_err_response(error_msg) + error_exit(error_msg) diff --git a/main.py b/main.py index a527120a..7fdf4a3d 100644 --- a/main.py +++ b/main.py @@ -36,8 +36,7 @@ from cli.exit import exit_cli from cli.validate import validate_cli from cli.resources_allocation import resources_allocation_cli - -from core.helper import (safe_load_texts, init_default_logger) +from tools.helper import safe_load_texts, init_default_logger from configs import LONG_LINE from core.host import init_logs_dir diff --git a/tests/resources_test.py b/tests/resources_test.py index 93cc79d7..3358ad71 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -13,8 +13,7 @@ ResourceAlloc, SChainVolumeAlloc ) -from core.helper import safe_load_yml -from tools.helper import write_json +from tools.helper import write_json, safe_load_yml SCHAIN_VOLUME_PARTS = {'test4': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'test': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'small': {'max_consensus_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 0, 'max_file_storage_bytes': 0, 'max_reserved_storage_bytes': 0}, 'medium': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'large': {'max_consensus_storage_bytes': 38, 'max_skaled_leveldb_storage_bytes': 38, 'max_file_storage_bytes': 38, 'max_reserved_storage_bytes': 12}} # noqa diff --git a/tools/exit_codes.py b/tools/exit_codes.py new file mode 100644 index 00000000..959f3fd6 --- /dev/null +++ b/tools/exit_codes.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from enum import IntEnum + + +class CLIExitCodes(IntEnum): + """This class contains exit codes for SKALE CLI tools""" + SUCCESS = 0 + FAILURE = 1 + BAD_API_RESPONSE = 3 + SCRIPT_EXECUTION_ERROR = 4 diff --git a/tools/helper.py b/tools/helper.py index cdf80eca..41fb40db 100644 --- a/tools/helper.py +++ b/tools/helper.py @@ -18,24 +18,49 @@ # along with this program. If not, see . import os +import re +import sys import json -import logging +import yaml +import shutil +import requests import subprocess import urllib.request from subprocess import PIPE +from functools import wraps + +import logging +from logging import Formatter +import logging.handlers as py_handlers import click from jinja2 import Environment from readsettings import ReadSettings +from core.print_formatters import print_err_response +from tools.exit_codes import CLIExitCodes + from configs.env import (absent_params as absent_env_params, get_params as get_env_params) -from configs import CONFIG_FILEPATH +from configs import CONFIG_FILEPATH, TEXT_FILE, ADMIN_HOST, ADMIN_PORT +from configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, + LOG_FILE_SIZE_BYTES, + LOG_FILEPATH, DEBUG_LOG_FILEPATH) +from configs.routes import get_route + logger = logging.getLogger(__name__) +HOST = f'http://{ADMIN_HOST}:{ADMIN_PORT}' + +DEFAULT_ERROR_DATA = { + 'status': 'error', + 'payload': 'Request failed. Check skale_api container logs' +} + + def read_json(path): with open(path, encoding='utf-8') as data_file: return json.loads(data_file.read()) @@ -113,3 +138,166 @@ def extract_env_params(env_filepath): err=True) return None return env_params + + +def error_exit(error_payload, exit_code=CLIExitCodes.FAILURE): + print_err_response(error_payload) + sys.exit(exit_code) + + +def safe_get_config(config, key): + try: + return config[key] + except KeyError as e: + logger.error(e) + return None + + +def no_node(f): + @wraps(f) + def inner(*args, **kwargs): + # todo: check that node is not installed yet! + return f(*args, **kwargs) + + return inner + + +def safe_load_texts(): + with open(TEXT_FILE, 'r') as stream: + try: + return yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + +def safe_load_yml(filepath): + with open(filepath, 'r') as stream: + try: + return yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + +def construct_url(route): + return urllib.parse.urljoin(HOST, route) + + +def abort_if_false(ctx, param, value): + if not value: + ctx.abort() + + +def post_request(blueprint, method, json=None, files=None): + route = get_route(blueprint, method) + url = construct_url(route) + try: + response = requests.post(url, json=json, files=files) + data = response.json() + except Exception as err: + logger.error('Request failed', exc_info=err) + data = DEFAULT_ERROR_DATA + status = data['status'] + payload = data['payload'] + return status, payload + + +def get_request(blueprint, method, params=None): + route = get_route(blueprint, method) + url = construct_url(route) + try: + response = requests.get(url, params=params) + data = response.json() + except Exception as err: + logger.error('Request failed', exc_info=err) + data = DEFAULT_ERROR_DATA + + status = data['status'] + payload = data['payload'] + return status, payload + + +def download_dump(path, container_name=None): + route = get_route('logs', 'dump') + url = construct_url(route) + params = {} + if container_name: + params['container_name'] = container_name + with requests.get(url, params=params, stream=True) as r: + if r is None: + return None + if r.status_code != requests.codes.ok: # pylint: disable=no-member + print('Request failed, status code:', r.status_code) + error_exit(r.json()) + return None + d = r.headers['Content-Disposition'] + fname_q = re.findall("filename=(.+)", d)[0] + fname = fname_q.replace('"', '') + filepath = os.path.join(path, fname) + with open(filepath, 'wb') as f: + shutil.copyfileobj(r.raw, f) + return fname + + +def init_default_logger(): + f_handler = get_file_handler(LOG_FILEPATH, logging.INFO) + debug_f_handler = get_file_handler(DEBUG_LOG_FILEPATH, logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, handlers=[ + f_handler, debug_f_handler]) + + +def get_file_handler(log_filepath, log_level): + formatter = Formatter(LOG_FORMAT) + f_handler = py_handlers.RotatingFileHandler( + log_filepath, maxBytes=LOG_FILE_SIZE_BYTES, + backupCount=LOG_BACKUP_COUNT) + f_handler.setFormatter(formatter) + f_handler.setLevel(log_level) + + return f_handler + + +def load_ssl_files(key_path, cert_path): + return { + 'ssl_key': (os.path.basename(key_path), + read_file(key_path), 'application/octet-stream'), + 'ssl_cert': (os.path.basename(cert_path), + read_file(cert_path), 'application/octet-stream') + } + + +def upload_certs(key_path, cert_path, force): + with open(key_path, 'rb') as key_file, open(cert_path, 'rb') as cert_file: + files_data = { + 'ssl_key': (os.path.basename(key_path), key_file, + 'application/octet-stream'), + 'ssl_cert': (os.path.basename(cert_path), cert_file, + 'application/octet-stream') + } + files_data['json'] = ( + None, json.dumps({'force': force}), + 'application/json' + ) + return post_request( + blueprint='ssl', + method='upload', + files=files_data + ) + + +def to_camel_case(snake_str): + components = snake_str.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + + +def validate_abi(abi_filepath: str) -> dict: + if not os.path.isfile(abi_filepath): + return {'filepath': abi_filepath, + 'status': 'error', + 'msg': 'No such file'} + try: + with open(abi_filepath) as abi_file: + json.load(abi_file) + except Exception: + return {'filepath': abi_filepath, 'status': 'error', + 'msg': 'Failed to load abi file as json'} + return {'filepath': abi_filepath, 'status': 'ok', 'msg': ''} From 13aa06266e57d46046edb44e36f123afcb33b195 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 23 Dec 2020 18:58:00 +0200 Subject: [PATCH 013/140] SKALE-3333 Fix tests, upd error codes --- cli/ssl.py | 5 ++-- core/node.py | 22 +++++++-------- core/schains.py | 16 +++++------ core/wallet.py | 8 +++--- tests/cli/exit_test.py | 2 +- tests/cli/health_test.py | 8 +++--- tests/cli/node_test.py | 38 +++++++++++++------------- tests/cli/resources_allocation_test.py | 8 +++--- tests/cli/schains_test.py | 18 ++++++------ tests/cli/wallet_test.py | 8 +++--- tools/helper.py | 2 +- 11 files changed, 67 insertions(+), 68 deletions(-) diff --git a/cli/ssl.py b/cli/ssl.py index c5322f7c..663839cb 100644 --- a/cli/ssl.py +++ b/cli/ssl.py @@ -20,6 +20,7 @@ import click from terminaltables import SingleTable +from tools.exit_codes import CLIExitCodes from tools.helper import (get_request, safe_load_texts, upload_certs, error_exit) @@ -56,7 +57,7 @@ def status(): print('SSL certificates status:') print(table.table) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) @ssl.command(help="Upload new SSL certificates") @@ -76,4 +77,4 @@ def upload(key_path, cert_path, force): if status == 'ok': print(TEXTS['ssl']['uploaded']) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/core/node.py b/core/node.py index 3200692c..9f3d3be0 100644 --- a/core/node.py +++ b/core/node.py @@ -121,8 +121,7 @@ def init(env_filepath, dry_run=False): print('Waiting for transaction manager initialization ...') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): - print_node_cmd_error() - return + error_exit('Containers are not running', exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) logger.info('Generating resource allocation file ...') update_resource_allocation() print('Init procedure finished') @@ -274,7 +273,7 @@ def set_maintenance_mode_on(): else: error_msg = payload logger.error(f'Set maintenance mode error {error_msg}') - error_exit(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) def set_maintenance_mode_off(): @@ -290,7 +289,7 @@ def set_maintenance_mode_off(): else: error_msg = payload logger.error(f'Remove from maintenance mode error {error_msg}') - error_exit(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) def run_turn_off_script(): @@ -302,9 +301,9 @@ def run_turn_off_script(): try: run_cmd(['bash', TURN_OFF_SCRIPT], env=cmd_env) except Exception: - logger.exception('Turning off failed') - print_node_cmd_error() - return + error_msg = 'Turning off failed' + logger.exception(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) print('Node was successfully turned off') @@ -314,9 +313,9 @@ def run_turn_on_script(sync_schains, env_filepath): try: run_cmd(['bash', TURN_ON_SCRIPT], env=env) except Exception: - logger.exception('Turning on failed') - print_node_cmd_error() - return + error_msg = 'Turning on failed' + logger.exception(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) print('Waiting for transaction manager initialization ...') time.sleep(TM_INIT_TIMEOUT) print('Node was successfully turned on') @@ -336,7 +335,6 @@ def turn_on(maintenance_off, sync_schains, env_file): print(TEXTS['node']['not_inited']) return run_turn_on_script(sync_schains, env_file) - # TODO: Handle error from turn on script if maintenance_off: set_maintenance_mode_off() @@ -364,7 +362,7 @@ def get_node_info(config, format): else: print_node_info(node_info, get_node_status(int(node_info['status']))) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def get_node_status(status): diff --git a/core/schains.py b/core/schains.py index 0b018d44..2075d051 100644 --- a/core/schains.py +++ b/core/schains.py @@ -1,10 +1,10 @@ import logging import pprint -from tools.helper import get_request, post_request +from tools.helper import get_request, post_request, error_exit +from tools.exit_codes import CLIExitCodes from core.print_formatters import ( print_dkg_statuses, - error_exit, print_firewall_rules, print_schain_info, print_schains @@ -25,7 +25,7 @@ def get_schain_firewall_rules(schain: str) -> None: if status == 'ok': print_firewall_rules(payload['endpoints']) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def show_schains() -> None: @@ -40,7 +40,7 @@ def show_schains() -> None: return print_schains(schains) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def show_dkg_info(all_: bool = False) -> None: @@ -53,7 +53,7 @@ def show_dkg_info(all_: bool = False) -> None: if status == 'ok': print_dkg_statuses(payload) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def show_config(name: str) -> None: @@ -65,7 +65,7 @@ def show_config(name: str) -> None: if status == 'ok': pprint.pprint(payload) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def toggle_schain_repair_mode(schain: str) -> None: @@ -77,7 +77,7 @@ def toggle_schain_repair_mode(schain: str) -> None: if status == 'ok': print('Schain has been set for repair') else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def describe(schain: str, raw=False) -> None: @@ -89,4 +89,4 @@ def describe(schain: str, raw=False) -> None: if status == 'ok': print_schain_info(payload, raw=raw) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/core/wallet.py b/core/wallet.py index f3a3579d..fdf13cfc 100644 --- a/core/wallet.py +++ b/core/wallet.py @@ -21,6 +21,7 @@ from core.print_formatters import print_wallet_info, TEXTS from tools.helper import error_exit, get_request, post_request, logger +from tools.exit_codes import CLIExitCodes BLUEPRINT_NAME = 'wallet' @@ -34,7 +35,7 @@ def get_wallet_info(_format): else: print_wallet_info(payload) else: - error_exit(payload) + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): @@ -50,6 +51,5 @@ def send_eth(address: str, amount: float, gas_limit: int, gas_price: int): logger.info(msg) print(msg) else: - error_msg = payload - logger.error(f'Sending error {error_msg}') - error_exit(error_msg) + logger.error(f'Sending error {payload}') + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/tests/cli/exit_test.py b/tests/cli/exit_test.py index 25ebd9b2..fb4dd134 100644 --- a/tests/cli/exit_test.py +++ b/tests/cli/exit_test.py @@ -15,6 +15,6 @@ def test_exit_status(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, status, ['--format', 'json']) + result = run_command_mock('tools.helper.requests.get', resp_mock, status, ['--format', 'json']) assert result.exit_code == 0 assert result.output == "{'status': 'ACTIVE', 'data': [{'name': 'test', 'status': 'ACTIVE'}], 'exit_time': 0}\n" # noqa diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index cb5843bd..1aeabed1 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -42,7 +42,7 @@ def test_containers(): requests.codes.ok, json_data=OK_LS_RESPONSE_DATA ) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, containers) assert result.exit_code == 0 assert result.output == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa @@ -69,7 +69,7 @@ def test_checks(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, schains) print(result) @@ -78,7 +78,7 @@ def test_checks(): assert result.exit_code == 0 assert result.output == 'sChain Name Data directory DKG Config file Volume Container IMA Firewall RPC Blocks\n-----------------------------------------------------------------------------------------------------------\ntest_schain True False False False False False False False False \n' # noqa - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, schains, ['--json']) assert result.exit_code == 0 @@ -97,7 +97,7 @@ def test_sgx_status(): json_data={'payload': payload, 'status': 'ok'} ) result = run_command_mock( - 'core.helper.requests.get', resp_mock, sgx) + 'tools.helper.requests.get', resp_mock, sgx) assert result.exit_code == 0 assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status \x1b(0x\x1b(B CONNECTED \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 892b39bd..b3f57c53 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -46,7 +46,7 @@ def test_register_node(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, register_node, ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080']) @@ -60,11 +60,11 @@ def test_register_node_with_error(config): {'status': 'error', 'payload': ['Strange error']}, ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, register_node, ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80']) - assert result.exit_code == 0 + assert result.exit_code == 3 assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa @@ -74,7 +74,7 @@ def test_register_node_with_prompted_ip(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, register_node, ['--name', 'test-node', '--port', '8080'], input='0.0.0.0\n') @@ -88,7 +88,7 @@ def test_register_node_with_default_port(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, register_node, ['--name', 'test-node'], input='0.0.0.0\n') @@ -106,7 +106,7 @@ def test_init_node(config): return_value=True), \ mock.patch('core.node.is_node_inited', return_value=False): result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, init_node, ['./tests/test-env']) @@ -141,7 +141,7 @@ def test_update_node(config): mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock), \ mock.patch('core.host.init_data_dir'): result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, update_node, params, @@ -162,7 +162,7 @@ def test_update_node_without_init(config): return_value=True), \ mock.patch('core.node.is_node_inited', return_value=False): result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, update_node, params, @@ -189,7 +189,7 @@ def test_node_info_node_info(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nStatus: Active\n--------------------------------------------------\n' # noqa @@ -212,7 +212,7 @@ def test_node_info_node_info_not_created(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == 'This SKALE node is not registered on SKALE Manager yet\n' @@ -235,7 +235,7 @@ def test_node_info_node_info_frozen(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nStatus: Frozen\n--------------------------------------------------\n' # noqa @@ -258,7 +258,7 @@ def test_node_info_node_info_left(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nStatus: Left\n--------------------------------------------------\n' # noqa @@ -281,7 +281,7 @@ def test_node_info_node_info_leaving(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nStatus: Leaving\n--------------------------------------------------\n' # noqa @@ -304,7 +304,7 @@ def test_node_info_node_info_in_maintenance(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, node_info) + result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nStatus: In Maintenance\n--------------------------------------------------\n' # noqa @@ -316,7 +316,7 @@ def test_node_signature(): 'payload': {'signature': signature_sample} } resp_mock = response_mock(requests.codes.ok, json_data=response_data) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, signature, ['1']) assert result.exit_code == 0 assert result.output == f'Signature: {signature_sample}\n' @@ -359,7 +359,7 @@ def test_maintenance_on(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, set_node_in_maintenance, ['--yes']) @@ -373,7 +373,7 @@ def test_maintenance_off(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, remove_node_from_maintenance) assert result.exit_code == 0 @@ -387,7 +387,7 @@ def test_turn_off_maintenance_on(): ) with mock.patch('subprocess.run', new=subprocess_run_mock): result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, _turn_off, [ @@ -406,7 +406,7 @@ def test_turn_on_maintenance_off(): with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('core.node.get_flask_secret_key'): result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, _turn_on, [ diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index 9fd4729e..b886dd5e 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -55,7 +55,7 @@ def test_show(config, resource_alloc_config): resp_mock = response_mock(requests.codes.created) write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, show ) @@ -69,7 +69,7 @@ def test_generate(): with mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock): result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, generate, ['--yes'] @@ -85,7 +85,7 @@ def test_generate_already_exists(resource_alloc_config): with mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock): result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, generate, ['--yes'] @@ -94,7 +94,7 @@ def test_generate_already_exists(resource_alloc_config): assert result.exit_code == 0 result = run_command_mock( - 'core.helper.post_request', + 'tools.helper.post_request', resp_mock, generate, ['--yes', '--force'] diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index ba1baf99..1e6596ea 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -49,7 +49,7 @@ def test_ls(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, ls) + result = run_command_mock('tools.helper.requests.get', resp_mock, ls) assert result.exit_code == 0 assert result.output == ' Name Owner Size Lifetime Created At Deposit \n-----------------------------------------------------------------------------------\ntest_schain1 0x123 0 5 Oct 03 2019 16:09:45 1000000000000000000\ncrazy_cats1 0x321 0 5 Oct 07 2019 18:30:10 1000000000000000000\n' # noqa @@ -70,12 +70,12 @@ def test_dkg(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, dkg) assert result.exit_code == 0 assert result.output == ' sChain Name DKG Status Added At sChain Status\n---------------------------------------------------------------------\nmelodic-aldhibah IN_PROGRESS Jan 08 2020 15:26:52 Exists \n' # noqa - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, dkg, ['--all']) assert result.exit_code == 0 assert result.output == ' sChain Name DKG Status Added At sChain Status\n---------------------------------------------------------------------\nmelodic-aldhibah IN_PROGRESS Jan 08 2020 15:26:52 Exists \n' # noqa @@ -120,7 +120,7 @@ def test_get_schain_config(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', resp_mock, get_schain_config, ['test1']) assert result.exit_code == 0 @@ -145,7 +145,7 @@ def test_schain_rules(): json_data={'payload': payload, 'status': 'ok'} ) result = run_command_mock( - 'core.helper.requests.get', resp_mock, show_rules, ['schain-test']) + 'tools.helper.requests.get', resp_mock, show_rules, ['schain-test']) assert result.exit_code == 0 print(repr(result.output)) assert result.output == 'Port Ip \n-----------------\n10000 127.0.0.1\n10001 127.0.0.1\n10002 None \n10003 None \n10004 127.0.0.1\n10005 127.0.0.1\n10007 None \n10008 None \n' # noqa @@ -159,7 +159,7 @@ def test_repair(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.post', resp_mock, repair, + result = run_command_mock('tools.helper.requests.post', resp_mock, repair, ['test-schain', '--yes']) assert result.output == 'Schain has been set for repair\n' assert result.exit_code == 0 @@ -169,7 +169,7 @@ def test_repair(): requests.codes.ok, json_data={'payload': payload, 'status': 'error'} ) - result = run_command_mock('core.helper.requests.post', resp_mock, repair, + result = run_command_mock('tools.helper.requests.post', resp_mock, repair, ['test-schain', '--yes']) print(repr(result.output)) assert result.exit_code == 0 @@ -188,7 +188,7 @@ def test_info(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, info_, + result = run_command_mock('tools.helper.requests.get', resp_mock, info_, ['attractive-ed-asich']) assert result.output == ' Name Id Owner Part_of_node Dkg_status Is_deleted First_run Repair_mode\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nattractive-ed-asich 0xfb3b68013fa494407b691b4b603d84c66076c0a5ac96a7d6b162d7341d74fa61 0x1111111111111111111111111111111111111111 0 3 False False False \n' # noqa assert result.exit_code == 0 @@ -198,7 +198,7 @@ def test_info(): requests.codes.ok, json_data={'payload': payload, 'status': 'error'} ) - result = run_command_mock('core.helper.requests.get', resp_mock, info_, + result = run_command_mock('tools.helper.requests.get', resp_mock, info_, ['schain not found']) assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa assert result.exit_code == 0 diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index 9af96117..692436c5 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -38,7 +38,7 @@ def test_wallet_info(config): response_mock = MagicMock() response_mock.status_code = requests.codes.ok response_mock.json = Mock(return_value=response_data) - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', response_mock, wallet_info) assert result.exit_code == 0 @@ -51,7 +51,7 @@ def test_wallet_info(config): ) assert result.output == expected - result = run_command_mock('core.helper.requests.get', + result = run_command_mock('tools.helper.requests.get', response_mock, wallet_info, ['--format', 'json']) @@ -69,7 +69,7 @@ def test_wallet_send(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, send, ['0x00000000000000000000000000000000', '10', '--yes']) @@ -83,7 +83,7 @@ def test_wallet_send_with_error(): {'status': 'error', 'payload': ['Strange error']}, ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, send, ['0x00000000000000000000000000000000', '10', '--yes']) diff --git a/tools/helper.py b/tools/helper.py index 41fb40db..bcde0d80 100644 --- a/tools/helper.py +++ b/tools/helper.py @@ -142,7 +142,7 @@ def extract_env_params(env_filepath): def error_exit(error_payload, exit_code=CLIExitCodes.FAILURE): print_err_response(error_payload) - sys.exit(exit_code) + sys.exit(exit_code.value) def safe_get_config(config, key): From 52dbe1af0c86a67a4a129065928c3a9f5cf21544 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 23 Dec 2020 19:02:01 +0200 Subject: [PATCH 014/140] SKALE-3333 Update exit codes in tests --- tests/cli/schains_test.py | 4 ++-- tests/cli/wallet_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 1e6596ea..aa47a8c8 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -172,7 +172,7 @@ def test_repair(): result = run_command_mock('tools.helper.requests.post', resp_mock, repair, ['test-schain', '--yes']) print(repr(result.output)) - assert result.exit_code == 0 + assert result.exit_code == 3 assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa @@ -201,4 +201,4 @@ def test_info(): result = run_command_mock('tools.helper.requests.get', resp_mock, info_, ['schain not found']) assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa - assert result.exit_code == 0 + assert result.exit_code == 3 diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index 692436c5..5da1ed72 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -87,5 +87,5 @@ def test_wallet_send_with_error(): resp_mock, send, ['0x00000000000000000000000000000000', '10', '--yes']) - assert result.exit_code == 0 + assert result.exit_code == 3 assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa From 3a8596e2437f31320dbb7814a995e5df3de8b1ad Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 20 Jan 2021 19:08:16 +0200 Subject: [PATCH 015/140] SKALE-3744 Fix tests --- tests/cli/node_test.py | 2 +- tests/routes_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 634b8580..c86cfe8a 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -432,7 +432,7 @@ def test_set_domain_name(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'core.helper.requests.post', + 'tools.helper.requests.post', resp_mock, _set_domain_name, ['-d', 'skale.test', '--yes']) assert result.exit_code == 0 diff --git a/tests/routes_test.py b/tests/routes_test.py index 00e8a60b..6286de5d 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -14,6 +14,7 @@ '/api/v1/node/send-tg-notification', '/api/v1/node/exit/start', '/api/v1/node/exit/status', + '/api/v1/node/set-domain-name', '/api/v1/health/containers', '/api/v1/health/schains', From 46a113cfaf5766fce280d6f78848813d5311eb38 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 20 Jan 2021 19:33:08 +0200 Subject: [PATCH 016/140] SKALE-3744 Fix tests --- core/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/node.py b/core/node.py index ddd91721..8baa05bb 100644 --- a/core/node.py +++ b/core/node.py @@ -378,7 +378,8 @@ def set_domain_name(domain_name): return print(f'Setting new domain name: {domain_name}') status, payload = post_request( - url_name='set_domain_name', + blueprint=BLUEPRINT_NAME, + method='set-domain-name', json={ 'domain_name': domain_name } From 824477c0f0ab471c7d9640447867a1d4be1941e1 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 21 Jan 2021 20:39:06 +0200 Subject: [PATCH 017/140] SKALE-3753 Add new dir --- configs/__init__.py | 1 + core/host.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configs/__init__.py b/configs/__init__.py index fcd7e112..6e7581e8 100644 --- a/configs/__init__.py +++ b/configs/__init__.py @@ -30,6 +30,7 @@ INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') LOG_PATH = os.path.join(SKALE_DIR, NODE_DATA_PATH, 'log') +ETH_STATE_PATH = os.path.join(NODE_DATA_PATH, 'eth-state') NODE_CERTS_PATH = os.path.join(SKALE_DIR, NODE_DATA_PATH, 'ssl') SGX_CERTS_PATH = os.path.join(NODE_DATA_PATH, 'sgx_certs') SCHAINS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') diff --git a/core/host.py b/core/host.py index 88f529ae..17b48e90 100644 --- a/core/host.py +++ b/core/host.py @@ -28,8 +28,9 @@ from configs import (ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, - NODE_CERTS_PATH, SGX_CERTS_PATH, REDIS_DATA_PATH, - SCHAINS_DATA_PATH, LOG_PATH, MYSQL_BACKUP_FOLDER, + ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, + REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, + MYSQL_BACKUP_FOLDER, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) from configs.cli_logger import LOG_DATA_PATH from core.print_formatters import print_abi_validation_errors @@ -79,7 +80,8 @@ def is_node_inited(): def make_dirs(): for dir_path in ( SKALE_DIR, NODE_DATA_PATH, CONTAINER_CONFIG_PATH, - CONTRACTS_PATH, NODE_CERTS_PATH, MYSQL_BACKUP_FOLDER, + CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, + MYSQL_BACKUP_FOLDER, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH ): safe_mk_dirs(dir_path) From 67b1205d019a49d09f2c5fee7d4e7d562de8c63a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Feb 2021 14:10:48 +0200 Subject: [PATCH 018/140] SKALE-29724 Refactor node-cli structure --- helper-scripts | 2 +- main.spec | 2 +- __init__.py => node_cli/__init__.py | 0 {cli => node_cli/cli}/__init__.py | 0 {cli => node_cli/cli}/exit.py | 8 +++---- {cli => node_cli/cli}/health.py | 4 ++-- node_cli/cli/info.py | 5 ++++ {cli => node_cli/cli}/logs.py | 6 ++--- {cli => node_cli/cli}/node.py | 4 ++-- {cli => node_cli/cli}/resources_allocation.py | 4 ++-- {cli => node_cli/cli}/schains.py | 4 ++-- {cli => node_cli/cli}/ssl.py | 4 ++-- {cli => node_cli/cli}/validate.py | 2 +- {cli => node_cli/cli}/wallet.py | 4 ++-- {configs => node_cli/configs}/__init__.py | 6 +++-- {configs => node_cli/configs}/cli_logger.py | 0 {configs => node_cli/configs}/env.py | 0 .../configs}/resource_allocation.py | 0 {configs => node_cli/configs}/routes.py | 0 {core => node_cli/core}/__init__.py | 0 {core => node_cli/core}/health.py | 6 ++--- {core => node_cli/core}/host.py | 12 +++++----- {core => node_cli/core}/mysql_backup.py | 2 +- {core => node_cli/core}/node.py | 24 +++++++++---------- .../core}/operations/__init__.py | 2 +- {core => node_cli/core}/operations/base.py | 10 ++++---- {core => node_cli/core}/operations/common.py | 6 ++--- .../core}/operations/git_helper.py | 0 {core => node_cli/core}/print_formatters.py | 4 ++-- {core => node_cli/core}/resources.py | 6 ++--- {core => node_cli/core}/schains.py | 6 ++--- {core => node_cli/core}/texts.py | 0 {core => node_cli/core}/wallet.py | 6 ++--- main.py => node_cli/main.py | 24 +++++++++---------- {tools => node_cli/utils}/__init__.py | 0 {tools => node_cli/utils}/docker_utils.py | 2 +- {tools => node_cli/utils}/exit_codes.py | 0 {tools => node_cli/utils}/helper.py | 10 ++++---- {tools => node_cli/utils}/meta.py | 0 {tools => node_cli/utils}/schain_types.py | 0 {tools => node_cli/utils}/texts.py | 0 scripts/build.sh | 2 +- tests/cli/exit_test.py | 2 +- tests/cli/health_test.py | 2 +- tests/cli/logs_test.py | 2 +- tests/cli/node_test.py | 4 ++-- tests/cli/resources_allocation_test.py | 8 +++---- tests/cli/schains_test.py | 2 +- tests/cli/validate_test.py | 2 +- tests/cli/wallet_test.py | 2 +- tests/core_node_test.py | 2 +- tests/resources_test.py | 6 ++--- tests/routes_test.py | 2 +- tests/tools_meta_test.py | 2 +- 54 files changed, 110 insertions(+), 103 deletions(-) rename __init__.py => node_cli/__init__.py (100%) rename {cli => node_cli/cli}/__init__.py (100%) rename {cli => node_cli/cli}/exit.py (89%) rename {cli => node_cli/cli}/health.py (92%) create mode 100644 node_cli/cli/info.py rename {cli => node_cli/cli}/logs.py (89%) rename {cli => node_cli/cli}/node.py (97%) rename {cli => node_cli/cli}/resources_allocation.py (92%) rename {cli => node_cli/cli}/schains.py (96%) rename {cli => node_cli/cli}/ssl.py (94%) rename {cli => node_cli/cli}/validate.py (95%) rename {cli => node_cli/cli}/wallet.py (94%) rename {configs => node_cli/configs}/__init__.py (96%) rename {configs => node_cli/configs}/cli_logger.py (100%) rename {configs => node_cli/configs}/env.py (100%) rename {configs => node_cli/configs}/resource_allocation.py (100%) rename {configs => node_cli/configs}/routes.py (100%) rename {core => node_cli/core}/__init__.py (100%) rename {core => node_cli/core}/health.py (93%) rename {core => node_cli/core}/host.py (91%) rename {core => node_cli/core}/mysql_backup.py (96%) rename {core => node_cli/core}/node.py (94%) rename {core => node_cli/core}/operations/__init__.py (91%) rename {core => node_cli/core}/operations/base.py (86%) rename {core => node_cli/core}/operations/common.py (93%) rename {core => node_cli/core}/operations/git_helper.py (100%) rename {core => node_cli/core}/print_formatters.py (98%) rename {core => node_cli/core}/resources.py (97%) rename {core => node_cli/core}/schains.py (93%) rename {core => node_cli/core}/texts.py (100%) rename {core => node_cli/core}/wallet.py (89%) rename main.py => node_cli/main.py (82%) rename {tools => node_cli/utils}/__init__.py (100%) rename {tools => node_cli/utils}/docker_utils.py (99%) rename {tools => node_cli/utils}/exit_codes.py (100%) rename {tools => node_cli/utils}/helper.py (96%) rename {tools => node_cli/utils}/meta.py (100%) rename {tools => node_cli/utils}/schain_types.py (100%) rename {tools => node_cli/utils}/texts.py (100%) diff --git a/helper-scripts b/helper-scripts index ced34747..bdc8c1a2 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit ced34747acb3335f1ea858b4e847f3c21b4ccf7e +Subproject commit bdc8c1a2f11bc4d15c884033e10e06b157b4a2e9 diff --git a/main.spec b/main.spec index 97fe8f67..36b83b6f 100644 --- a/main.spec +++ b/main.spec @@ -7,7 +7,7 @@ block_cipher = None a = Analysis( - ['main.py'], + ['node_cli/main.py'], pathex=['.'], binaries=[], datas=[ diff --git a/__init__.py b/node_cli/__init__.py similarity index 100% rename from __init__.py rename to node_cli/__init__.py diff --git a/cli/__init__.py b/node_cli/cli/__init__.py similarity index 100% rename from cli/__init__.py rename to node_cli/cli/__init__.py diff --git a/cli/exit.py b/node_cli/cli/exit.py similarity index 89% rename from cli/exit.py rename to node_cli/cli/exit.py index 0d0599a2..ff9443cc 100644 --- a/cli/exit.py +++ b/node_cli/cli/exit.py @@ -21,10 +21,10 @@ import click -from core.print_formatters import print_exit_status -from tools.helper import error_exit, get_request, post_request, abort_if_false -from tools.exit_codes import CLIExitCodes -from tools.texts import Texts +from node_cli.core.print_formatters import print_exit_status +from node_cli.utils.helper import error_exit, get_request, post_request, abort_if_false +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.texts import Texts logger = logging.getLogger(__name__) TEXTS = Texts() diff --git a/cli/health.py b/node_cli/cli/health.py similarity index 92% rename from cli/health.py rename to node_cli/cli/health.py index 884a20e3..808082c8 100644 --- a/cli/health.py +++ b/node_cli/cli/health.py @@ -18,9 +18,9 @@ # along with this program. If not, see . import click -from tools.texts import Texts +from node_cli.utils.texts import Texts -from core.health import get_containers, get_schains_checks, get_sgx_info +from node_cli.core.health import get_containers, get_schains_checks, get_sgx_info G_TEXTS = Texts() diff --git a/node_cli/cli/info.py b/node_cli/cli/info.py new file mode 100644 index 00000000..1c2c908a --- /dev/null +++ b/node_cli/cli/info.py @@ -0,0 +1,5 @@ +BUILD_DATETIME = '2021-02-16 14:09:45' +COMMIT = '20b163c683a833b2f10f48a0bdfa9054ba27e2c2' +BRANCH = 'test' +OS = 'Darwin-x86_64' +VERSION = '0.0.0' diff --git a/cli/logs.py b/node_cli/cli/logs.py similarity index 89% rename from cli/logs.py rename to node_cli/cli/logs.py index e0a5e338..88bb87b3 100644 --- a/cli/logs.py +++ b/node_cli/cli/logs.py @@ -20,9 +20,9 @@ import sys import click -from tools.helper import download_dump -from configs.cli_logger import LOG_FILEPATH, DEBUG_LOG_FILEPATH -from tools.exit_codes import CLIExitCodes +from node_cli.utils.helper import download_dump +from node_cli.configs.cli_logger import LOG_FILEPATH, DEBUG_LOG_FILEPATH +from node_cli.utils.exit_codes import CLIExitCodes @click.group() diff --git a/cli/node.py b/node_cli/cli/node.py similarity index 97% rename from cli/node.py rename to node_cli/cli/node.py index fd01b2fe..0ba0e285 100644 --- a/cli/node.py +++ b/node_cli/cli/node.py @@ -22,12 +22,12 @@ import click -from core.node import (get_node_signature, init, restore, +from node_cli.core.node import (get_node_signature, init, restore, register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info, set_domain_name) from configs import DEFAULT_NODE_BASE_PORT -from tools.helper import session_config, abort_if_false, safe_load_texts +from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts config = session_config() diff --git a/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py similarity index 92% rename from cli/resources_allocation.py rename to node_cli/cli/resources_allocation.py index fa9875d2..a9610f81 100644 --- a/cli/resources_allocation.py +++ b/node_cli/cli/resources_allocation.py @@ -20,9 +20,9 @@ import json import click -from core.resources import (get_resource_allocation_info, +from node_cli.core.resources import (get_resource_allocation_info, generate_resource_allocation_config) -from tools.helper import session_config, abort_if_false, safe_load_texts +from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts config = session_config() TEXTS = safe_load_texts() diff --git a/cli/schains.py b/node_cli/cli/schains.py similarity index 96% rename from cli/schains.py rename to node_cli/cli/schains.py index 1e6c02be..db70a49e 100644 --- a/cli/schains.py +++ b/node_cli/cli/schains.py @@ -19,8 +19,8 @@ import click -from tools.helper import abort_if_false -from core.schains import ( +from node_cli.utils.helper import abort_if_false +from node_cli.core.schains import ( describe, get_schain_firewall_rules, show_config, diff --git a/cli/ssl.py b/node_cli/cli/ssl.py similarity index 94% rename from cli/ssl.py rename to node_cli/cli/ssl.py index 663839cb..5ac5e9a1 100644 --- a/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -20,8 +20,8 @@ import click from terminaltables import SingleTable -from tools.exit_codes import CLIExitCodes -from tools.helper import (get_request, safe_load_texts, upload_certs, +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import (get_request, safe_load_texts, upload_certs, error_exit) diff --git a/cli/validate.py b/node_cli/cli/validate.py similarity index 95% rename from cli/validate.py rename to node_cli/cli/validate.py index f3377a55..f8134df9 100644 --- a/cli/validate.py +++ b/node_cli/cli/validate.py @@ -19,7 +19,7 @@ import click -from core.host import validate_abi_files +from node_cli.core.host import validate_abi_files @click.group() diff --git a/cli/wallet.py b/node_cli/cli/wallet.py similarity index 94% rename from cli/wallet.py rename to node_cli/cli/wallet.py index 365853ca..543e1071 100644 --- a/cli/wallet.py +++ b/node_cli/cli/wallet.py @@ -21,8 +21,8 @@ import click -from tools.helper import abort_if_false -from core.wallet import get_wallet_info, send_eth +from node_cli.utils.helper import abort_if_false +from node_cli.core.wallet import get_wallet_info, send_eth logger = logging.getLogger(__name__) diff --git a/configs/__init__.py b/node_cli/configs/__init__.py similarity index 96% rename from configs/__init__.py rename to node_cli/configs/__init__.py index dd722a61..a461f2e7 100644 --- a/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -73,8 +73,10 @@ def _get_env(): else: PARDIR = os.path.join(sys._MEIPASS, 'data') -TEXT_FILE = os.path.join(PARDIR, 'text.yml') -DATAFILES_FOLDER = os.path.join(PARDIR, 'datafiles') + +PROJECT_DIR = os.path.join(PARDIR, os.pardir) +TEXT_FILE = os.path.join(PROJECT_DIR, 'text.yml') +DATAFILES_FOLDER = os.path.join(PROJECT_DIR, 'datafiles') THIRDPARTY_FOLDER_PATH = os.path.join(DATAFILES_FOLDER, 'third_party') diff --git a/configs/cli_logger.py b/node_cli/configs/cli_logger.py similarity index 100% rename from configs/cli_logger.py rename to node_cli/configs/cli_logger.py diff --git a/configs/env.py b/node_cli/configs/env.py similarity index 100% rename from configs/env.py rename to node_cli/configs/env.py diff --git a/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py similarity index 100% rename from configs/resource_allocation.py rename to node_cli/configs/resource_allocation.py diff --git a/configs/routes.py b/node_cli/configs/routes.py similarity index 100% rename from configs/routes.py rename to node_cli/configs/routes.py diff --git a/core/__init__.py b/node_cli/core/__init__.py similarity index 100% rename from core/__init__.py rename to node_cli/core/__init__.py diff --git a/core/health.py b/node_cli/core/health.py similarity index 93% rename from core/health.py rename to node_cli/core/health.py index f0cbb586..3725059a 100644 --- a/core/health.py +++ b/node_cli/core/health.py @@ -20,12 +20,12 @@ import json from terminaltables import SingleTable -from core.print_formatters import ( +from node_cli.core.print_formatters import ( print_containers, print_schains_healthchecks ) -from tools.helper import error_exit, get_request -from tools.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, get_request +from node_cli.utils.exit_codes import CLIExitCodes BLUEPRINT_NAME = 'health' diff --git a/core/host.py b/node_cli/core/host.py similarity index 91% rename from core/host.py rename to node_cli/core/host.py index a0f853bb..c9028ba5 100644 --- a/core/host.py +++ b/node_cli/core/host.py @@ -23,7 +23,7 @@ from shutil import copyfile from urllib.parse import urlparse -from core.resources import update_resource_allocation +from node_cli.core.resources import update_resource_allocation from configs import (ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, @@ -32,13 +32,13 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) -from configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH -from configs.cli_logger import LOG_DATA_PATH -from core.print_formatters import print_abi_validation_errors -from configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, +from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.configs.cli_logger import LOG_DATA_PATH +from node_cli.core.print_formatters import print_abi_validation_errors +from node_cli.configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, SGX_SERVER_URL_FILEPATH) -from tools.helper import safe_load_texts, validate_abi +from node_cli.utils.helper import safe_load_texts, validate_abi TEXTS = safe_load_texts() diff --git a/core/mysql_backup.py b/node_cli/core/mysql_backup.py similarity index 96% rename from core/mysql_backup.py rename to node_cli/core/mysql_backup.py index 7ea6b7a3..170958f7 100644 --- a/core/mysql_backup.py +++ b/node_cli/core/mysql_backup.py @@ -3,7 +3,7 @@ import shlex from configs import MYSQL_BACKUP_CONTAINER_PATH, MYSQL_BACKUP_PATH -from tools.helper import run_cmd, extract_env_params +from node_cli.utils.helper import run_cmd, extract_env_params logger = logging.getLogger(__name__) diff --git a/core/node.py b/node_cli/core/node.py similarity index 94% rename from core/node.py rename to node_cli/core/node.py index 00b3f5c3..f936dd99 100644 --- a/core/node.py +++ b/node_cli/core/node.py @@ -27,25 +27,25 @@ import docker -from cli.info import VERSION +from node_cli.cli.info import VERSION from configs import (SKALE_DIR, INSTALL_SCRIPT, UNINSTALL_SCRIPT, BACKUP_INSTALL_SCRIPT, DATAFILES_FOLDER, INIT_ENV_FILEPATH, BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) -from configs.cli_logger import LOG_DIRNAME +from node_cli.configs.cli_logger import LOG_DIRNAME -from core.operations import update_op -from core.mysql_backup import create_mysql_backup, restore_mysql_backup -from core.host import (is_node_inited, prepare_host, +from node_cli.core.operations import update_op +from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup +from node_cli.core.host import (is_node_inited, prepare_host, save_env_params, get_flask_secret_key) -from core.print_formatters import print_node_cmd_error, print_node_info -from tools.helper import error_exit, get_request, post_request -from core.resources import update_resource_allocation -from tools.meta import update_meta -from tools.helper import run_cmd, extract_env_params -from tools.texts import Texts -from tools.exit_codes import CLIExitCodes +from node_cli.core.print_formatters import print_node_cmd_error, print_node_info +from node_cli.utils.helper import error_exit, get_request, post_request +from node_cli.core.resources import update_resource_allocation +from node_cli.utils.meta import update_meta +from node_cli.utils.helper import run_cmd, extract_env_params +from node_cli.utils.texts import Texts +from node_cli.utils.exit_codes import CLIExitCodes logger = logging.getLogger(__name__) TEXTS = Texts() diff --git a/core/operations/__init__.py b/node_cli/core/operations/__init__.py similarity index 91% rename from core/operations/__init__.py rename to node_cli/core/operations/__init__.py index 92a9f1dd..5f28ae8c 100644 --- a/core/operations/__init__.py +++ b/node_cli/core/operations/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from core.operations.base import update as update_op # noqa +from node_cli.core.operations.base import update as update_op # noqa diff --git a/core/operations/base.py b/node_cli/core/operations/base.py similarity index 86% rename from core/operations/base.py rename to node_cli/core/operations/base.py index bf590b92..eeb10455 100644 --- a/core/operations/base.py +++ b/node_cli/core/operations/base.py @@ -17,14 +17,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from cli.info import VERSION -from core.host import prepare_host -from core.operations.common import ( +from node_cli.cli.info import VERSION +from node_cli.core.host import prepare_host +from node_cli.core.operations.common import ( remove_dynamic_containers, backup_old_contracts, download_contracts, docker_lvmpy_update, update_skale_node, download_filestorage_artifacts ) -from tools.docker_utils import compose_rm, compose_up -from tools.meta import update_meta +from node_cli.utils.docker_utils import compose_rm, compose_up +from node_cli.utils.meta import update_meta def update(env_filepath: str, env: str) -> None: diff --git a/core/operations/common.py b/node_cli/core/operations/common.py similarity index 93% rename from core/operations/common.py rename to node_cli/core/operations/common.py index f1fdf882..34a95a6f 100644 --- a/core/operations/common.py +++ b/node_cli/core/operations/common.py @@ -23,13 +23,13 @@ import urllib.request from distutils.dir_util import copy_tree -from core.operations.git_helper import update_repo -from tools.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, compose_pull, +from node_cli.core.operations.git_helper import update_repo +from node_cli.utils.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, compose_pull, compose_build) from configs import (CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, DOCKER_LVMPY_PATH, CONTAINER_CONFIG_PATH, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE) -from tools.helper import run_cmd, read_json +from node_cli.utils.helper import run_cmd, read_json logger = logging.getLogger(__name__) diff --git a/core/operations/git_helper.py b/node_cli/core/operations/git_helper.py similarity index 100% rename from core/operations/git_helper.py rename to node_cli/core/operations/git_helper.py diff --git a/core/print_formatters.py b/node_cli/core/print_formatters.py similarity index 98% rename from core/print_formatters.py rename to node_cli/core/print_formatters.py index 2e3d460a..729277f5 100644 --- a/core/print_formatters.py +++ b/node_cli/core/print_formatters.py @@ -26,8 +26,8 @@ import inspect from configs import LONG_LINE -from configs.cli_logger import DEBUG_LOG_FILEPATH -from tools.texts import Texts +from node_cli.configs.cli_logger import DEBUG_LOG_FILEPATH +from node_cli.utils.texts import Texts TEXTS = Texts() diff --git a/core/resources.py b/node_cli/core/resources.py similarity index 97% rename from core/resources.py rename to node_cli/core/resources.py index f2121ac4..8a979ed9 100644 --- a/core/resources.py +++ b/node_cli/core/resources.py @@ -24,10 +24,10 @@ import psutil -from tools.schain_types import SchainTypes -from tools.helper import write_json, read_json, run_cmd, format_output, safe_load_yml +from node_cli.utils.schain_types import SchainTypes +from node_cli.utils.helper import write_json, read_json, run_cmd, format_output, safe_load_yml from configs import ALLOCATION_FILEPATH -from configs.resource_allocation import ( +from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, MEMORY_FACTOR, DISK_FACTOR, DISK_MOUNTPOINT_FILEPATH, diff --git a/core/schains.py b/node_cli/core/schains.py similarity index 93% rename from core/schains.py rename to node_cli/core/schains.py index 2075d051..cd961ef2 100644 --- a/core/schains.py +++ b/node_cli/core/schains.py @@ -1,9 +1,9 @@ import logging import pprint -from tools.helper import get_request, post_request, error_exit -from tools.exit_codes import CLIExitCodes -from core.print_formatters import ( +from node_cli.utils.helper import get_request, post_request, error_exit +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.core.print_formatters import ( print_dkg_statuses, print_firewall_rules, print_schain_info, diff --git a/core/texts.py b/node_cli/core/texts.py similarity index 100% rename from core/texts.py rename to node_cli/core/texts.py diff --git a/core/wallet.py b/node_cli/core/wallet.py similarity index 89% rename from core/wallet.py rename to node_cli/core/wallet.py index fdf13cfc..e7d832e0 100644 --- a/core/wallet.py +++ b/node_cli/core/wallet.py @@ -19,9 +19,9 @@ import json -from core.print_formatters import print_wallet_info, TEXTS -from tools.helper import error_exit, get_request, post_request, logger -from tools.exit_codes import CLIExitCodes +from node_cli.core.print_formatters import print_wallet_info, TEXTS +from node_cli.utils.helper import error_exit, get_request, post_request, logger +from node_cli.utils.exit_codes import CLIExitCodes BLUEPRINT_NAME = 'wallet' diff --git a/main.py b/node_cli/main.py similarity index 82% rename from main.py rename to node_cli/main.py index 6a87c88b..2c8752f0 100644 --- a/main.py +++ b/node_cli/main.py @@ -26,19 +26,19 @@ import click from cli import __version__ -from cli.health import health_cli -from cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION -from cli.logs import logs_cli -from cli.node import node_cli -from cli.schains import schains_cli -from cli.wallet import wallet_cli -from cli.ssl import ssl_cli -from cli.exit import exit_cli -from cli.validate import validate_cli -from cli.resources_allocation import resources_allocation_cli -from tools.helper import safe_load_texts, init_default_logger +from node_cli.cli.health import health_cli +from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION +from node_cli.cli.logs import logs_cli +from node_cli.cli.node import node_cli +from node_cli.cli.schains import schains_cli +from node_cli.cli.wallet import wallet_cli +from node_cli.cli.ssl import ssl_cli +from node_cli.cli.exit import exit_cli +from node_cli.cli.validate import validate_cli +from node_cli.cli.resources_allocation import resources_allocation_cli +from node_cli.utils.helper import safe_load_texts, init_default_logger from configs import LONG_LINE -from core.host import init_logs_dir +from node_cli.core.host import init_logs_dir TEXTS = safe_load_texts() diff --git a/tools/__init__.py b/node_cli/utils/__init__.py similarity index 100% rename from tools/__init__.py rename to node_cli/utils/__init__.py diff --git a/tools/docker_utils.py b/node_cli/utils/docker_utils.py similarity index 99% rename from tools/docker_utils.py rename to node_cli/utils/docker_utils.py index f529231e..b3d199f1 100644 --- a/tools/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -25,7 +25,7 @@ from docker.client import DockerClient from docker.models.containers import Container -from tools.helper import run_cmd, str_to_bool +from node_cli.utils.helper import run_cmd, str_to_bool from configs import (COMPOSE_PATH, SKALE_DIR, SGX_CERTIFICATES_DIR_NAME, REMOVED_CONTAINERS_FOLDER_PATH) diff --git a/tools/exit_codes.py b/node_cli/utils/exit_codes.py similarity index 100% rename from tools/exit_codes.py rename to node_cli/utils/exit_codes.py diff --git a/tools/helper.py b/node_cli/utils/helper.py similarity index 96% rename from tools/helper.py rename to node_cli/utils/helper.py index a26db0bc..cb754196 100644 --- a/tools/helper.py +++ b/node_cli/utils/helper.py @@ -41,16 +41,16 @@ from jinja2 import Environment from readsettings import ReadSettings -from core.print_formatters import print_err_response -from tools.exit_codes import CLIExitCodes +from node_cli.core.print_formatters import print_err_response +from node_cli.utils.exit_codes import CLIExitCodes -from configs.env import (absent_params as absent_env_params, +from node_cli.configs.env import (absent_params as absent_env_params, get_params as get_env_params) from configs import CONFIG_FILEPATH, TEXT_FILE, ADMIN_HOST, ADMIN_PORT -from configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, +from node_cli.configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, LOG_FILEPATH, DEBUG_LOG_FILEPATH) -from configs.routes import get_route +from node_cli.configs.routes import get_route logger = logging.getLogger(__name__) diff --git a/tools/meta.py b/node_cli/utils/meta.py similarity index 100% rename from tools/meta.py rename to node_cli/utils/meta.py diff --git a/tools/schain_types.py b/node_cli/utils/schain_types.py similarity index 100% rename from tools/schain_types.py rename to node_cli/utils/schain_types.py diff --git a/tools/texts.py b/node_cli/utils/texts.py similarity index 100% rename from tools/texts.py rename to node_cli/utils/texts.py diff --git a/scripts/build.sh b/scripts/build.sh index 87dae043..89345775 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -28,7 +28,7 @@ OS=`uname -s`-`uname -m` #CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) LATEST_COMMIT=$(git rev-parse HEAD) CURRENT_DATETIME="`date "+%Y-%m-%d %H:%M:%S"`"; -DIST_INFO_FILEPATH=$PARENT_DIR/cli/info.py +DIST_INFO_FILEPATH=$PARENT_DIR/node_cli/cli/info.py touch $DIST_INFO_FILEPATH diff --git a/tests/cli/exit_test.py b/tests/cli/exit_test.py index fb4dd134..ab515634 100644 --- a/tests/cli/exit_test.py +++ b/tests/cli/exit_test.py @@ -1,5 +1,5 @@ import requests -from cli.exit import status +from node_cli.cli.exit import status from tests.helper import response_mock, run_command_mock diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 1aeabed1..b94b4bb4 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -1,7 +1,7 @@ import requests from tests.helper import response_mock, run_command_mock -from cli.health import containers, schains, sgx +from node_cli.cli.health import containers, schains, sgx OK_LS_RESPONSE_DATA = { diff --git a/tests/cli/logs_test.py b/tests/cli/logs_test.py index 885263c8..75c849cb 100644 --- a/tests/cli/logs_test.py +++ b/tests/cli/logs_test.py @@ -25,7 +25,7 @@ from io import BytesIO from tests.helper import response_mock, run_command -from cli.logs import dump +from node_cli.cli.logs import dump def test_dump(config): diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index d2c69fc0..ceff34b5 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -24,8 +24,8 @@ import requests from configs import NODE_DATA_PATH, SKALE_DIR -from core.resources import ResourceAlloc -from cli.node import (init_node, node_info, register_node, signature, +from node_cli.core.resources import ResourceAlloc +from node_cli.cli.node import (init_node, node_info, register_node, signature, update_node, backup_node, restore_node, set_node_in_maintenance, remove_node_from_maintenance, _turn_off, _turn_on, _set_domain_name) diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index b886dd5e..7a017040 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -24,15 +24,15 @@ import pytest -from core.host import safe_mk_dirs -from configs.resource_allocation import ( +from node_cli.core.host import safe_mk_dirs +from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, NODE_DATA_PATH ) -from tools.helper import write_json +from node_cli.utils.helper import write_json from tests.resources_test import disk_alloc_mock from tests.helper import response_mock, run_command_mock -from cli.resources_allocation import show, generate +from node_cli.cli.resources_allocation import show, generate TEST_CONFIG = {'test': 1} diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index aa47a8c8..28a260d6 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -23,7 +23,7 @@ import requests from tests.helper import response_mock, run_command_mock -from cli.schains import (get_schain_config, ls, dkg, show_rules, +from node_cli.cli.schains import (get_schain_config, ls, dkg, show_rules, repair, info_) diff --git a/tests/cli/validate_test.py b/tests/cli/validate_test.py index 5e7b75c1..72cb3e2b 100644 --- a/tests/cli/validate_test.py +++ b/tests/cli/validate_test.py @@ -6,7 +6,7 @@ from configs import (CONTRACTS_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) -from cli.validate import abi +from node_cli.cli.validate import abi from tests.helper import run_command diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index 5da1ed72..3e0eee6a 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -22,7 +22,7 @@ from mock import MagicMock, Mock -from cli.wallet import wallet_info, send +from node_cli.cli.wallet import wallet_info, send from tests.helper import run_command_mock, response_mock diff --git a/tests/core_node_test.py b/tests/core_node_test.py index 9f477963..7ac8698d 100644 --- a/tests/core_node_test.py +++ b/tests/core_node_test.py @@ -3,7 +3,7 @@ import docker import pytest -from core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive +from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive dclient = docker.from_env() diff --git a/tests/resources_test.py b/tests/resources_test.py index 3358ad71..ab4482d9 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -5,15 +5,15 @@ import pytest from configs import ALLOCATION_FILEPATH -from configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH -from core.resources import ( +from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.core.resources import ( compose_resource_allocation_config, get_schain_volume_proportions, update_resource_allocation, ResourceAlloc, SChainVolumeAlloc ) -from tools.helper import write_json, safe_load_yml +from node_cli.utils.helper import write_json, safe_load_yml SCHAIN_VOLUME_PARTS = {'test4': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'test': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'small': {'max_consensus_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 0, 'max_file_storage_bytes': 0, 'max_reserved_storage_bytes': 0}, 'medium': {'max_consensus_storage_bytes': 1, 'max_file_storage_bytes': 1, 'max_reserved_storage_bytes': 0, 'max_skaled_leveldb_storage_bytes': 1}, 'large': {'max_consensus_storage_bytes': 38, 'max_skaled_leveldb_storage_bytes': 38, 'max_file_storage_bytes': 38, 'max_reserved_storage_bytes': 12}} # noqa diff --git a/tests/routes_test.py b/tests/routes_test.py index 6286de5d..6e847b07 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -1,5 +1,5 @@ import pytest -from configs.routes import (route_exists, get_route, get_all_available_routes, +from node_cli.configs.routes import (route_exists, get_route, get_all_available_routes, RouteNotFoundException) diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index fcbff406..cc92c8e5 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -4,7 +4,7 @@ import pytest from configs import META_FILEPATH -from tools.meta import ( +from node_cli.utils.meta import ( CliMeta, compose_default_meta, DEFAULT_CONFIG_STREAM, DEFAULT_VERSION, ensure_meta, get_meta_info, From d11de63a83e72379debd307fd4f8c24b4a90f9c6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Feb 2021 14:21:18 +0200 Subject: [PATCH 019/140] SKALE-2972 Fix flake, fix setup.py --- node_cli/cli/node.py | 11 ++++++----- node_cli/cli/resources_allocation.py | 6 ++++-- node_cli/cli/ssl.py | 5 +++-- node_cli/configs/cli_logger.py | 2 +- node_cli/configs/resource_allocation.py | 2 +- node_cli/core/host.py | 17 +++++++++-------- node_cli/core/mysql_backup.py | 2 +- node_cli/core/node.py | 12 ++++++------ node_cli/core/operations/common.py | 12 +++++++----- node_cli/core/print_formatters.py | 2 +- node_cli/core/resources.py | 2 +- node_cli/core/texts.py | 2 +- node_cli/main.py | 2 +- node_cli/utils/docker_utils.py | 5 +++-- node_cli/utils/helper.py | 15 +++++++++------ node_cli/utils/meta.py | 2 +- node_cli/utils/texts.py | 2 +- setup.py | 2 +- tests/cli/node_test.py | 8 ++++---- tests/cli/schains_test.py | 2 +- tests/cli/validate_test.py | 4 ++-- tests/conftest.py | 2 +- tests/resources_test.py | 2 +- tests/routes_test.py | 2 +- tests/tools_meta_test.py | 2 +- 25 files changed, 68 insertions(+), 57 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 0ba0e285..691221d6 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -22,11 +22,12 @@ import click -from node_cli.core.node import (get_node_signature, init, restore, - register_node as register, update, backup, set_maintenance_mode_on, - set_maintenance_mode_off, turn_off, turn_on, get_node_info, - set_domain_name) -from configs import DEFAULT_NODE_BASE_PORT +from node_cli.core.node import ( + get_node_signature, init, restore, register_node as register, update, backup, + set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info, + set_domain_name +) +from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py index a9610f81..1168eacb 100644 --- a/node_cli/cli/resources_allocation.py +++ b/node_cli/cli/resources_allocation.py @@ -20,8 +20,10 @@ import json import click -from node_cli.core.resources import (get_resource_allocation_info, - generate_resource_allocation_config) +from node_cli.core.resources import ( + get_resource_allocation_info, + generate_resource_allocation_config +) from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts config = session_config() diff --git a/node_cli/cli/ssl.py b/node_cli/cli/ssl.py index 5ac5e9a1..0c6e41c4 100644 --- a/node_cli/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -21,8 +21,9 @@ from terminaltables import SingleTable from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.helper import (get_request, safe_load_texts, upload_certs, - error_exit) +from node_cli.utils.helper import ( + get_request, safe_load_texts, upload_certs, error_exit +) TEXTS = safe_load_texts() diff --git a/node_cli/configs/cli_logger.py b/node_cli/configs/cli_logger.py index 3962e086..9c7720e5 100644 --- a/node_cli/configs/cli_logger.py +++ b/node_cli/configs/cli_logger.py @@ -18,7 +18,7 @@ # along with this program. If not, see . import os -from configs import SKALE_DIR +from node_cli.configs import SKALE_DIR LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' diff --git a/node_cli/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py index d59bc508..c38ca271 100644 --- a/node_cli/configs/resource_allocation.py +++ b/node_cli/configs/resource_allocation.py @@ -18,7 +18,7 @@ # along with this program. If not, see . import os -from configs import NODE_DATA_PATH +from node_cli.configs import NODE_DATA_PATH LARGE_DIVIDER = 1 MEDIUM_DIVIDER = 32 diff --git a/node_cli/core/host.py b/node_cli/core/host.py index c9028ba5..205b4ccd 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -25,18 +25,19 @@ from node_cli.core.resources import update_resource_allocation -from configs import (ADMIN_PORT, - DEFAULT_URL_SCHEME, NODE_DATA_PATH, - SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, - ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, - REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, - MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, - IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) +from node_cli.configs import ( + ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, + SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, + ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, + REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, + MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, + IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH +) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH from node_cli.core.print_formatters import print_abi_validation_errors from node_cli.configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, - SGX_SERVER_URL_FILEPATH) + SGX_SERVER_URL_FILEPATH) from node_cli.utils.helper import safe_load_texts, validate_abi diff --git a/node_cli/core/mysql_backup.py b/node_cli/core/mysql_backup.py index 170958f7..f29fc8b1 100644 --- a/node_cli/core/mysql_backup.py +++ b/node_cli/core/mysql_backup.py @@ -2,7 +2,7 @@ import subprocess import shlex -from configs import MYSQL_BACKUP_CONTAINER_PATH, MYSQL_BACKUP_PATH +from node_cli.configs import MYSQL_BACKUP_CONTAINER_PATH, MYSQL_BACKUP_PATH from node_cli.utils.helper import run_cmd, extract_env_params diff --git a/node_cli/core/node.py b/node_cli/core/node.py index f936dd99..b2ce3344 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,17 +28,17 @@ import docker from node_cli.cli.info import VERSION -from configs import (SKALE_DIR, INSTALL_SCRIPT, UNINSTALL_SCRIPT, - BACKUP_INSTALL_SCRIPT, - DATAFILES_FOLDER, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, - TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) +from node_cli.configs import ( + SKALE_DIR, INSTALL_SCRIPT, UNINSTALL_SCRIPT, + BACKUP_INSTALL_SCRIPT, DATAFILES_FOLDER, INIT_ENV_FILEPATH, + BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, + TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) from node_cli.configs.cli_logger import LOG_DIRNAME from node_cli.core.operations import update_op from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import (is_node_inited, prepare_host, - save_env_params, get_flask_secret_key) + save_env_params, get_flask_secret_key) from node_cli.core.print_formatters import print_node_cmd_error, print_node_info from node_cli.utils.helper import error_exit, get_request, post_request from node_cli.core.resources import update_resource_allocation diff --git a/node_cli/core/operations/common.py b/node_cli/core/operations/common.py index 34a95a6f..aad9156e 100644 --- a/node_cli/core/operations/common.py +++ b/node_cli/core/operations/common.py @@ -24,11 +24,13 @@ from distutils.dir_util import copy_tree from node_cli.core.operations.git_helper import update_repo -from node_cli.utils.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, compose_pull, - compose_build) -from configs import (CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, DOCKER_LVMPY_PATH, - CONTAINER_CONFIG_PATH, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE) +from node_cli.utils.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, + compose_pull, compose_build) +from node_cli.configs import ( + CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, + MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, DOCKER_LVMPY_PATH, + CONTAINER_CONFIG_PATH, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE +) from node_cli.utils.helper import run_cmd, read_json logger = logging.getLogger(__name__) diff --git a/node_cli/core/print_formatters.py b/node_cli/core/print_formatters.py index 729277f5..09929e77 100644 --- a/node_cli/core/print_formatters.py +++ b/node_cli/core/print_formatters.py @@ -25,7 +25,7 @@ import inspect -from configs import LONG_LINE +from node_cli.configs import LONG_LINE from node_cli.configs.cli_logger import DEBUG_LOG_FILEPATH from node_cli.utils.texts import Texts diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 8a979ed9..11f54a84 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -26,7 +26,7 @@ from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, format_output, safe_load_yml -from configs import ALLOCATION_FILEPATH +from node_cli.configs import ALLOCATION_FILEPATH from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, diff --git a/node_cli/core/texts.py b/node_cli/core/texts.py index 5b8ff4be..ecc20a17 100644 --- a/node_cli/core/texts.py +++ b/node_cli/core/texts.py @@ -18,7 +18,7 @@ # along with this program. If not, see . import yaml -from configs import TEXT_FILE +from node_cli.configs import TEXT_FILE class Texts(): diff --git a/node_cli/main.py b/node_cli/main.py index 2c8752f0..36702b68 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -37,7 +37,7 @@ from node_cli.cli.validate import validate_cli from node_cli.cli.resources_allocation import resources_allocation_cli from node_cli.utils.helper import safe_load_texts, init_default_logger -from configs import LONG_LINE +from node_cli.configs import LONG_LINE from node_cli.core.host import init_logs_dir TEXTS = safe_load_texts() diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index b3d199f1..23a7d02e 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -26,8 +26,9 @@ from docker.models.containers import Container from node_cli.utils.helper import run_cmd, str_to_bool -from configs import (COMPOSE_PATH, SKALE_DIR, SGX_CERTIFICATES_DIR_NAME, - REMOVED_CONTAINERS_FOLDER_PATH) +from node_cli.configs import ( + COMPOSE_PATH, SKALE_DIR, SGX_CERTIFICATES_DIR_NAME, REMOVED_CONTAINERS_FOLDER_PATH +) logger = logging.getLogger(__name__) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index cb754196..33d104c6 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -44,12 +44,15 @@ from node_cli.core.print_formatters import print_err_response from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.configs.env import (absent_params as absent_env_params, - get_params as get_env_params) -from configs import CONFIG_FILEPATH, TEXT_FILE, ADMIN_HOST, ADMIN_PORT -from node_cli.configs.cli_logger import (LOG_FORMAT, LOG_BACKUP_COUNT, - LOG_FILE_SIZE_BYTES, - LOG_FILEPATH, DEBUG_LOG_FILEPATH) +from node_cli.configs.env import ( + absent_params as absent_env_params, + get_params as get_env_params +) +from node_cli.configs import CONFIG_FILEPATH, TEXT_FILE, ADMIN_HOST, ADMIN_PORT +from node_cli.configs.cli_logger import ( + LOG_FORMAT, LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, + LOG_FILEPATH, DEBUG_LOG_FILEPATH +) from node_cli.configs.routes import get_route diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 3c0da88c..f5011ef3 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -1,7 +1,7 @@ import json import os from collections import namedtuple -from configs import META_FILEPATH +from node_cli.configs import META_FILEPATH DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' diff --git a/node_cli/utils/texts.py b/node_cli/utils/texts.py index 5352f1af..6f7a8b00 100644 --- a/node_cli/utils/texts.py +++ b/node_cli/utils/texts.py @@ -8,7 +8,7 @@ # along with this program. If not, see . import yaml -from configs import TEXT_FILE +from node_cli.configs import TEXT_FILE class Texts(): diff --git a/setup.py b/setup.py index 89e4dfe6..3f7258da 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def find_version(*file_paths): name='node-cli', # *IMPORTANT*: Don't manually change the version here. # Use the 'bumpversion' utility instead. - version=find_version("cli", "__init__.py"), + version=find_version("node_cli", "cli", "__init__.py"), include_package_data=True, description='SKALE client tools', long_description_markdown_filename='README.md', diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index ceff34b5..c19fad2d 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -23,12 +23,12 @@ import mock import requests -from configs import NODE_DATA_PATH, SKALE_DIR +from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.resources import ResourceAlloc from node_cli.cli.node import (init_node, node_info, register_node, signature, - update_node, backup_node, restore_node, - set_node_in_maintenance, - remove_node_from_maintenance, _turn_off, _turn_on, _set_domain_name) + update_node, backup_node, restore_node, + set_node_in_maintenance, + remove_node_from_maintenance, _turn_off, _turn_on, _set_domain_name) from tests.helper import ( response_mock, run_command_mock, diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 28a260d6..e3969a60 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -24,7 +24,7 @@ from tests.helper import response_mock, run_command_mock from node_cli.cli.schains import (get_schain_config, ls, dkg, show_rules, - repair, info_) + repair, info_) def test_ls(config): diff --git a/tests/cli/validate_test.py b/tests/cli/validate_test.py index 72cb3e2b..183c8be0 100644 --- a/tests/cli/validate_test.py +++ b/tests/cli/validate_test.py @@ -4,8 +4,8 @@ import pytest -from configs import (CONTRACTS_PATH, - IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) +from node_cli.configs import (CONTRACTS_PATH, + IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) from node_cli.cli.validate import abi from tests.helper import run_command diff --git a/tests/conftest.py b/tests/conftest.py index 1eec4fbc..f2eeb003 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ import pytest from readsettings import ReadSettings -from configs import CONFIG_FILEPATH +from node_cli.configs import CONFIG_FILEPATH @pytest.fixture diff --git a/tests/resources_test.py b/tests/resources_test.py index ab4482d9..93db1ded 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -4,7 +4,7 @@ import mock import pytest -from configs import ALLOCATION_FILEPATH +from node_cli.configs import ALLOCATION_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.resources import ( compose_resource_allocation_config, diff --git a/tests/routes_test.py b/tests/routes_test.py index 6e847b07..533a968d 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -1,6 +1,6 @@ import pytest from node_cli.configs.routes import (route_exists, get_route, get_all_available_routes, - RouteNotFoundException) + RouteNotFoundException) ALL_V1_ROUTES = [ diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index cc92c8e5..7cdd68a6 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -3,7 +3,7 @@ import pytest -from configs import META_FILEPATH +from node_cli.configs import META_FILEPATH from node_cli.utils.meta import ( CliMeta, compose_default_meta, DEFAULT_CONFIG_STREAM, DEFAULT_VERSION, From be0fa15b9e200b5e61e2c080a060220ecf07936e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Feb 2021 14:23:43 +0200 Subject: [PATCH 020/140] SKALE-2972 Update .gitignore --- .gitignore | 2 +- node_cli/cli/info.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 node_cli/cli/info.py diff --git a/.gitignore b/.gitignore index a8d77341..77b9aefb 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,4 @@ venv.bak/ .DS_Store # info.py file generated automatically during package build -cli/info.py \ No newline at end of file +node_cli/cli/info.py \ No newline at end of file diff --git a/node_cli/cli/info.py b/node_cli/cli/info.py deleted file mode 100644 index 1c2c908a..00000000 --- a/node_cli/cli/info.py +++ /dev/null @@ -1,5 +0,0 @@ -BUILD_DATETIME = '2021-02-16 14:09:45' -COMMIT = '20b163c683a833b2f10f48a0bdfa9054ba27e2c2' -BRANCH = 'test' -OS = 'Darwin-x86_64' -VERSION = '0.0.0' From 90b0527b3d4ff43fad2cf05559712b2a8d1a46f4 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Feb 2021 14:45:47 +0200 Subject: [PATCH 021/140] SKALE-2972 Fix imports --- node_cli/main.py | 2 +- tests/cli/main_test.py | 2 +- tests/helper.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node_cli/main.py b/node_cli/main.py index 36702b68..f6213e54 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -25,7 +25,7 @@ import click -from cli import __version__ +from node_cli.cli import __version__ from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION from node_cli.cli.logs import logs_cli diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 6643cb4a..ad083272 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -from cli import info +from node_cli.cli import info from main import version from tests.helper import run_command diff --git a/tests/helper.py b/tests/helper.py index c4b66516..23e470be 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -18,7 +18,7 @@ # along with this program. If not, see . import mock -from click.testing import CliRunner +from node_cli.click.testing import CliRunner from mock import Mock, MagicMock From 816b17aedc80b6ad31481fa52073eb5ace65fd88 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Feb 2021 16:06:55 +0200 Subject: [PATCH 022/140] SKALE-2972 Update tests naming, update gitignore --- .gitignore | 4 +- tests/.skale/node_data/meta.json | 1 - tests/cli/exit_test.py | 3 +- tests/cli/health_test.py | 8 +-- tests/cli/main_test.py | 2 +- tests/cli/node_test.py | 82 +++++++++++++------------- tests/cli/resources_allocation_test.py | 12 ++-- tests/cli/schains_test.py | 18 +++--- tests/cli/wallet_test.py | 8 +-- tests/helper.py | 2 +- tests/resources_test.py | 4 +- 11 files changed, 73 insertions(+), 71 deletions(-) delete mode 100644 tests/.skale/node_data/meta.json diff --git a/.gitignore b/.gitignore index 77b9aefb..bd2c23d9 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ venv.bak/ .DS_Store # info.py file generated automatically during package build -node_cli/cli/info.py \ No newline at end of file +node_cli/cli/info.py + +meta.json \ No newline at end of file diff --git a/tests/.skale/node_data/meta.json b/tests/.skale/node_data/meta.json deleted file mode 100644 index 492fcaa9..00000000 --- a/tests/.skale/node_data/meta.json +++ /dev/null @@ -1 +0,0 @@ -{"version": "0.0.0", "config_stream": "master"} \ No newline at end of file diff --git a/tests/cli/exit_test.py b/tests/cli/exit_test.py index ab515634..a243e2b8 100644 --- a/tests/cli/exit_test.py +++ b/tests/cli/exit_test.py @@ -15,6 +15,7 @@ def test_exit_status(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, status, ['--format', 'json']) + result = run_command_mock( + 'node_cli.utils.helper.requests.get', resp_mock, status, ['--format', 'json']) assert result.exit_code == 0 assert result.output == "{'status': 'ACTIVE', 'data': [{'name': 'test', 'status': 'ACTIVE'}], 'exit_time': 0}\n" # noqa diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index b94b4bb4..fbd69e41 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -42,7 +42,7 @@ def test_containers(): requests.codes.ok, json_data=OK_LS_RESPONSE_DATA ) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, containers) assert result.exit_code == 0 assert result.output == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa @@ -69,7 +69,7 @@ def test_checks(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, schains) print(result) @@ -78,7 +78,7 @@ def test_checks(): assert result.exit_code == 0 assert result.output == 'sChain Name Data directory DKG Config file Volume Container IMA Firewall RPC Blocks\n-----------------------------------------------------------------------------------------------------------\ntest_schain True False False False False False False False False \n' # noqa - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, schains, ['--json']) assert result.exit_code == 0 @@ -97,7 +97,7 @@ def test_sgx_status(): json_data={'payload': payload, 'status': 'ok'} ) result = run_command_mock( - 'tools.helper.requests.get', resp_mock, sgx) + 'node_cli.utils.helper.requests.get', resp_mock, sgx) assert result.exit_code == 0 assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status \x1b(0x\x1b(B CONNECTED \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index ad083272..16176ccf 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -19,7 +19,7 @@ from node_cli.cli import info -from main import version +from node_cli.main import version from tests.helper import run_command diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index c19fad2d..e4dfbad7 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -46,7 +46,7 @@ def test_register_node(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, register_node, ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080', '-d', 'skale.test']) @@ -60,7 +60,7 @@ def test_register_node_with_error(config): {'status': 'error', 'payload': ['Strange error']}, ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, register_node, ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test']) @@ -74,7 +74,7 @@ def test_register_node_with_prompted_ip(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, register_node, ['--name', 'test-node', '--port', '8080', '-d', 'skale.test'], input='0.0.0.0\n') @@ -88,7 +88,7 @@ def test_register_node_with_default_port(config): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, register_node, ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') @@ -99,14 +99,14 @@ def test_register_node_with_default_port(config): def test_init_node(config): resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock), \ - mock.patch('core.node.prepare_host'), \ - mock.patch('core.host.init_data_dir'), \ - mock.patch('core.node.is_base_containers_alive', + mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ + mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.host.init_data_dir'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('core.node.is_node_inited', return_value=False): + mock.patch('node_cli.core.node.is_node_inited', return_value=False): result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, init_node, ['./tests/test-env']) @@ -117,9 +117,9 @@ def test_init_node(config): # def test_purge(config): # params = ['--yes'] # resp_mock = response_mock(requests.codes.created) -# with mock.patch('core.node.subprocess.run'): +# with mock.patch('node_cli.core.node.subprocess.run'): # result = run_command_mock( -# 'core.node.post', +# 'node_cli.core.node.post', # resp_mock, # purge_node, # params @@ -133,16 +133,16 @@ def test_update_node(config): params = ['./tests/test-env', '--yes'] resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('core.node.update_op'), \ - mock.patch('core.node.get_flask_secret_key'), \ - mock.patch('core.node.save_env_params'), \ - mock.patch('core.node.prepare_host'), \ - mock.patch('core.node.is_base_containers_alive', + mock.patch('node_cli.core.node.update_op'), \ + mock.patch('node_cli.core.node.get_flask_secret_key'), \ + mock.patch('node_cli.core.node.save_env_params'), \ + mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock), \ - mock.patch('core.host.init_data_dir'): + mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ + mock.patch('node_cli.core.host.init_data_dir'): result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, update_node, params, @@ -155,15 +155,15 @@ def test_update_node_without_init(config): params = ['./tests/test-env', '--yes'] resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('core.node.get_flask_secret_key'), \ - mock.patch('core.node.save_env_params'), \ - mock.patch('core.node.prepare_host'), \ - mock.patch('core.host.init_data_dir'), \ - mock.patch('core.node.is_base_containers_alive', + mock.patch('node_cli.core.node.get_flask_secret_key'), \ + mock.patch('node_cli.core.node.save_env_params'), \ + mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.host.init_data_dir'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('core.node.is_node_inited', return_value=False): + mock.patch('node_cli.core.node.is_node_inited', return_value=False): result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, update_node, params, @@ -191,7 +191,7 @@ def test_node_info_node_info(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n' # noqa @@ -215,7 +215,7 @@ def test_node_info_node_info_not_created(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == 'This SKALE node is not registered on SKALE Manager yet\n' @@ -239,7 +239,7 @@ def test_node_info_node_info_frozen(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' # noqa @@ -263,7 +263,7 @@ def test_node_info_node_info_left(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' # noqa @@ -287,7 +287,7 @@ def test_node_info_node_info_leaving(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' # noqa @@ -311,7 +311,7 @@ def test_node_info_node_info_in_maintenance(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, node_info) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: In Maintenance\n--------------------------------------------------\n' # noqa @@ -323,7 +323,7 @@ def test_node_signature(): 'payload': {'signature': signature_sample} } resp_mock = response_mock(requests.codes.ok, json_data=response_data) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, signature, ['1']) assert result.exit_code == 0 assert result.output == f'Signature: {signature_sample}\n' @@ -331,7 +331,7 @@ def test_node_signature(): def test_backup(): Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - with mock.patch('core.mysql_backup.run_mysql_cmd'): + with mock.patch('node_cli.core.mysql_backup.run_mysql_cmd'): result = run_command( backup_node, [ @@ -366,7 +366,7 @@ def test_maintenance_on(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, set_node_in_maintenance, ['--yes']) @@ -380,7 +380,7 @@ def test_maintenance_off(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, remove_node_from_maintenance) assert result.exit_code == 0 @@ -394,7 +394,7 @@ def test_turn_off_maintenance_on(): ) with mock.patch('subprocess.run', new=subprocess_run_mock): result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, _turn_off, [ @@ -411,9 +411,9 @@ def test_turn_on_maintenance_off(): {'status': 'ok', 'payload': None} ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('core.node.get_flask_secret_key'): + mock.patch('node_cli.core.node.get_flask_secret_key'): result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, _turn_on, [ @@ -433,7 +433,7 @@ def test_set_domain_name(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, _set_domain_name, ['-d', 'skale.test', '--yes']) assert result.exit_code == 0 diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index 7a017040..6cbb3d84 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -55,7 +55,7 @@ def test_show(config, resource_alloc_config): resp_mock = response_mock(requests.codes.created) write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, show ) @@ -66,10 +66,10 @@ def test_show(config, resource_alloc_config): def test_generate(): check_node_dir() resp_mock = response_mock(requests.codes.created) - with mock.patch('core.resources.get_disk_alloc', + with mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock): result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, generate, ['--yes'] @@ -82,10 +82,10 @@ def test_generate(): def test_generate_already_exists(resource_alloc_config): check_node_dir() resp_mock = response_mock(requests.codes.created) - with mock.patch('core.resources.get_disk_alloc', + with mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock): result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, generate, ['--yes'] @@ -94,7 +94,7 @@ def test_generate_already_exists(resource_alloc_config): assert result.exit_code == 0 result = run_command_mock( - 'tools.helper.post_request', + 'node_cli.utils.helper.post_request', resp_mock, generate, ['--yes', '--force'] diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index e3969a60..7f7be4fe 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -49,7 +49,7 @@ def test_ls(config): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, ls) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, ls) assert result.exit_code == 0 assert result.output == ' Name Owner Size Lifetime Created At Deposit \n-----------------------------------------------------------------------------------\ntest_schain1 0x123 0 5 Oct 03 2019 16:09:45 1000000000000000000\ncrazy_cats1 0x321 0 5 Oct 07 2019 18:30:10 1000000000000000000\n' # noqa @@ -70,12 +70,12 @@ def test_dkg(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, dkg) assert result.exit_code == 0 assert result.output == ' sChain Name DKG Status Added At sChain Status\n---------------------------------------------------------------------\nmelodic-aldhibah IN_PROGRESS Jan 08 2020 15:26:52 Exists \n' # noqa - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, dkg, ['--all']) assert result.exit_code == 0 assert result.output == ' sChain Name DKG Status Added At sChain Status\n---------------------------------------------------------------------\nmelodic-aldhibah IN_PROGRESS Jan 08 2020 15:26:52 Exists \n' # noqa @@ -120,7 +120,7 @@ def test_get_schain_config(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, get_schain_config, ['test1']) assert result.exit_code == 0 @@ -145,7 +145,7 @@ def test_schain_rules(): json_data={'payload': payload, 'status': 'ok'} ) result = run_command_mock( - 'tools.helper.requests.get', resp_mock, show_rules, ['schain-test']) + 'node_cli.utils.helper.requests.get', resp_mock, show_rules, ['schain-test']) assert result.exit_code == 0 print(repr(result.output)) assert result.output == 'Port Ip \n-----------------\n10000 127.0.0.1\n10001 127.0.0.1\n10002 None \n10003 None \n10004 127.0.0.1\n10005 127.0.0.1\n10007 None \n10008 None \n' # noqa @@ -159,7 +159,7 @@ def test_repair(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.post', resp_mock, repair, + result = run_command_mock('node_cli.utils.helper.requests.post', resp_mock, repair, ['test-schain', '--yes']) assert result.output == 'Schain has been set for repair\n' assert result.exit_code == 0 @@ -169,7 +169,7 @@ def test_repair(): requests.codes.ok, json_data={'payload': payload, 'status': 'error'} ) - result = run_command_mock('tools.helper.requests.post', resp_mock, repair, + result = run_command_mock('node_cli.utils.helper.requests.post', resp_mock, repair, ['test-schain', '--yes']) print(repr(result.output)) assert result.exit_code == 3 @@ -188,7 +188,7 @@ def test_info(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, info_, + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, info_, ['attractive-ed-asich']) assert result.output == ' Name Id Owner Part_of_node Dkg_status Is_deleted First_run Repair_mode\n--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nattractive-ed-asich 0xfb3b68013fa494407b691b4b603d84c66076c0a5ac96a7d6b162d7341d74fa61 0x1111111111111111111111111111111111111111 0 3 False False False \n' # noqa assert result.exit_code == 0 @@ -198,7 +198,7 @@ def test_info(): requests.codes.ok, json_data={'payload': payload, 'status': 'error'} ) - result = run_command_mock('tools.helper.requests.get', resp_mock, info_, + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, info_, ['schain not found']) assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa assert result.exit_code == 3 diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index 3e0eee6a..aa2b14c9 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -38,7 +38,7 @@ def test_wallet_info(config): response_mock = MagicMock() response_mock.status_code = requests.codes.ok response_mock.json = Mock(return_value=response_data) - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', response_mock, wallet_info) assert result.exit_code == 0 @@ -51,7 +51,7 @@ def test_wallet_info(config): ) assert result.output == expected - result = run_command_mock('tools.helper.requests.get', + result = run_command_mock('node_cli.utils.helper.requests.get', response_mock, wallet_info, ['--format', 'json']) @@ -69,7 +69,7 @@ def test_wallet_send(): {'status': 'ok', 'payload': None} ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, send, ['0x00000000000000000000000000000000', '10', '--yes']) @@ -83,7 +83,7 @@ def test_wallet_send_with_error(): {'status': 'error', 'payload': ['Strange error']}, ) result = run_command_mock( - 'tools.helper.requests.post', + 'node_cli.utils.helper.requests.post', resp_mock, send, ['0x00000000000000000000000000000000', '10', '--yes']) diff --git a/tests/helper.py b/tests/helper.py index 23e470be..c4b66516 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -18,7 +18,7 @@ # along with this program. If not, see . import mock -from node_cli.click.testing import CliRunner +from click.testing import CliRunner from mock import Mock, MagicMock diff --git a/tests/resources_test.py b/tests/resources_test.py index 93db1ded..bebabb63 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -41,7 +41,7 @@ def test_schain_resources_allocation(): def test_generate_resource_allocation_config(): - with mock.patch('core.resources.get_disk_alloc', new=disk_alloc_mock): + with mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock): resource_allocation_config = compose_resource_allocation_config() assert resource_allocation_config['schain']['cpu_shares']['test4'] == 22 @@ -76,7 +76,7 @@ def test_generate_resource_allocation_config(): def test_update_allocation_config(resource_alloc_config): - with mock.patch('core.resources.get_disk_alloc', + with mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock): update_resource_allocation() with open(RESOURCE_ALLOCATION_FILEPATH) as jfile: From fdfb9238995a0b783dddc1cba950d0a52c8c51ce Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 17 Feb 2021 16:38:04 +0200 Subject: [PATCH 023/140] SKALE-2972 Drop readsettings dependency, remove session config, remove tokens file --- node_cli/cli/node.py | 9 +++------ node_cli/cli/resources_allocation.py | 3 +-- node_cli/configs/__init__.py | 5 ----- node_cli/core/node.py | 4 ++-- node_cli/utils/helper.py | 7 +------ scripts/run_tests.sh | 2 +- setup.py | 1 - tests/conftest.py | 18 +----------------- tests/test-skale-cli.yaml | 1 - tests/test-skalecli.yaml | 1 - 10 files changed, 9 insertions(+), 42 deletions(-) delete mode 100644 tests/test-skale-cli.yaml delete mode 100644 tests/test-skalecli.yaml diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 691221d6..31f0138a 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -28,10 +28,9 @@ set_domain_name ) from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts +from node_cli.utils.helper import abort_if_false, safe_load_texts -config = session_config() TEXTS = safe_load_texts() @@ -78,8 +77,7 @@ def node(): @node.command('info', help="Get info about SKALE node") @click.option('--format', '-f', type=click.Choice(['json', 'text'])) def node_info(format): - config = session_config() - get_node_info(config, format) + get_node_info(format) @node.command('register', help="Register current node in the SKALE Manager") @@ -126,8 +124,7 @@ def node_info(format): help='Skip dry run for registration transaction' ) def register_node(name, ip, port, domain, gas_limit, gas_price, skip_dry_run): - config = session_config() - register(config, name, ip, ip, port, domain, gas_limit, gas_price, skip_dry_run) + register(name, ip, ip, port, domain, gas_limit, gas_price, skip_dry_run) @node.command('init', help="Initialize SKALE node") diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py index 1168eacb..2eb1c5b5 100644 --- a/node_cli/cli/resources_allocation.py +++ b/node_cli/cli/resources_allocation.py @@ -24,9 +24,8 @@ get_resource_allocation_info, generate_resource_allocation_config ) -from node_cli.utils.helper import session_config, abort_if_false, safe_load_texts +from node_cli.utils.helper import abort_if_false, safe_load_texts -config = session_config() TEXTS = safe_load_texts() diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index a461f2e7..b482cbf2 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -46,11 +46,6 @@ SGX_CERTS_PATH = os.path.join(NODE_DATA_PATH, 'sgx_certs') SCHAINS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') -CONFIG_FILEPATH = os.environ.get('CONFIG_FILEPATH') or \ - os.path.join(SKALE_DIR, '.skale-cli.yaml') - -TOKENS_FILEPATH = os.path.join(NODE_DATA_PATH, 'tokens.json') - CURRENT_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)) DOTENV_FILEPATH = os.path.join(os.path.dirname(CURRENT_FILE_LOCATION), '.env') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index b2ce3344..c5996f34 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -64,7 +64,7 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def register_node(config, name, p2p_ip, +def register_node(name, p2p_ip, public_ip, port, domain_name, gas_limit=None, gas_price=None, @@ -339,7 +339,7 @@ def is_base_containers_alive(): return len(skale_containers) >= BASE_CONTAINERS_AMOUNT -def get_node_info(config, format): +def get_node_info(format): status, payload = get_request( blueprint=BLUEPRINT_NAME, method='info' diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 33d104c6..5d217eae 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -39,7 +39,6 @@ import click from jinja2 import Environment -from readsettings import ReadSettings from node_cli.core.print_formatters import print_err_response from node_cli.utils.exit_codes import CLIExitCodes @@ -48,7 +47,7 @@ absent_params as absent_env_params, get_params as get_env_params ) -from node_cli.configs import CONFIG_FILEPATH, TEXT_FILE, ADMIN_HOST, ADMIN_PORT +from node_cli.configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT from node_cli.configs.cli_logger import ( LOG_FORMAT, LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, LOG_FILEPATH, DEBUG_LOG_FILEPATH @@ -126,10 +125,6 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def session_config(): - return ReadSettings(CONFIG_FILEPATH) - - def extract_env_params(env_filepath): env_params = get_env_params(env_filepath) if not env_params.get('DB_ROOT_PASSWORD'): diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 5ccc89b2..9b044cf3 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HOME_DIR='tests/' ENV=dev CONFIG_FILEPATH='tests/test-skalecli.yaml' DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ +HOME_DIR='tests/' ENV=dev DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index 3f7258da..324bfbbe 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ def find_version(*file_paths): install_requires=[ "click==7.1.2", "docker==4.2.2", - "readsettings==3.4.5", "PyInstaller==3.6", "texttable==1.6.2", "python-dateutil==2.8.1", diff --git a/tests/conftest.py b/tests/conftest.py index f2eeb003..8d419d67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# This file is part of SKALE.py +# This file is part of node-cli # # Copyright (C) 2019 SKALE Labs # @@ -16,19 +16,3 @@ # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -""" SKALE config test """ - - -import pytest - -from readsettings import ReadSettings -from node_cli.configs import CONFIG_FILEPATH - - -@pytest.fixture -def config(monkeypatch): - cli_config = ReadSettings(CONFIG_FILEPATH) - cli_config['host'] = 'https://test.com' - cli_config.save() - yield - cli_config.clear() diff --git a/tests/test-skale-cli.yaml b/tests/test-skale-cli.yaml deleted file mode 100644 index 03d160ba..00000000 --- a/tests/test-skale-cli.yaml +++ /dev/null @@ -1 +0,0 @@ -host: https://test.com diff --git a/tests/test-skalecli.yaml b/tests/test-skalecli.yaml deleted file mode 100644 index 03d160ba..00000000 --- a/tests/test-skalecli.yaml +++ /dev/null @@ -1 +0,0 @@ -host: https://test.com From f5171062aa5839ad234490c58cf01fa9e12125c6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 17 Feb 2021 19:42:56 +0200 Subject: [PATCH 024/140] SKALE-2972 Restructure modules, add PyYAML dependency --- node_cli/cli/exit.py | 2 +- node_cli/core/health.py | 2 +- node_cli/core/host.py | 2 +- node_cli/core/node.py | 4 +-- node_cli/core/schains.py | 2 +- node_cli/core/texts.py | 36 ------------------- node_cli/core/wallet.py | 2 +- node_cli/{core => }/operations/__init__.py | 2 +- node_cli/{core => }/operations/base.py | 2 +- node_cli/{core => }/operations/common.py | 2 +- .../git_helper.py => utils/git_utils.py} | 0 node_cli/utils/helper.py | 2 +- node_cli/{core => utils}/print_formatters.py | 0 setup.py | 3 +- 14 files changed, 13 insertions(+), 48 deletions(-) delete mode 100644 node_cli/core/texts.py rename node_cli/{core => }/operations/__init__.py (91%) rename node_cli/{core => }/operations/base.py (97%) rename node_cli/{core => }/operations/common.py (98%) rename node_cli/{core/operations/git_helper.py => utils/git_utils.py} (100%) rename node_cli/{core => utils}/print_formatters.py (100%) diff --git a/node_cli/cli/exit.py b/node_cli/cli/exit.py index ff9443cc..1ef0223a 100644 --- a/node_cli/cli/exit.py +++ b/node_cli/cli/exit.py @@ -21,7 +21,7 @@ import click -from node_cli.core.print_formatters import print_exit_status +from node_cli.utils.print_formatters import print_exit_status from node_cli.utils.helper import error_exit, get_request, post_request, abort_if_false from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.texts import Texts diff --git a/node_cli/core/health.py b/node_cli/core/health.py index 3725059a..623c20d7 100644 --- a/node_cli/core/health.py +++ b/node_cli/core/health.py @@ -20,7 +20,7 @@ import json from terminaltables import SingleTable -from node_cli.core.print_formatters import ( +from node_cli.utils.print_formatters import ( print_containers, print_schains_healthchecks ) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 205b4ccd..2377f8f3 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -35,7 +35,7 @@ ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH -from node_cli.core.print_formatters import print_abi_validation_errors +from node_cli.utils.print_formatters import print_abi_validation_errors from node_cli.configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, SGX_SERVER_URL_FILEPATH) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index c5996f34..a172d5e7 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -35,11 +35,11 @@ TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) from node_cli.configs.cli_logger import LOG_DIRNAME -from node_cli.core.operations import update_op +from node_cli.operations import update_op from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import (is_node_inited, prepare_host, save_env_params, get_flask_secret_key) -from node_cli.core.print_formatters import print_node_cmd_error, print_node_info +from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info from node_cli.utils.helper import error_exit, get_request, post_request from node_cli.core.resources import update_resource_allocation from node_cli.utils.meta import update_meta diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index cd961ef2..4904def1 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -3,7 +3,7 @@ from node_cli.utils.helper import get_request, post_request, error_exit from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.core.print_formatters import ( +from node_cli.utils.print_formatters import ( print_dkg_statuses, print_firewall_rules, print_schain_info, diff --git a/node_cli/core/texts.py b/node_cli/core/texts.py deleted file mode 100644 index ecc20a17..00000000 --- a/node_cli/core/texts.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of validator-cli -# -# Copyright (C) 2020 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import yaml -from node_cli.configs import TEXT_FILE - - -class Texts(): - def __init__(self): - self._texts = self._load() - - def __getitem__(self, key): - return self._texts.get(key) - - def _load(self): - with open(TEXT_FILE, 'r') as stream: - try: - return yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) diff --git a/node_cli/core/wallet.py b/node_cli/core/wallet.py index e7d832e0..dd74d6eb 100644 --- a/node_cli/core/wallet.py +++ b/node_cli/core/wallet.py @@ -19,7 +19,7 @@ import json -from node_cli.core.print_formatters import print_wallet_info, TEXTS +from node_cli.utils.print_formatters import print_wallet_info, TEXTS from node_cli.utils.helper import error_exit, get_request, post_request, logger from node_cli.utils.exit_codes import CLIExitCodes diff --git a/node_cli/core/operations/__init__.py b/node_cli/operations/__init__.py similarity index 91% rename from node_cli/core/operations/__init__.py rename to node_cli/operations/__init__.py index 5f28ae8c..46ff12e2 100644 --- a/node_cli/core/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from node_cli.core.operations.base import update as update_op # noqa +from node_cli.operations.base import update as update_op # noqa diff --git a/node_cli/core/operations/base.py b/node_cli/operations/base.py similarity index 97% rename from node_cli/core/operations/base.py rename to node_cli/operations/base.py index eeb10455..e15be206 100644 --- a/node_cli/core/operations/base.py +++ b/node_cli/operations/base.py @@ -19,7 +19,7 @@ from node_cli.cli.info import VERSION from node_cli.core.host import prepare_host -from node_cli.core.operations.common import ( +from node_cli.operations.common import ( remove_dynamic_containers, backup_old_contracts, download_contracts, docker_lvmpy_update, update_skale_node, download_filestorage_artifacts ) diff --git a/node_cli/core/operations/common.py b/node_cli/operations/common.py similarity index 98% rename from node_cli/core/operations/common.py rename to node_cli/operations/common.py index aad9156e..462e7d8a 100644 --- a/node_cli/core/operations/common.py +++ b/node_cli/operations/common.py @@ -23,7 +23,7 @@ import urllib.request from distutils.dir_util import copy_tree -from node_cli.core.operations.git_helper import update_repo +from node_cli.utils.git_utils import update_repo from node_cli.utils.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, compose_pull, compose_build) from node_cli.configs import ( diff --git a/node_cli/core/operations/git_helper.py b/node_cli/utils/git_utils.py similarity index 100% rename from node_cli/core/operations/git_helper.py rename to node_cli/utils/git_utils.py diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 5d217eae..4327e737 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -40,7 +40,7 @@ from jinja2 import Environment -from node_cli.core.print_formatters import print_err_response +from node_cli.utils.print_formatters import print_err_response from node_cli.utils.exit_codes import CLIExitCodes from node_cli.configs.env import ( diff --git a/node_cli/core/print_formatters.py b/node_cli/utils/print_formatters.py similarity index 100% rename from node_cli/core/print_formatters.py rename to node_cli/utils/print_formatters.py diff --git a/setup.py b/setup.py index 324bfbbe..fa9dcb61 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ def find_version(*file_paths): "python-dotenv==0.13.0", "terminaltables==3.1.0", "requests==2.23.0", - "GitPython==3.1.12" + "GitPython==3.1.12", + "PyYAML==5.4.1" ], python_requires='>=3.6,<4', extras_require=extras_require, From 2c03d5387d5d134980de72f4e7c83041dbd78e57 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 17 Feb 2021 19:51:48 +0200 Subject: [PATCH 025/140] SKALE-2972 Remove config fixture --- tests/cli/node_test.py | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index e4dfbad7..3372c9a2 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -40,7 +40,7 @@ def disk_alloc_mock(): return ResourceAlloc(128) -def test_register_node(config): +def test_register_node(): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -54,7 +54,7 @@ def test_register_node(config): assert result.output == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_register_node_with_error(config): +def test_register_node_with_error(): resp_mock = response_mock( requests.codes.ok, {'status': 'error', 'payload': ['Strange error']}, @@ -68,7 +68,7 @@ def test_register_node_with_error(config): assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa -def test_register_node_with_prompted_ip(config): +def test_register_node_with_prompted_ip(): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -82,7 +82,7 @@ def test_register_node_with_prompted_ip(config): assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_register_node_with_default_port(config): +def test_register_node_with_default_port(): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -96,7 +96,7 @@ def test_register_node_with_default_port(config): assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_init_node(config): +def test_init_node(): resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ @@ -114,21 +114,7 @@ def test_init_node(config): assert result.exit_code == 0 -# def test_purge(config): -# params = ['--yes'] -# resp_mock = response_mock(requests.codes.created) -# with mock.patch('node_cli.core.node.subprocess.run'): -# result = run_command_mock( -# 'node_cli.core.node.post', -# resp_mock, -# purge_node, -# params -# ) -# assert result.exit_code == 0 -# assert result.output == '' # noqa - - -def test_update_node(config): +def test_update_node(): os.makedirs(NODE_DATA_PATH, exist_ok=True) params = ['./tests/test-env', '--yes'] resp_mock = response_mock(requests.codes.created) @@ -151,7 +137,7 @@ def test_update_node(config): # assert result.output == 'Updating the node...\nWaiting for transaction manager initialization ...\nUpdate procedure finished\n' # noqa -def test_update_node_without_init(config): +def test_update_node_without_init(): params = ['./tests/test-env', '--yes'] resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ @@ -172,7 +158,7 @@ def test_update_node_without_init(config): assert result.output == "Node hasn't been inited before.\nYou should run < skale node init >\n" # noqa -def test_node_info_node_info(config): +def test_node_info_node_info(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', @@ -196,7 +182,7 @@ def test_node_info_node_info(config): assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n' # noqa -def test_node_info_node_info_not_created(config): +def test_node_info_node_info_not_created(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', @@ -220,7 +206,7 @@ def test_node_info_node_info_not_created(config): assert result.output == 'This SKALE node is not registered on SKALE Manager yet\n' -def test_node_info_node_info_frozen(config): +def test_node_info_node_info_frozen(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', @@ -244,7 +230,7 @@ def test_node_info_node_info_frozen(config): assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' # noqa -def test_node_info_node_info_left(config): +def test_node_info_node_info_left(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', @@ -268,7 +254,7 @@ def test_node_info_node_info_left(config): assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' # noqa -def test_node_info_node_info_leaving(config): +def test_node_info_node_info_leaving(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', @@ -292,7 +278,7 @@ def test_node_info_node_info_leaving(config): assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' # noqa -def test_node_info_node_info_in_maintenance(config): +def test_node_info_node_info_in_maintenance(): payload = { 'node_info': { 'name': 'test', 'ip': '0.0.0.0', From c4e956ce26048cc362a6fff67b2d64739571e3b4 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 17 Feb 2021 20:08:10 +0200 Subject: [PATCH 026/140] SKALE-2972 Remove config fixture --- tests/cli/exit_test.py | 2 +- tests/cli/logs_test.py | 2 +- tests/cli/main_test.py | 2 +- tests/cli/resources_allocation_test.py | 2 +- tests/cli/schains_test.py | 2 +- tests/cli/wallet_test.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cli/exit_test.py b/tests/cli/exit_test.py index a243e2b8..ae6e9530 100644 --- a/tests/cli/exit_test.py +++ b/tests/cli/exit_test.py @@ -4,7 +4,7 @@ from tests.helper import response_mock, run_command_mock -def test_exit_status(config): +def test_exit_status(): payload = { 'status': 'ACTIVE', 'data': [{'name': 'test', 'status': 'ACTIVE'}], diff --git a/tests/cli/logs_test.py b/tests/cli/logs_test.py index 75c849cb..53675bbd 100644 --- a/tests/cli/logs_test.py +++ b/tests/cli/logs_test.py @@ -28,7 +28,7 @@ from node_cli.cli.logs import dump -def test_dump(config): +def test_dump(): archive_filename = 'skale-logs-dump-2019-10-08-17:40:00.tar.gz' resp_mock = response_mock( requests.codes.ok, diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 16176ccf..2703f0b5 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -23,7 +23,7 @@ from tests.helper import run_command -def test_version(config): +def test_version(): result = run_command(version, []) expected = f'SKALE Node CLI version: {info.VERSION}\n' assert result.output == expected diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index 6cbb3d84..20d2ea8a 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -50,7 +50,7 @@ def resource_alloc_config(): os.remove(RESOURCE_ALLOCATION_FILEPATH) -def test_show(config, resource_alloc_config): +def test_show(resource_alloc_config): check_node_dir() resp_mock = response_mock(requests.codes.created) write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 7f7be4fe..6d5d9895 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -27,7 +27,7 @@ repair, info_) -def test_ls(config): +def test_ls(): os.environ['TZ'] = 'Europe/London' time.tzset() payload = [ diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index aa2b14c9..b803eda7 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -26,7 +26,7 @@ from tests.helper import run_command_mock, response_mock -def test_wallet_info(config): +def test_wallet_info(): response_data = { 'status': 'ok', 'payload': { From 8012190ae4b3ef5bbb0e5c0125d52a52565aa9b0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 18 Feb 2021 13:26:51 +0200 Subject: [PATCH 027/140] SKALE-2972 Remove dry_run for node init --- README.md | 6 +----- node_cli/cli/node.py | 9 ++------- node_cli/core/node.py | 4 +--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b4547b8e..e91e8e7b 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,7 @@ Arguments: - `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command) -Required options: - -- `--dry-run` - create only needed files and directories and don't create containers - -You should also specify the following environment variables: +You should specify the following environment variables: - `SGX_SERVER_URL` - SGX server URL - `DISK_MOUNTPOINT` - disk mount point for storing sChains data diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 31f0138a..a45a3933 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -129,13 +129,8 @@ def register_node(name, ip, port, domain, gas_limit, gas_price, skip_dry_run): @node.command('init', help="Initialize SKALE node") @click.argument('env_file') -@click.option( - '--dry-run', - is_flag=True, - help="Dry run node init (don't setup containers)" -) -def init_node(env_file, dry_run): - init(env_file, dry_run) +def init_node(env_file): + init(env_file) @node.command('update', help='Update node from .env file') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index a172d5e7..310d1256 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -95,7 +95,7 @@ def register_node(name, p2p_ip, error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def init(env_filepath, dry_run=False): +def init(env_filepath): if is_node_inited(): print(TEXTS['node']['already_inited']) return @@ -108,11 +108,9 @@ def init(env_filepath, dry_run=False): env_params['SGX_SERVER_URL'] ) update_meta(VERSION, env_params['CONTAINER_CONFIGS_STREAM']) - dry_run = 'yes' if dry_run else '' env = { 'SKALE_DIR': SKALE_DIR, 'DATAFILES_FOLDER': DATAFILES_FOLDER, - 'DRY_RUN': dry_run, **env_params } try: From 1fd0703e4cf381bd671823650095ad2ab6f303f5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 22 Feb 2021 20:33:28 +0200 Subject: [PATCH 028/140] SKALE-2972 Restructure operations module, move init script to python --- datafiles/configure-iptables.sh | 31 +++++++++++ datafiles/install.sh | 53 ------------------ datafiles/update.sh | 42 -------------- main.spec | 3 +- node_cli/configs/__init__.py | 17 ++++-- node_cli/configs/env.py | 5 ++ node_cli/core/host.py | 8 ++- node_cli/core/node.py | 42 ++++---------- node_cli/core/node_config.py | 35 ++++++++++++ node_cli/operations/__init__.py | 5 +- node_cli/operations/base.py | 40 +++++++++++--- node_cli/operations/common.py | 86 ++++++++++++----------------- node_cli/operations/docker_lvmpy.py | 55 ++++++++++++++++++ node_cli/operations/skale_node.py | 67 ++++++++++++++++++++++ node_cli/utils/docker_utils.py | 7 +++ node_cli/utils/git_utils.py | 19 +++++-- tests/operations/common_test.py | 2 +- 17 files changed, 318 insertions(+), 199 deletions(-) create mode 100644 datafiles/configure-iptables.sh delete mode 100644 datafiles/install.sh delete mode 100644 datafiles/update.sh create mode 100644 node_cli/core/node_config.py create mode 100644 node_cli/operations/docker_lvmpy.py create mode 100644 node_cli/operations/skale_node.py diff --git a/datafiles/configure-iptables.sh b/datafiles/configure-iptables.sh new file mode 100644 index 00000000..1d08a9bc --- /dev/null +++ b/datafiles/configure-iptables.sh @@ -0,0 +1,31 @@ +echo "Configuring iptables ..." +mkdir -p /etc/iptables/ +# Base policies (drop all incoming, allow all outcoming, drop all forwarding) +sudo iptables -P INPUT ACCEPT +sudo iptables -P OUTPUT ACCEPT +sudo iptables -P FORWARD DROP +# Allow conntrack established connections +sudo iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +# Allow local loopback services +sudo iptables -A INPUT -i lo -j ACCEPT +# Allow ssh +sudo iptables -A INPUT -p tcp --dport 22 -j ACCEPT +# Allow http +sudo iptables -A INPUT -p tcp --dport 8080 -j ACCEPT +# Allow https +sudo iptables -A INPUT -p tcp --dport 443 -j ACCEPT +# Allow dns +sudo iptables -A INPUT -p tcp --dport 53 -j ACCEPT +sudo iptables -A INPUT -p udp --dport 53 -j ACCEPT +# Allow watchdog +sudo iptables -A INPUT -p tcp --dport 3009 -j ACCEPT +# Allow monitor node exporter +sudo iptables -A INPUT -p tcp --dport 9100 -j ACCEPT +# Drop all the rest +sudo iptables -A INPUT -p tcp -j DROP +sudo iptables -A INPUT -p udp -j DROP +# Allow pings +sudo iptables -I INPUT -p icmp --icmp-type destination-unreachable -j ACCEPT +sudo iptables -I INPUT -p icmp --icmp-type source-quench -j ACCEPT +sudo iptables -I INPUT -p icmp --icmp-type time-exceeded -j ACCEPT +sudo bash -c 'iptables-save > /etc/iptables/rules.v4' diff --git a/datafiles/install.sh b/datafiles/install.sh deleted file mode 100644 index ee239d40..00000000 --- a/datafiles/install.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -set -e - -CONFIG_DIR="$SKALE_DIR"/config -CONTRACTS_DIR="$SKALE_DIR"/contracts_info -NODE_DATA_DIR=$SKALE_DIR/node_data - -source "$DATAFILES_FOLDER"/helper.sh - -if [[ -z $CONTAINER_CONFIGS_DIR ]]; then - cd $CONFIG_DIR - if [[ ! -d .git ]]; then - echo "Cloning container configs ..." - git clone "https://github.com/skalenetwork/skale-node.git" "$CONFIG_DIR" - fi - echo "Fetching new branches and tags..." - git fetch - echo "Checkouting to container configs branch $CONTAINER_CONFIGS_STREAM ..." - git checkout $CONTAINER_CONFIGS_STREAM - is_branch="$(git show-ref --verify refs/heads/$CONTAINER_CONFIGS_STREAM >/dev/null 2>&1; echo $?)" - if [[ $is_branch -eq 0 ]] ; then - echo "Pulling recent changes from $CONTAINER_CONFIGS_STREAM ..." - git pull - fi -else - echo "Syncing container configs ..." - rsync -r $CONTAINER_CONFIGS_DIR/* $CONFIG_DIR - rsync -r $CONTAINER_CONFIGS_DIR/.git $CONFIG_DIR -fi - -echo "Creating .env symlink to $CONFIG_DIR/.env ..." -if [[ -f $CONFIG_DIR/.env ]]; then - rm "$CONFIG_DIR/.env" -fi -ln -s $SKALE_DIR/.env $CONFIG_DIR/.env - -cd $SKALE_DIR - -download_contracts -download_filestorage_artifacts -configure_filebeat -configure_flask -iptables_configure - -if [[ -z $DRY_RUN ]]; then - docker_lvmpy_install - cd $CONFIG_DIR - if [[ ! -z $CONTAINER_CONFIGS_DIR ]]; then - echo "Building containers ..." - SKALE_DIR=$SKALE_DIR docker-compose -f docker-compose.yml build - fi - up_compose -fi diff --git a/datafiles/update.sh b/datafiles/update.sh deleted file mode 100644 index 460261b7..00000000 --- a/datafiles/update.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -e - -CONFIG_DIR="$SKALE_DIR"/config -CONTRACTS_DIR="$SKALE_DIR"/contracts_info -NODE_DATA_DIR=$SKALE_DIR/node_data - -source "$DATAFILES_FOLDER"/helper.sh - -cd $SKALE_DIR - -remove_compose_containers -remove_dynamic_containers - -backup_old_contracts -download_contracts -docker_lvmpy_update - -cd $CONFIG_DIR -if [[ -z $CONTAINER_CONFIGS_DIR ]]; then - echo "Fetching new branches and tags..." - git fetch - echo "Checkouting to container configs branch $CONTAINER_CONFIGS_STREAM ..." - git checkout $CONTAINER_CONFIGS_STREAM - is_branch="$(git show-ref --verify refs/heads/$CONTAINER_CONFIGS_STREAM >/dev/null 2>&1; echo $?)" - if [[ $is_branch -eq 0 ]] ; then - echo "Pulling changes ..." - git pull - fi - echo "Pulling new version of images ..." - SKALE_DIR=$SKALE_DIR docker-compose -f docker-compose.yml pull -else - echo "Syncing configs with CONTAINER_CONFIGS_DIR" - rsync -r "$CONTAINER_CONFIGS_DIR/" "$CONFIG_DIR" - rsync -r "$CONTAINER_CONFIGS_DIR/.git" "$CONFIG_DIR" - echo "Building containers ..." - SKALE_DIR=$SKALE_DIR docker-compose -f docker-compose.yml build -fi - -download_filestorage_artifacts - -up_compose diff --git a/main.spec b/main.spec index 36b83b6f..212df43e 100644 --- a/main.spec +++ b/main.spec @@ -12,9 +12,8 @@ a = Analysis( binaries=[], datas=[ ("./text.yml", "data"), - ("./datafiles/install.sh", "data/datafiles"), + ("./datafiles/configure-iptables.sh", "data/datafiles"), ("./datafiles/backup-install.sh", "data/datafiles"), - ("./datafiles/update.sh", "data/datafiles"), ("./datafiles/helper.sh", "data/datafiles"), ("./datafiles/turn-off.sh", "data/datafiles"), ("./datafiles/turn-on.sh", "data/datafiles") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index b482cbf2..6e9dc356 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -49,9 +49,16 @@ CURRENT_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)) DOTENV_FILEPATH = os.path.join(os.path.dirname(CURRENT_FILE_LOCATION), '.env') +SRC_FILEBEAT_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'filebeat.yml') +FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') + DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' +FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) + + def _get_env(): try: sys._MEIPASS @@ -65,20 +72,19 @@ def _get_env(): if ENV == 'dev': PARDIR = os.path.join(CURRENT_FILE_LOCATION, os.pardir) + PROJECT_DIR = os.path.join(PARDIR, os.pardir) else: PARDIR = os.path.join(sys._MEIPASS, 'data') + PROJECT_DIR = PARDIR - -PROJECT_DIR = os.path.join(PARDIR, os.pardir) TEXT_FILE = os.path.join(PROJECT_DIR, 'text.yml') DATAFILES_FOLDER = os.path.join(PROJECT_DIR, 'datafiles') THIRDPARTY_FOLDER_PATH = os.path.join(DATAFILES_FOLDER, 'third_party') -INSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'install.sh') +CONFIGURE_IPTABLES_SCRIPT = os.path.join(DATAFILES_FOLDER, 'configure-iptables.sh') BACKUP_INSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'backup-install.sh') UNINSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'uninstall.sh') -UPDATE_SCRIPT = os.path.join(DATAFILES_FOLDER, 'update.sh') TURN_OFF_SCRIPT = os.path.join(DATAFILES_FOLDER, 'turn-off.sh') TURN_ON_SCRIPT = os.path.join(DATAFILES_FOLDER, 'turn-on.sh') REDIS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'redis-data') @@ -109,3 +115,6 @@ def _get_env(): IMA_CONTRACTS_FILEPATH = os.path.join(CONTRACTS_PATH, 'ima.json') META_FILEPATH = os.path.join(NODE_DATA_PATH, 'meta.json') + +SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' +DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 515bd84a..994892a2 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -1,5 +1,10 @@ import os from dotenv import load_dotenv +from node_cli.configs import SKALE_DIR, CONTAINER_CONFIG_PATH + + +SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') +CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') base_params = { diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 2377f8f3..312e1c2e 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -35,6 +35,7 @@ ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH +from node_cli.configs.env import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH from node_cli.utils.print_formatters import print_abi_validation_errors from node_cli.configs.resource_allocation import (DISK_MOUNTPOINT_FILEPATH, SGX_SERVER_URL_FILEPATH) @@ -102,7 +103,12 @@ def save_sgx_server_url(sgx_server_url): def save_env_params(env_filepath): - copyfile(env_filepath, os.path.join(SKALE_DIR, '.env')) + copyfile(env_filepath, SKALE_DIR_ENV_FILEPATH) + + +def link_env_file(): + logger.info(f'Creating symlink {SKALE_DIR_ENV_FILEPATH} → {CONFIGS_ENV_FILEPATH}') + os.symlink(SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) def init_logs_dir(): diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 310d1256..0a0931a9 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -27,22 +27,19 @@ import docker -from node_cli.cli.info import VERSION from node_cli.configs import ( - SKALE_DIR, INSTALL_SCRIPT, UNINSTALL_SCRIPT, - BACKUP_INSTALL_SCRIPT, DATAFILES_FOLDER, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, - TURN_OFF_SCRIPT, TURN_ON_SCRIPT, TM_INIT_TIMEOUT) + SKALE_DIR, UNINSTALL_SCRIPT, BACKUP_INSTALL_SCRIPT, DATAFILES_FOLDER, INIT_ENV_FILEPATH, + BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TURN_OFF_SCRIPT, TURN_ON_SCRIPT, + TM_INIT_TIMEOUT +) from node_cli.configs.cli_logger import LOG_DIRNAME -from node_cli.operations import update_op +from node_cli.operations import update_op, init_op from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup -from node_cli.core.host import (is_node_inited, prepare_host, - save_env_params, get_flask_secret_key) +from node_cli.core.host import ( + is_node_inited, save_env_params, get_flask_secret_key) from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info from node_cli.utils.helper import error_exit, get_request, post_request -from node_cli.core.resources import update_resource_allocation -from node_cli.utils.meta import update_meta from node_cli.utils.helper import run_cmd, extract_env_params from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes @@ -102,30 +99,13 @@ def init(env_filepath): env_params = extract_env_params(env_filepath) if env_params is None: return - prepare_host( - env_filepath, - env_params['DISK_MOUNTPOINT'], - env_params['SGX_SERVER_URL'] - ) - update_meta(VERSION, env_params['CONTAINER_CONFIGS_STREAM']) - env = { - 'SKALE_DIR': SKALE_DIR, - 'DATAFILES_FOLDER': DATAFILES_FOLDER, - **env_params - } - try: - run_cmd(['bash', INSTALL_SCRIPT], env=env) - except Exception: - error_msg = 'Install script process errored' - logger.exception(error_msg) - error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - print('Waiting for transaction manager initialization ...') + init_op(env_filepath, env_params) + logger.info('Waiting for transaction manager initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): error_exit('Containers are not running', exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - logger.info('Generating resource allocation file ...') - update_resource_allocation() - print('Init procedure finished') + return + logger.info('Init procedure finished') def restore(backup_path, env_filepath): diff --git a/node_cli/core/node_config.py b/node_cli/core/node_config.py new file mode 100644 index 00000000..c7050918 --- /dev/null +++ b/node_cli/core/node_config.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +class NodeConfig: + def __init__(self, config_filepath, env_filepath=None): + pass + + def load_env(self): + pass + + def validate_env(self): + pass + + def load_config(self): + pass + + def validate_config(self): + pass diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 46ff12e2..53078b23 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -17,4 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from node_cli.operations.base import update as update_op # noqa +from node_cli.operations.base import ( # noqa + update as update_op, + init as init_op +) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index e15be206..6ca4ce98 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -18,12 +18,17 @@ # along with this program. If not, see . from node_cli.cli.info import VERSION -from node_cli.core.host import prepare_host +from node_cli.core.host import prepare_host, link_env_file +from node_cli.core.resources import update_resource_allocation + from node_cli.operations.common import ( - remove_dynamic_containers, backup_old_contracts, download_contracts, docker_lvmpy_update, - update_skale_node, download_filestorage_artifacts + backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, + configure_flask, configure_iptables ) -from node_cli.utils.docker_utils import compose_rm, compose_up +from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install +from node_cli.operations.skale_node import init_skale_node_repo, update_skale_node_repo + +from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta @@ -33,8 +38,9 @@ def update(env_filepath: str, env: str) -> None: backup_old_contracts() download_contracts(env) + download_filestorage_artifacts() docker_lvmpy_update(env) - update_skale_node(env) + update_skale_node_repo(env) prepare_host( env_filepath, @@ -43,12 +49,30 @@ def update(env_filepath: str, env: str) -> None: allocation=True ) update_meta(VERSION, env['CONTAINER_CONFIGS_STREAM']) - download_filestorage_artifacts() compose_up(env) -def init(env): - pass +def init(env_filepath: str, env: str) -> None: + init_skale_node_repo(env) + # todo: add hardware checks + prepare_host( + env_filepath, + env['DISK_MOUNTPOINT'], + env['SGX_SERVER_URL'] + ) + link_env_file() + download_contracts(env) + download_filestorage_artifacts() + + configure_filebeat() + configure_flask() + configure_iptables() + + docker_lvmpy_install(env) + + update_meta(VERSION, env['CONTAINER_CONFIGS_STREAM']) + update_resource_allocation() + compose_up(env) def backup_init(env): diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 462e7d8a..0fdd7ab2 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -19,28 +19,25 @@ import os import logging +import shutil +import secrets import urllib.request +from shutil import copyfile from distutils.dir_util import copy_tree -from node_cli.utils.git_utils import update_repo -from node_cli.utils.docker_utils import (rm_all_schain_containers, rm_all_ima_containers, - compose_pull, compose_build) from node_cli.configs import ( CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, DOCKER_LVMPY_PATH, - CONTAINER_CONFIG_PATH, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE + MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, + FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE, + CONFIGURE_IPTABLES_SCRIPT ) -from node_cli.utils.helper import run_cmd, read_json - -logger = logging.getLogger(__name__) +from node_cli.utils.helper import read_json +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, run_cmd -def remove_dynamic_containers(): - logger.info(f'Removing sChains containers') - rm_all_schain_containers() - logger.info(f'Removing IMA containers') - rm_all_ima_containers() +logger = logging.getLogger(__name__) def backup_old_contracts(): @@ -53,48 +50,37 @@ def download_contracts(env): urllib.request.urlretrieve(env['IMA_CONTRACTS_ABI_URL'], IMA_CONTRACTS_FILEPATH) -def docker_lvmpy_update(env): - update_repo(DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) - logging.info('Running docker-lvmpy update script') - env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] - env['VOLUME_GROUP'] = 'schains' - env['PATH'] = os.environ.get('PATH', None) - run_cmd( - cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(), - env=env - ) - logging.info('docker-lvmpy update done') - - def download_filestorage_artifacts(): - logger.info(f'Updating filestorage artifacts') + logger.info('Updating filestorage artifacts') fs_artifacts_url = read_json(FILESTORAGE_INFO_FILE)['artifacts_url'] logger.debug(f'Downloading {fs_artifacts_url} to {FILESTORAGE_ARTIFACTS_FILE}') urllib.request.urlretrieve(fs_artifacts_url, FILESTORAGE_ARTIFACTS_FILE) -def update_skale_node(env): - if 'CONTAINER_CONFIGS_DIR' in env: - update_skale_node_dev(env) - else: - update_skale_node_git(env) +def configure_filebeat(): + logger.info('Configuring filebeat...') + copyfile(SRC_FILEBEAT_CONFIG_PATH, FILEBEAT_CONFIG_PATH) + shutil.chown(FILEBEAT_CONFIG_PATH, user='root') + os.chmod(FILEBEAT_CONFIG_PATH, 'go-w') + logger.info('Filebeat configured') -def update_skale_node_git(env): - update_repo(CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) - compose_pull() - - -def update_skale_node_dev(env): - sync_skale_node_dev(env) - compose_build() - - -def sync_skale_node_dev(env): - logger.info(f'Syncing {CONTAINER_CONFIG_PATH} with {env["CONTAINER_CONFIGS_DIR"]}') - run_cmd( - cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/ {CONTAINER_CONFIG_PATH}'.split() - ) - run_cmd( - cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/.git {CONTAINER_CONFIG_PATH}'.split() - ) +def configure_flask(): + if os.path.isfile(FLASK_SECRET_KEY_FILE): + logger.info('Flask secret key already exists') + else: + logger.info('Generating Flask secret key...') + flask_secret_key = secrets.token_urlsafe(16) + with open(FLASK_SECRET_KEY_FILE, 'w') as f: + f.write(flask_secret_key) + logger.info('Flask secret key generated and saved') + + +def configure_iptables(): + try: + run_cmd(['bash', CONFIGURE_IPTABLES_SCRIPT]) + except Exception: + error_msg = 'iptables configure script failed' + logger.exception(error_msg) + error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) + logger.info('iptables configured successfully') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py new file mode 100644 index 00000000..f0e107a2 --- /dev/null +++ b/node_cli/operations/docker_lvmpy.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import logging + +from node_cli.utils.helper import run_cmd +from node_cli.utils.git_utils import update_repo, init_repo +from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL + +logger = logging.getLogger(__name__) + + +def update_docker_lvmpy_env(env): + env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['VOLUME_GROUP'] = 'schains' + env['PATH'] = os.environ.get('PATH', None) + return env + + +def docker_lvmpy_update(env): + update_repo(DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + logging.info('Running docker-lvmpy update script') + update_docker_lvmpy_env(env) + run_cmd( + cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(), + env=env + ) + logging.info('docker-lvmpy update done') + + +def docker_lvmpy_install(env): + init_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + update_docker_lvmpy_env(env) + run_cmd( + cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), + env=env + ) + logging.info('docker-lvmpy installed') diff --git a/node_cli/operations/skale_node.py b/node_cli/operations/skale_node.py new file mode 100644 index 00000000..187d17b2 --- /dev/null +++ b/node_cli/operations/skale_node.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from node_cli.utils.helper import run_cmd +from node_cli.utils.git_utils import update_repo, init_repo +from node_cli.utils.docker_utils import compose_pull, compose_build +from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_NODE_REPO_URL + + +logger = logging.getLogger(__name__) + + +def update_skale_node_repo(env): + if 'CONTAINER_CONFIGS_DIR' in env: + update_skale_node_dev(env) + else: + update_skale_node_git(env) + + +def init_skale_node_repo(env): + if 'CONTAINER_CONFIGS_DIR' in env: + update_skale_node_dev(env) + else: + init_skale_node_git(env) + + +def update_skale_node_git(env): + update_repo(CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) + compose_pull() + + +def init_skale_node_git(env): + init_repo(SKALE_NODE_REPO_URL, CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) + compose_pull() + + +def update_skale_node_dev(env): + sync_skale_node_dev(env) + compose_build() + + +def sync_skale_node_dev(env): + logger.info(f'Syncing {CONTAINER_CONFIG_PATH} with {env["CONTAINER_CONFIGS_DIR"]}') + run_cmd( + cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/ {CONTAINER_CONFIG_PATH}'.split() + ) + run_cmd( + cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/.git {CONTAINER_CONFIG_PATH}'.split() + ) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index ad8807ea..debdd3d8 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -58,6 +58,13 @@ def get_all_ima_containers(_all=True) -> list: return docker_client().containers.list(all=_all, filters={'name': 'skale_ima_*'}) +def remove_dynamic_containers(): + logger.info(f'Removing sChains containers') + rm_all_schain_containers() + logger.info(f'Removing IMA containers') + rm_all_ima_containers() + + def rm_all_schain_containers(): schain_containers = get_all_schain_containers() remove_containers(schain_containers, stop_timeout=SCHAIN_REMOVE_TIMEOUT) diff --git a/node_cli/utils/git_utils.py b/node_cli/utils/git_utils.py index 4221967f..90ddafa3 100644 --- a/node_cli/utils/git_utils.py +++ b/node_cli/utils/git_utils.py @@ -22,11 +22,8 @@ from git.repo.base import Repo from git.exc import GitCommandError -logger = logging.getLogger(__name__) - -def init_repo(repo_path: str) -> Repo: - return Repo(repo_path) +logger = logging.getLogger(__name__) def check_is_branch(repo: Repo, ref_name: str) -> bool: @@ -41,8 +38,18 @@ def check_is_branch(repo: Repo, ref_name: str) -> bool: def update_repo(repo_path: str, ref_name: str) -> None: logger.info(f'Updating {repo_path} sources') - repo = init_repo(repo_path) - logger.info(f'Fetching {repo_path} changes') + repo = Repo(repo_path) + fetch_pull_repo(repo, ref_name) + + +def init_repo(repo_url: str, repo_path: str, ref_name: str) -> None: + logger.info(f'Cloning {repo_path}') + repo = Repo.clone_from(repo_url, repo_path) + fetch_pull_repo(repo, ref_name) + + +def fetch_pull_repo(repo: Repo, ref_name: str) -> None: + logger.info(f'Fetching {repo.name} changes') repo.remotes.origin.fetch() logger.info(f'Checkouting docker-lvmpy to {ref_name}') repo.git.checkout(ref_name) diff --git a/tests/operations/common_test.py b/tests/operations/common_test.py index 52d65dee..8a019ffd 100644 --- a/tests/operations/common_test.py +++ b/tests/operations/common_test.py @@ -20,7 +20,7 @@ def test_download_filestorage_artifacts(): assert False -def test_update_skale_node(): +def test_update_skale_node_repo(): assert False From 7e7a15f1d55e95798e613c8c15bf8b455449be3e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 23 Feb 2021 14:12:48 +0200 Subject: [PATCH 029/140] SKALE-2972 Fix symlink, fix repo init, update logger --- node_cli/configs/cli_logger.py | 1 + node_cli/core/host.py | 5 +++-- node_cli/operations/common.py | 2 +- node_cli/utils/git_utils.py | 12 +++++++++--- node_cli/utils/helper.py | 6 +++--- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/node_cli/configs/cli_logger.py b/node_cli/configs/cli_logger.py index 9c7720e5..7ebc3e91 100644 --- a/node_cli/configs/cli_logger.py +++ b/node_cli/configs/cli_logger.py @@ -21,6 +21,7 @@ from node_cli.configs import SKALE_DIR LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +LOG_FORMAT_FILE = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' LOG_FILE_SIZE_MB = 300 LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000 diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 312e1c2e..862e9bc5 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -107,8 +107,9 @@ def save_env_params(env_filepath): def link_env_file(): - logger.info(f'Creating symlink {SKALE_DIR_ENV_FILEPATH} → {CONFIGS_ENV_FILEPATH}') - os.symlink(SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) + if not (os.path.islink(CONFIGS_ENV_FILEPATH) or os.path.isfile(CONFIGS_ENV_FILEPATH)): + logger.info(f'Creating symlink {SKALE_DIR_ENV_FILEPATH} → {CONFIGS_ENV_FILEPATH}') + os.symlink(SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) def init_logs_dir(): diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 0fdd7ab2..b7836237 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -61,7 +61,7 @@ def configure_filebeat(): logger.info('Configuring filebeat...') copyfile(SRC_FILEBEAT_CONFIG_PATH, FILEBEAT_CONFIG_PATH) shutil.chown(FILEBEAT_CONFIG_PATH, user='root') - os.chmod(FILEBEAT_CONFIG_PATH, 'go-w') + os.chmod(FILEBEAT_CONFIG_PATH, 700) logger.info('Filebeat configured') diff --git a/node_cli/utils/git_utils.py b/node_cli/utils/git_utils.py index 90ddafa3..6a3b5dee 100644 --- a/node_cli/utils/git_utils.py +++ b/node_cli/utils/git_utils.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os import logging from git.repo.base import Repo @@ -43,13 +44,18 @@ def update_repo(repo_path: str, ref_name: str) -> None: def init_repo(repo_url: str, repo_path: str, ref_name: str) -> None: - logger.info(f'Cloning {repo_path}') - repo = Repo.clone_from(repo_url, repo_path) + logger.info(f'Cloning {repo_url} → {repo_path}') + if not os.path.isdir(os.path.join(repo_path, '.git')): + repo = Repo.clone_from(repo_url, repo_path) + else: + logger.info(f'{repo_path} git repo already cloned') + repo = Repo(repo_path) fetch_pull_repo(repo, ref_name) def fetch_pull_repo(repo: Repo, ref_name: str) -> None: - logger.info(f'Fetching {repo.name} changes') + repo_name = os.path.basename(repo.working_dir) + logger.info(f'Fetching {repo_name} changes') repo.remotes.origin.fetch() logger.info(f'Checkouting docker-lvmpy to {ref_name}') repo.git.checkout(ref_name) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 4327e737..8101dce2 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -49,8 +49,8 @@ ) from node_cli.configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT from node_cli.configs.cli_logger import ( - LOG_FORMAT, LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, - LOG_FILEPATH, DEBUG_LOG_FILEPATH + LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, + LOG_FILEPATH, DEBUG_LOG_FILEPATH, LOG_FORMAT_FILE ) from node_cli.configs.routes import get_route @@ -260,7 +260,7 @@ def get_stream_handler(): def get_file_handler(log_filepath, log_level): - formatter = Formatter(LOG_FORMAT) + formatter = Formatter(LOG_FORMAT_FILE) f_handler = py_handlers.RotatingFileHandler( log_filepath, maxBytes=LOG_FILE_SIZE_BYTES, backupCount=LOG_BACKUP_COUNT) From b60110a35b7930edc3348344a2e69e8f27c675fa Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Feb 2021 17:23:20 +0200 Subject: [PATCH 030/140] SKALE-2972 Restructure skale node init and update, fix filebeat permissions --- .gitignore | 5 +++- node_cli/core/node.py | 22 ++++++++--------- node_cli/operations/base.py | 6 ++--- node_cli/operations/common.py | 8 +++++- node_cli/operations/docker_lvmpy.py | 10 +++++--- node_cli/operations/skale_node.py | 38 +++++++---------------------- node_cli/utils/git_utils.py | 13 +++------- tests/cli/node_test.py | 2 +- 8 files changed, 46 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index bd2c23d9..a4441006 100644 --- a/.gitignore +++ b/.gitignore @@ -111,4 +111,7 @@ venv.bak/ # info.py file generated automatically during package build node_cli/cli/info.py -meta.json \ No newline at end of file +meta.json + +disk_mountpoint.txt +sgx_server_url.txt \ No newline at end of file diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 0a0931a9..1bac576a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -96,10 +96,10 @@ def init(env_filepath): if is_node_inited(): print(TEXTS['node']['already_inited']) return - env_params = extract_env_params(env_filepath) - if env_params is None: + env = get_node_env(env_filepath) + if env is None: return - init_op(env_filepath, env_params) + init_op(env_filepath, env) logger.info('Waiting for transaction manager initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): @@ -146,7 +146,7 @@ def purge(): print('Success') -def get_inited_node_env(env_filepath, sync_schains): +def get_node_env(env_filepath, inited_node=False, sync_schains=None): if env_filepath is not None: env_params = extract_env_params(env_filepath) if env_params is None: @@ -154,16 +154,17 @@ def get_inited_node_env(env_filepath, sync_schains): save_env_params(env_filepath) else: env_params = extract_env_params(INIT_ENV_FILEPATH) - flask_secret_key = get_flask_secret_key() env = { 'SKALE_DIR': SKALE_DIR, - 'FLASK_SECRET_KEY': flask_secret_key, 'DATAFILES_FOLDER': DATAFILES_FOLDER, **env_params } + if inited_node: + flask_secret_key = get_flask_secret_key() + env['FLASK_SECRET_KEY'] = flask_secret_key if sync_schains: env['BACKUP_RUN'] = 'True' - return env + return {k: v for k, v in env.items() if v != ''} def update(env_filepath, sync_schains): @@ -171,9 +172,8 @@ def update(env_filepath, sync_schains): print(TEXTS['node']['not_inited']) return logger.info('Node update started') - env = get_inited_node_env(env_filepath, sync_schains) - clear_env = {k: v for k, v in env.items() if v != ''} # todo: tmp fix for update procedure - update_op(env_filepath, clear_env) + env = get_node_env(env_filepath, inited_node=True, sync_schains=sync_schains) + update_op(env_filepath, env) logger.info('Waiting for transaction manager initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): @@ -278,7 +278,7 @@ def run_turn_off_script(): def run_turn_on_script(sync_schains, env_filepath): print('Turning on the node...') - env = get_inited_node_env(env_filepath, sync_schains) + env = get_node_env(env_filepath, inited_node=True, sync_schains=sync_schains) try: run_cmd(['bash', TURN_ON_SCRIPT], env=env) except Exception: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 6ca4ce98..1e90a3d6 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -26,7 +26,7 @@ configure_flask, configure_iptables ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install -from node_cli.operations.skale_node import init_skale_node_repo, update_skale_node_repo +from node_cli.operations.skale_node import sync_skale_node from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta @@ -40,7 +40,7 @@ def update(env_filepath: str, env: str) -> None: download_contracts(env) download_filestorage_artifacts() docker_lvmpy_update(env) - update_skale_node_repo(env) + sync_skale_node(env) prepare_host( env_filepath, @@ -53,7 +53,7 @@ def update(env_filepath: str, env: str) -> None: def init(env_filepath: str, env: str) -> None: - init_skale_node_repo(env) + sync_skale_node(env) # todo: add hardware checks prepare_host( env_filepath, diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index b7836237..8b3717f7 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import os +import stat import logging import shutil import secrets @@ -61,7 +62,12 @@ def configure_filebeat(): logger.info('Configuring filebeat...') copyfile(SRC_FILEBEAT_CONFIG_PATH, FILEBEAT_CONFIG_PATH) shutil.chown(FILEBEAT_CONFIG_PATH, user='root') - os.chmod(FILEBEAT_CONFIG_PATH, 700) + os.chmod( + FILEBEAT_CONFIG_PATH, + stat.S_IREAD | + stat.S_IWRITE | + stat.S_IEXEC + ) logger.info('Filebeat configured') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index f0e107a2..a15df8de 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -21,7 +21,7 @@ import logging from node_cli.utils.helper import run_cmd -from node_cli.utils.git_utils import update_repo, init_repo +from node_cli.utils.git_utils import sync_repo from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL logger = logging.getLogger(__name__) @@ -34,8 +34,12 @@ def update_docker_lvmpy_env(env): return env +def sync_docker_lvmpy_repo(env): + sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + + def docker_lvmpy_update(env): - update_repo(DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + sync_docker_lvmpy_repo(env) logging.info('Running docker-lvmpy update script') update_docker_lvmpy_env(env) run_cmd( @@ -46,7 +50,7 @@ def docker_lvmpy_update(env): def docker_lvmpy_install(env): - init_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + sync_docker_lvmpy_repo(env) update_docker_lvmpy_env(env) run_cmd( cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), diff --git a/node_cli/operations/skale_node.py b/node_cli/operations/skale_node.py index 187d17b2..a03b7bea 100644 --- a/node_cli/operations/skale_node.py +++ b/node_cli/operations/skale_node.py @@ -20,7 +20,7 @@ import logging from node_cli.utils.helper import run_cmd -from node_cli.utils.git_utils import update_repo, init_repo +from node_cli.utils.git_utils import sync_repo from node_cli.utils.docker_utils import compose_pull, compose_build from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_NODE_REPO_URL @@ -28,40 +28,20 @@ logger = logging.getLogger(__name__) -def update_skale_node_repo(env): +def sync_skale_node(env): if 'CONTAINER_CONFIGS_DIR' in env: - update_skale_node_dev(env) + sync_skale_node_dev(env) else: - update_skale_node_git(env) + sync_skale_node_git(env) -def init_skale_node_repo(env): - if 'CONTAINER_CONFIGS_DIR' in env: - update_skale_node_dev(env) - else: - init_skale_node_git(env) - - -def update_skale_node_git(env): - update_repo(CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) +def sync_skale_node_git(env): + sync_repo(SKALE_NODE_REPO_URL, CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) compose_pull() -def init_skale_node_git(env): - init_repo(SKALE_NODE_REPO_URL, CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) - compose_pull() - - -def update_skale_node_dev(env): - sync_skale_node_dev(env) - compose_build() - - def sync_skale_node_dev(env): logger.info(f'Syncing {CONTAINER_CONFIG_PATH} with {env["CONTAINER_CONFIGS_DIR"]}') - run_cmd( - cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/ {CONTAINER_CONFIG_PATH}'.split() - ) - run_cmd( - cmd=f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/.git {CONTAINER_CONFIG_PATH}'.split() - ) + run_cmd(f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/ {CONTAINER_CONFIG_PATH}'.split()) + run_cmd(f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/.git {CONTAINER_CONFIG_PATH}'.split()) + compose_build() diff --git a/node_cli/utils/git_utils.py b/node_cli/utils/git_utils.py index 6a3b5dee..e0b9a20d 100644 --- a/node_cli/utils/git_utils.py +++ b/node_cli/utils/git_utils.py @@ -37,19 +37,14 @@ def check_is_branch(repo: Repo, ref_name: str) -> bool: return False -def update_repo(repo_path: str, ref_name: str) -> None: - logger.info(f'Updating {repo_path} sources') - repo = Repo(repo_path) - fetch_pull_repo(repo, ref_name) - - -def init_repo(repo_url: str, repo_path: str, ref_name: str) -> None: - logger.info(f'Cloning {repo_url} → {repo_path}') +def sync_repo(repo_url: str, repo_path: str, ref_name: str) -> None: + logger.info(f'Sync repo {repo_url} → {repo_path}') if not os.path.isdir(os.path.join(repo_path, '.git')): + logger.info(f'Cloning {repo_url} → {repo_path}') repo = Repo.clone_from(repo_url, repo_path) else: - logger.info(f'{repo_path} git repo already cloned') repo = Repo(repo_path) + logger.info(f'Updating {repo_path} sources') fetch_pull_repo(repo, ref_name) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3372c9a2..4b18be1e 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -100,7 +100,7 @@ def test_init_node(): resp_mock = response_mock(requests.codes.created) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ - mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.host.prepare_host'), \ mock.patch('node_cli.core.host.init_data_dir'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ From 21f6b61325a299b8d9eddda4c5c2dc252dacefdf Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 24 Feb 2021 18:01:49 +0200 Subject: [PATCH 031/140] SKALE-2972 Fix node-cli tests --- tests/.skale/.skale-cli-log/.gitkeep | 0 tests/cli/node_test.py | 57 +++++++++++++++++----------- 2 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 tests/.skale/.skale-cli-log/.gitkeep diff --git a/tests/.skale/.skale-cli-log/.gitkeep b/tests/.skale/.skale-cli-log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 4b18be1e..6d7789df 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -22,6 +22,7 @@ import mock import requests +import logging from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.resources import ResourceAlloc @@ -29,12 +30,16 @@ update_node, backup_node, restore_node, set_node_in_maintenance, remove_node_from_maintenance, _turn_off, _turn_on, _set_domain_name) +from node_cli.utils.helper import init_default_logger from tests.helper import ( response_mock, run_command_mock, run_command, subprocess_run_mock ) +logger = logging.getLogger(__name__) +init_default_logger() + def disk_alloc_mock(): return ResourceAlloc(128) @@ -96,22 +101,24 @@ def test_register_node_with_default_port(): assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_init_node(): +def test_init_node(caplog): # todo: write new init node test resp_mock = response_mock(requests.codes.created) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ - mock.patch('node_cli.core.host.prepare_host'), \ - mock.patch('node_cli.core.host.init_data_dir'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', - return_value=True), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=False): - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - init_node, - ['./tests/test-env']) - assert result.output == 'Waiting for transaction manager initialization ...\nInit procedure finished\n' # noqa - assert result.exit_code == 0 + with caplog.at_level(logging.INFO): + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ + mock.patch('node_cli.core.host.prepare_host'), \ + mock.patch('node_cli.core.host.init_data_dir'), \ + mock.patch('node_cli.core.node.init_op'), \ + mock.patch('node_cli.core.node.is_base_containers_alive', + return_value=True), \ + mock.patch('node_cli.core.node.is_node_inited', return_value=False): + result = run_command_mock( + 'node_cli.utils.helper.post_request', + resp_mock, + init_node, + ['./tests/test-env']) + assert 'Init procedure finished' in caplog.text + assert result.exit_code == 0 def test_update_node(): @@ -122,7 +129,7 @@ def test_update_node(): mock.patch('node_cli.core.node.update_op'), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ mock.patch('node_cli.core.node.save_env_params'), \ - mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.host.prepare_host'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ mock.patch('node_cli.core.resources.get_disk_alloc', new=disk_alloc_mock), \ @@ -143,7 +150,7 @@ def test_update_node_without_init(): with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ mock.patch('node_cli.core.node.save_env_params'), \ - mock.patch('node_cli.core.node.prepare_host'), \ + mock.patch('node_cli.core.host.prepare_host'), \ mock.patch('node_cli.core.host.init_data_dir'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ @@ -378,7 +385,8 @@ def test_turn_off_maintenance_on(): requests.codes.ok, {'status': 'ok', 'payload': None} ) - with mock.patch('subprocess.run', new=subprocess_run_mock): + with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -397,7 +405,8 @@ def test_turn_on_maintenance_off(): {'status': 'ok', 'payload': None} ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.get_flask_secret_key'): + mock.patch('node_cli.core.node.get_flask_secret_key'), \ + mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -418,9 +427,11 @@ def test_set_domain_name(): requests.codes.ok, {'status': 'ok', 'payload': None} ) - result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - _set_domain_name, ['-d', 'skale.test', '--yes']) + + with mock.patch('node_cli.core.node.is_node_inited', return_value=True): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + _set_domain_name, ['-d', 'skale.test', '--yes']) assert result.exit_code == 0 assert result.output == 'Setting new domain name: skale.test\nDomain name successfully changed\n' # noqa \ No newline at end of file From 6f5c1cc541ac3dc296549bb94a0ed698dcae9c41 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 4 Mar 2021 13:06:56 +0200 Subject: [PATCH 032/140] Fix tests --- tests/.skale/contracts_info/.gitkeep | 0 tests/cli/node_test.py | 2 +- tests/configs_env_test.py | 2 +- tests/resources_test.py | 12 ++++++------ 4 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 tests/.skale/contracts_info/.gitkeep diff --git a/tests/.skale/contracts_info/.gitkeep b/tests/.skale/contracts_info/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index d92c3aed..4c0ec733 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -345,7 +345,7 @@ def test_restore(): backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('core.resources.get_static_disk_alloc', new=disk_alloc_mock): + mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock): result = run_command( restore_node, [backup_path, './tests/test-env'] diff --git a/tests/configs_env_test.py b/tests/configs_env_test.py index ea2caae8..1fe9ac4e 100644 --- a/tests/configs_env_test.py +++ b/tests/configs_env_test.py @@ -1,4 +1,4 @@ -from configs.env import NotValidEnvParamsError, validate_params +from node_cli.configs.env import NotValidEnvParamsError, validate_params def test_validate_params(): diff --git a/tests/resources_test.py b/tests/resources_test.py index 26da8d2f..73b70b52 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -48,7 +48,7 @@ def test_schain_resources_allocation(): def test_generate_resource_allocation_config(): - with mock.patch('core.resources.get_static_disk_alloc', new=disk_alloc_mock): + with mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) assert resource_allocation_config['schain']['cpu_shares']['test4'] == 22 @@ -91,14 +91,14 @@ def test_update_allocation_config(resource_alloc_config): def test_get_static_disk_alloc_devnet(): - with mock.patch('core.resources.get_disk_size', return_value=SMALL_DISK_SIZE): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=SMALL_DISK_SIZE): with pytest.raises(Exception): get_static_disk_alloc(DEFAULT_ENV_TYPE) - with mock.patch('core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): normal_static_disk_alloc = get_static_disk_alloc(DEFAULT_ENV_TYPE) - with mock.patch('core.resources.get_disk_size', return_value=BIG_DISK_SIZE): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE): big_static_disk_alloc = get_static_disk_alloc(DEFAULT_ENV_TYPE) assert normal_static_disk_alloc.dict() == big_static_disk_alloc.dict() @@ -113,11 +113,11 @@ def test_get_static_disk_alloc_devnet(): def test_get_static_disk_alloc_mainnet(): env_type = 'mainnet' - with mock.patch('core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): with pytest.raises(Exception): get_static_disk_alloc(env_type) - with mock.patch('core.resources.get_disk_size', return_value=BIG_DISK_SIZE): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE): big_static_disk_alloc = get_static_disk_alloc(env_type) assert big_static_disk_alloc.dict() == { From 98ae047fbbb08984a591cd9e8a43e6d6c6b4686d Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 11 Mar 2021 15:54:54 +0200 Subject: [PATCH 033/140] SKALE-3649 Add cli to check that host meet requirements --- node_cli/cli/node.py | 36 ++++- node_cli/configs/__init__.py | 4 + node_cli/configs/cli_logger.py | 2 +- node_cli/core/checks.py | 245 +++++++++++++++++++++++++++++ node_cli/core/host.py | 17 ++ node_cli/core/resources.py | 16 +- node_cli/main.py | 2 +- node_cli/utils/docker_utils.py | 4 +- node_cli/utils/helper.py | 42 +++-- node_cli/utils/print_formatters.py | 14 +- node_cli/utils/texts.py | 2 +- scripts/run_tests.sh | 3 + tests/core_checks_test.py | 220 ++++++++++++++++++++++++++ 13 files changed, 572 insertions(+), 35 deletions(-) create mode 100644 node_cli/core/checks.py create mode 100644 tests/core_checks_test.py diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index c4b5fa2c..a61b8650 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -23,12 +23,16 @@ import click from node_cli.core.node import ( - get_node_signature, init, restore, register_node as register, update, backup, - set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info, + get_node_signature, init, restore, register_node as register, + update, backup, + set_maintenance_mode_on, set_maintenance_mode_off, + turn_off, turn_on, get_node_info, set_domain_name ) +from node_cli.core.host import run_preinstall_checks from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.utils.helper import abort_if_false, safe_load_texts +from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd +from node_cli.utils.print_formatters import print_requirements_check_result TEXTS = safe_load_texts() @@ -123,12 +127,14 @@ def node_info(format): default=False, help='Skip dry run for registration transaction' ) +@streamed_cmd def register_node(name, ip, port, domain, gas_limit, gas_price, skip_dry_run): register(name, ip, ip, port, domain, gas_limit, gas_price, skip_dry_run) @node.command('init', help="Initialize SKALE node") @click.argument('env_file') +@streamed_cmd def init_node(env_file): init(env_file) @@ -138,6 +144,7 @@ def init_node(env_file): expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.argument('env_file') +@streamed_cmd def update_node(env_file): update(env_file) @@ -154,6 +161,7 @@ def signature(validator_id): @click.argument('env_file') @click.option('--no-database', is_flag=True, help="Skip mysql backup") +@streamed_cmd def backup_node(backup_folder_path, env_file, no_database): backup_mysql = True if not no_database else False backup(backup_folder_path, env_file, backup_mysql) @@ -162,6 +170,7 @@ def backup_node(backup_folder_path, env_file, no_database): @node.command('restore', help="Restore SKALE node on another machine") @click.argument('backup_path') @click.argument('env_file') +@streamed_cmd def restore_node(backup_path, env_file): restore(backup_path, env_file) @@ -170,11 +179,13 @@ def restore_node(backup_path, env_file): @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure you want to set SKALE node into maintenance mode?') +@streamed_cmd def set_node_in_maintenance(): set_maintenance_mode_on() @node.command('maintenance-off', help="Remove SKALE node from maintenance mode") +@streamed_cmd def remove_node_from_maintenance(): set_maintenance_mode_off() @@ -188,6 +199,7 @@ def remove_node_from_maintenance(): @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure you want to turn off the node?') +@streamed_cmd def _turn_off(maintenance_on): turn_off(maintenance_on) @@ -208,6 +220,7 @@ def _turn_off(maintenance_on): expose_value=False, prompt='Are you sure you want to turn on the node?') @click.argument('env_file') +@streamed_cmd def _turn_on(maintenance_off, sync_schains, env_file): turn_on(maintenance_off, sync_schains, env_file) @@ -222,5 +235,22 @@ def _turn_on(maintenance_off, sync_schains, env_file): @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure you want to set domain name?') +@streamed_cmd def _set_domain_name(domain): set_domain_name(domain) + + +@node.command(help='Check if node meet network requirements') +@click.option( + '--network', '-n', + type=str, + default='mainnet', + help='Network to check' +) +def check_requirements(network): + result = run_preinstall_checks(network) + if not result: + print('Requirements checking succesfully finished!') + else: + print('Node is not fully meet the requirements!') + print_requirements_check_result(result) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8e42e4d9..641d2e40 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -59,6 +59,10 @@ FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) +REQUIREMENTS_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'requirements.yaml') +DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json' +HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG') + def _get_env(): try: diff --git a/node_cli/configs/cli_logger.py b/node_cli/configs/cli_logger.py index d076f1bb..f0e4dff5 100644 --- a/node_cli/configs/cli_logger.py +++ b/node_cli/configs/cli_logger.py @@ -22,7 +22,7 @@ LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' STREAM_LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' -LOG_FORMAT_FILE = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' +FILE_LOG_FORMAT = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' # noqa LOG_FILE_SIZE_MB = 300 LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000 diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py new file mode 100644 index 00000000..5c32fca2 --- /dev/null +++ b/node_cli/core/checks.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2019 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import inspect +import json +import logging +import os +import psutil +import shutil +import socket +from collections import namedtuple +from typing import List + +import docker +import yaml + +from configs import DOCKER_CONFIG_FILEPATH, REQUIREMENTS_PATH +from tools.helper import run_cmd + +logger = logging.getLogger(__name__) + + +CheckResult = namedtuple('CheckResult', ['name', 'status', 'info']) +ListChecks = List[CheckResult] + + +def get_requirements(network: str = 'mainnet'): + with open(REQUIREMENTS_PATH) as requirements_file: + ydata = yaml.load(requirements_file, Loader=yaml.Loader) + logger.info(ydata) + return ydata[network] + + +class BaseChecker: + def _ok(self, name: str, info=None) -> CheckResult: + return CheckResult(name=name, status='ok', info=info) + + def _error(self, name: str, info=None) -> CheckResult: + return CheckResult(name=name, status='error', info=info) + + def check(self) -> ListChecks: + myself = inspect.stack()[0][3] + check_methods = inspect.getmembers( + type(self), + predicate=lambda m: inspect.isfunction(m) and + not m.__name__.startswith('_') and not m.__name__ == myself + ) + return [cm[1](self) for cm in check_methods] + + +class MachineChecker(BaseChecker): + def __init__(self, requirements: dict) -> None: + self.requirements = requirements + + def cpu_total(self) -> CheckResult: + name = 'cpu_total' + actual = psutil.cpu_count(logical=True) + expected = self.requirements['hardware']['cpu_total'] + info = f'Expected {expected} logical cores, actual {actual} cores' + if actual < expected: + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def cpu_physical(self) -> CheckResult: + name = 'cpu_physical' + actual = psutil.cpu_count(logical=False) + expected = self.requirements['hardware']['cpu_physical'] + info = f'Expected {expected} physical cores, actual {actual} cores' + if actual < expected: + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def memory(self) -> CheckResult: + name = 'memory' + actual = psutil.virtual_memory().total, + actual = actual[0] + expected = self.requirements['hardware']['memory'] + actual_gb = round(actual / 1024 ** 3, 2) + expected_gb = round(expected / 1024 ** 3, 2) + info = f'Expected RAM {expected_gb} GB, actual {actual_gb} GB' + if actual < expected: + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def swap(self) -> CheckResult: + name = 'swap' + actual = psutil.swap_memory().total + expected = self.requirements['hardware']['swap'] + actual_gb = round(actual / 1024 ** 3, 2) + expected_gb = round(expected / 1024 ** 3, 2) + info = f'Expected swap memory {expected_gb} GB, actual {actual_gb} GB' + if actual < expected: + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def network(self) -> CheckResult: + name = 'network' + timeout = 4 + cloudflare_dns_host = '1.1.1.1' + cloudflare_dns_host_port = 443 + try: + socket.setdefaulttimeout(timeout) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( + (cloudflare_dns_host, cloudflare_dns_host_port)) + return self._ok(name=name) + except socket.error as err: + info = f'Network checking returned error: {err}' + return self._error(name=name, info=info) + + +class PackagesChecker(BaseChecker): + def __init__(self, requirements: dict) -> None: + self.requirements = requirements + + def _compose_get_binary_cmd(self, binary_name: str) -> list: + return ['command', '-v', binary_name] + + def docker(self) -> CheckResult: + name = 'docker package' + cmd = shutil.which('docker') + if cmd is None: + info = 'No such command: "docker"' + return self._error(name=name, info=info) + + v_cmd_result = run_cmd(['docker', '-v'], check_code=False) + output = v_cmd_result.stdout.decode('utf-8') + if v_cmd_result.returncode == 0: + return self._ok(name=name, info=output) + else: + return self._error(name=name, info=output) + + def docker_compose(self) -> CheckResult: + name = 'docker-compose' + cmd = shutil.which('docker-compose') + if cmd is None: + info = 'No such command: "docker-compose"' + return self._error(name=name, info=info) + + v_cmd_result = run_cmd(['docker-compose', '-v'], check_code=False) + output = v_cmd_result.stdout.decode('utf-8') + if v_cmd_result.returncode != 0: + output = v_cmd_result.stdout.decode('utf-8') + info = f'Checking docker-compose version failed with: {output}' + return self._error(name=name, info=output) + + actual_version = output.split(',')[0].split()[-1].strip() + expected_version = self.requirements['packages']['docker-compose'] + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } + info = f'Expected docker-compose version {expected_version}, actual {actual_version}' + if actual_version < expected_version: + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def iptables_persistent(self) -> CheckResult: + name = 'iptables-persistent' + dpkg_cmd_result = run_cmd( + ['dpkg', '-s', 'iptables-persistent'], check_code=False) + output = dpkg_cmd_result.stdout.decode('utf-8') + if dpkg_cmd_result.returncode == 0: + return self._ok(name=name, info=output) + else: + return self._error(name=name, info=output) + + def lvm2(self) -> CheckResult: + name = 'lvm2' + dpkg_cmd_result = run_cmd( + ['dpkg', '-s', 'lvm2'], check_code=False) + output = dpkg_cmd_result.stdout.decode('utf-8') + if dpkg_cmd_result.returncode == 0: + return self._ok(name=name, info=output) + else: + return self._error(name=name, info=output) + + +class DockerChecker(BaseChecker): + def __init__(self) -> None: + self.docker_client = docker.from_env() + + def _get_docker_config(self) -> dict: + if not os.path.isfile(DOCKER_CONFIG_FILEPATH): + logger.error(f'No such file {DOCKER_CONFIG_FILEPATH}') + return {} + with open(DOCKER_CONFIG_FILEPATH) as docker_config_file: + try: + docker_config = json.load(docker_config_file) + except json.decoder.JSONDecodeError as err: + logger.error(f'Loading docker config json failed with {err}') + return {} + return docker_config + + def _check_docker_alive_option(self, config: dict) -> tuple: + actual_value = config.get('live-restore', None) + expected_value = True + if actual_value != expected_value: + info = ( + 'Docker daemon live-restore option ' + 'should be set as "true"' + ) + return False, info + else: + info = 'Docker daemon live-restore option is set as "true"' + return True, info + + def keeping_containers_alive(self) -> CheckResult: + name = 'live-restore' + config = self._get_docker_config() + is_ok, info = self._check_docker_alive_option(config) + if is_ok: + return self._ok(name=name, info=info) + else: + return self._error(name=name, info=info) + + def docker_service_status(self) -> CheckResult: + name = 'docker service status' + try: + self.docker_client.containers.list() + except Exception as err: + info = err + return self._error(name=name, info=info) + return self._ok(name=name) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index acc5ccfa..19537a03 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -24,6 +24,10 @@ from urllib.parse import urlparse from node_cli.core.resources import update_resource_allocation +from node_cli.core.checks import ( + get_requirements, DockerChecker, ListChecks, + MachineChecker, PackagesChecker +) from node_cli.configs import ( ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, @@ -76,6 +80,19 @@ def prepare_host(env_filepath, disk_mountpoint, sgx_server_url, env_type, update_resource_allocation(env_type) +def run_preinstall_checks(network: str = 'mainnet') -> ListChecks: + requirements = get_requirements(network) + checkers = [ + MachineChecker(requirements), + PackagesChecker(requirements), + DockerChecker() + ] + result = [] + for checker in checkers: + result.extend(filter(lambda r: r.status == 'error', checker.check())) + return result + + def is_node_inited(): return os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 1a81d25f..235c13be 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -19,7 +19,6 @@ import os import logging -import subprocess from time import sleep import psutil @@ -203,14 +202,9 @@ def check_disk_size(disk_size: int, env_disk_size: int): def get_disk_size(): disk_path = get_disk_path() disk_size_cmd = construct_disk_size_cmd(disk_path) - try: - res = run_cmd(disk_size_cmd, shell=True) - stdout, _ = format_output(res) - return int(stdout) - except subprocess.CalledProcessError: - raise Exception( - "Couldn't get disk size, check disk mountpoint option." - ) + res = run_cmd(disk_size_cmd) + stdout, _ = format_output(res) + return int(stdout) def construct_disk_size_cmd(disk_path): @@ -231,5 +225,5 @@ def get_allocation_option_name(schain): def get_disk_path(): - f = open(DISK_MOUNTPOINT_FILEPATH, "r") - return f.read() + with open(DISK_MOUNTPOINT_FILEPATH) as f: + return f.read().strip() diff --git a/node_cli/main.py b/node_cli/main.py index f6213e54..faf69ec6 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -109,5 +109,5 @@ def handle_exception(exc_type, exc_value, exc_traceback): except Exception as err: print(f'Command execution failed with {err}. Recheck your inputs') traceback.print_exc() - logger.error(err) + logger.exception(f'Command failed with {err}') logger.debug(f'execution time: {time.time() - start_time} seconds') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index debdd3d8..96394730 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -59,9 +59,9 @@ def get_all_ima_containers(_all=True) -> list: def remove_dynamic_containers(): - logger.info(f'Removing sChains containers') + logger.info('Removing sChains containers') rm_all_schain_containers() - logger.info(f'Removing IMA containers') + logger.info('Removing IMA containers') rm_all_ima_containers() diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 2613dab9..0ce7698b 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -26,7 +26,6 @@ import requests import subprocess import urllib.request -from subprocess import PIPE from functools import wraps import logging @@ -47,10 +46,10 @@ absent_params as absent_env_params, get_params as get_env_params ) -from node_cli.configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT +from node_cli.configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT, HIDE_STREAM_LOG from node_cli.configs.cli_logger import ( LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, STREAM_LOG_FORMAT, - LOG_FILEPATH, DEBUG_LOG_FILEPATH, LOG_FORMAT_FILE + LOG_FILEPATH, DEBUG_LOG_FILEPATH, FILE_LOG_FORMAT ) from node_cli.configs.routes import get_route @@ -76,25 +75,28 @@ def write_json(path, content): json.dump(content, outfile, indent=4) -def run_cmd(cmd, env={}, shell=False, secure=False): +def run_cmd(cmd, env={}, shell=False, secure=False, check_code=True): if not secure: logger.debug(f'Running: {cmd}') else: logger.debug('Running some secure command') - res = subprocess.run(cmd, shell=shell, stdout=PIPE, stderr=PIPE, env={**env, **os.environ}) - if res.returncode: - logger.debug(res.stdout.decode('UTF-8').rstrip()) - logger.error('Error during shell execution:') - logger.error(res.stderr.decode('UTF-8').rstrip()) - res.check_returncode() - else: - logger.debug('Command is executed successfully. Command log:') - logger.debug(res.stdout.decode('UTF-8').rstrip()) + res = subprocess.run(cmd, shell=shell, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env={**env, **os.environ}) + if check_code: + output = res.stdout.decode('utf-8') + if res.returncode: + logger.error(f'Error during shell execution: {output}') + res.check_returncode() + else: + logger.debug('Command is executed successfully. Command log:') + logger.debug(res.stdout.decode('UTF-8').rstrip()) return res def format_output(res): - return res.stdout.decode('UTF-8').rstrip(), res.stderr.decode('UTF-8').rstrip() + return res.stdout.decode('UTF-8').rstrip(), \ + res.stderr.decode('UTF-8').rstrip() def download_file(url, filepath): @@ -260,7 +262,7 @@ def get_stream_handler(): def get_file_handler(log_filepath, log_level): - formatter = Formatter(LOG_FORMAT_FILE) + formatter = Formatter(FILE_LOG_FORMAT) f_handler = py_handlers.RotatingFileHandler( log_filepath, maxBytes=LOG_FILE_SIZE_BYTES, backupCount=LOG_BACKUP_COUNT) @@ -315,3 +317,13 @@ def validate_abi(abi_filepath: str) -> dict: return {'filepath': abi_filepath, 'status': 'error', 'msg': 'Failed to load abi file as json'} return {'filepath': abi_filepath, 'status': 'ok', 'msg': ''} + + +def streamed_cmd(func): + """ Decorator that allow function to print logs into stderr """ + @wraps(func) + def wrapper(*args, **kwargs): + if HIDE_STREAM_LOG is None: + logging.getLogger('').addHandler(get_stream_handler()) + return func(*args, **kwargs) + return wrapper diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 09929e77..fb46d578 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -50,7 +50,7 @@ def get_tty_width(): return int(width) -class Formatter(object): +class Formatter: def table(self, headers, rows): table = texttable.Texttable(max_width=get_tty_width()) table.set_cols_dtype(['t' for h in headers]) @@ -288,3 +288,15 @@ def print_err_response(error_payload): print(error_msg) print(LONG_LINE) print(f'You can find more info in {DEBUG_LOG_FILEPATH}') + + +def print_requirements_check_result(result: list) -> None: + headers = ['Check', 'Info'] + rows = [[r.name, r.info] for r in result] + table = texttable.Texttable() + table.add_rows([headers, *rows]) + drawing = table.draw() + main_header = ' Failed checks ' + block_len = (len(drawing.split()[0]) - len(main_header)) // 2 + print('=' * block_len + main_header + '=' * block_len) + print(drawing) diff --git a/node_cli/utils/texts.py b/node_cli/utils/texts.py index 6f7a8b00..d4813d27 100644 --- a/node_cli/utils/texts.py +++ b/node_cli/utils/texts.py @@ -11,7 +11,7 @@ from node_cli.configs import TEXT_FILE -class Texts(): +class Texts: def __init__(self): self._texts = self._load() diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 9b044cf3..a56118d1 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -2,5 +2,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) +export HIDE_STREAM_LOG=true +export ENV=dev # IVD ?? +export HOME_DIR='tests/' HOME_DIR='tests/' ENV=dev DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py new file mode 100644 index 00000000..aa1a106b --- /dev/null +++ b/tests/core_checks_test.py @@ -0,0 +1,220 @@ +import time +from pip._internal import main as pipmain + +import mock +import pytest + +from core.checks import DockerChecker, MachineChecker, PackagesChecker + + +@pytest.fixture +def requirements_data(): + return { + 'hardware': { + 'cpu_total': 1, + 'cpu_physical': 1, + 'memory': 100, + 'swap': 100 + }, + 'packages': { + 'docker-compose': '1.27.4', + 'docker': None, + 'iptables_persistant': None, + 'lvm2': None + } + } + + +def test_checks_cpu_total(requirements_data): + checker = MachineChecker(requirements_data) + r = checker.cpu_total() + assert r.name == 'cpu_total' + assert r.status == 'ok' + requirements_data['hardware']['cpu_total'] = 10000 # too big + checker = MachineChecker(requirements_data) + r = checker.cpu_total() + assert r.name == 'cpu_total' + assert r.status == 'error' + assert checker.cpu_total().status == 'error' + + +def test_checks_cpu_physical(requirements_data): + checker = MachineChecker(requirements_data) + r = checker.cpu_physical() + assert r.name == 'cpu_physical' + assert r.status == 'ok' + requirements_data['hardware']['cpu_physical'] = 10000 # too big + checker = MachineChecker(requirements_data) + r = checker.cpu_physical() + assert r.name == 'cpu_physical' + assert r.status == 'error' + + +def test_checks_memory(requirements_data): + checker = MachineChecker(requirements_data) + r = checker.memory() + assert r.name == 'memory' + assert r.status == 'ok' + # too big + requirements_data['hardware']['memory'] = 10000000000000 + checker = MachineChecker(requirements_data) + r = checker.memory() + assert r.name == 'memory' + assert r.status == 'error' + + +def test_checks_machine_check(requirements_data): + checker = MachineChecker(requirements_data) + result = checker.check() + assert any([r.status == 'ok' for r in result]) + + +def test_checks_swap(requirements_data): + checker = MachineChecker(requirements_data) + r = checker.swap() + assert r.name == 'swap' + assert r.status == 'ok' + # too big + requirements_data['hardware']['swap'] = 10000000000000 + checker = MachineChecker(requirements_data) + r = checker.swap() + assert r.name == 'swap' + assert r.status == 'error' + + +def test_checks_network(requirements_data): + checker = MachineChecker(requirements_data) + r = checker.network() + assert r.status == 'ok' + assert r.name == 'network' + + +def test_checks_docker_version(requirements_data): + checker = PackagesChecker(requirements_data) + res_mock = mock.Mock() + res_mock.stdout = b'Test output' + + def run_cmd_mock(*args, **kwargs): + return res_mock + + res_mock.returncode = 0 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.docker() + r.name == 'docker' + r.status == 'ok' + res_mock.returncode = 1 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.docker() + r.name == 'docker' + r.status == 'error' + + +def test_checks_iptables_persistent(requirements_data): + checker = PackagesChecker(requirements_data) + res_mock = mock.Mock() + res_mock.stdout = b'Test output' + + def run_cmd_mock(*args, **kwargs): + return res_mock + + res_mock.returncode = 0 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.iptables_persistent() + r.name == 'iptables_persistent' + r.status == 'ok' + res_mock.returncode = 1 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.iptables_persistent() + r.name == 'iptables_persistent' + r.status == 'ok' + + +def test_checks_lvm2(requirements_data): + checker = PackagesChecker(requirements_data) + res_mock = mock.Mock() + res_mock.stdout = b'Test output' + + def run_cmd_mock(*args, **kwargs): + return res_mock + + res_mock.returncode = 0 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.lvm2() + r.name == 'lvm2' + r.status == 'ok' + res_mock.returncode = 1 + with mock.patch('core.checks.run_cmd', run_cmd_mock): + r = checker.lvm2() + r.name == 'lvm2' + r.status == 'ok' + + +@pytest.fixture +def docker_compose_pkg_1_27_4(): + pipmain(['install', 'docker-compose==1.27.4']) + time.sleep(10) + yield + pipmain(['uninstall', 'docker-compose', '-y']) + + +@pytest.fixture +def docker_compose_pkg_1_24_1(): + pipmain(['install', 'docker-compose==1.24.1']) + time.sleep(10) + yield + pipmain(['uninstall', 'docker-compose', '-y']) + + +def test_checks_docker_compose_valid_pkg( + requirements_data, docker_compose_pkg_1_27_4): + checker = PackagesChecker(requirements_data) + print('Debug: ', checker.docker_compose()) + + r = checker.docker_compose() + r.name == 'docker-compose' + r.status == 'ok' + + +def test_checks_docker_compose_no_pkg( + requirements_data): + checker = PackagesChecker(requirements_data) + r = checker.docker_compose() + r.name == 'docker-compose' + r.status == 'ok' + + +def test_checks_docker_compose_invalid_version( + requirements_data, docker_compose_pkg_1_24_1): + checker = PackagesChecker(requirements_data) + r = checker.docker_compose() + r.name == 'docker-compose' + r.status == 'ok' + + +def test_checks_docker_service_status(): + checker = DockerChecker() + checker.docker_client = mock.Mock() + r = checker.docker_service_status() + r.name == 'docker-compose' + r.status == 'ok' + + +def test_checks_docker_config(): + checker = DockerChecker() + valid_config = { + 'live-restore': True + } + r = checker._check_docker_alive_option(valid_config) + assert r[0] is True + assert r[1] == 'Docker daemon live-restore option is set as "true"' + + invalid_config = { + 'live-restore': False + } + r = checker._check_docker_alive_option(invalid_config) + assert r[0] is False + assert r[1] == 'Docker daemon live-restore option should be set as "true"' + + r = checker._check_docker_alive_option({}) + assert r[0] is False + assert r[1] == 'Docker daemon live-restore option should be set as "true"' From af648fa031f192cf4b550480fd26b99954354538 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 11 Mar 2021 16:09:32 +0200 Subject: [PATCH 034/140] SKALE-3649 Fix tests --- node_cli/core/checks.py | 4 ++-- tests/core_checks_test.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 5c32fca2..442581e8 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -31,8 +31,8 @@ import docker import yaml -from configs import DOCKER_CONFIG_FILEPATH, REQUIREMENTS_PATH -from tools.helper import run_cmd +from node_cli.configs import DOCKER_CONFIG_FILEPATH, REQUIREMENTS_PATH +from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index aa1a106b..2ec3b578 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -4,7 +4,7 @@ import mock import pytest -from core.checks import DockerChecker, MachineChecker, PackagesChecker +from node_cli.core.checks import DockerChecker, MachineChecker, PackagesChecker @pytest.fixture @@ -98,12 +98,12 @@ def run_cmd_mock(*args, **kwargs): return res_mock res_mock.returncode = 0 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.docker() r.name == 'docker' r.status == 'ok' res_mock.returncode = 1 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.docker() r.name == 'docker' r.status == 'error' @@ -118,12 +118,12 @@ def run_cmd_mock(*args, **kwargs): return res_mock res_mock.returncode = 0 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.iptables_persistent() r.name == 'iptables_persistent' r.status == 'ok' res_mock.returncode = 1 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.iptables_persistent() r.name == 'iptables_persistent' r.status == 'ok' @@ -138,12 +138,12 @@ def run_cmd_mock(*args, **kwargs): return res_mock res_mock.returncode = 0 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.lvm2() r.name == 'lvm2' r.status == 'ok' res_mock.returncode = 1 - with mock.patch('core.checks.run_cmd', run_cmd_mock): + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): r = checker.lvm2() r.name == 'lvm2' r.status == 'ok' From 2931fc690829064df085ac426e789f1eb68bc7d9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 11 Mar 2021 18:40:12 +0200 Subject: [PATCH 035/140] SKALE-2972 Move turn on/off scripts to python, move iptables - wip --- cli/info.py | 5 - datafiles/dependencies.sh | 25 ----- datafiles/turn-off.sh | 7 -- datafiles/turn-on.sh | 11 -- datafiles/uninstall.sh | 13 --- main.spec | 4 +- node_cli/configs/__init__.py | 3 +- node_cli/core/iptables.py | 175 ++++++++++++++++++++++++++++++++ node_cli/core/node.py | 81 ++++----------- node_cli/operations/__init__.py | 5 +- node_cli/operations/base.py | 19 +++- node_cli/operations/common.py | 1 + node_cli/utils/docker_utils.py | 2 +- setup.py | 3 +- 14 files changed, 219 insertions(+), 135 deletions(-) delete mode 100644 cli/info.py delete mode 100644 datafiles/dependencies.sh delete mode 100644 datafiles/turn-off.sh delete mode 100644 datafiles/turn-on.sh delete mode 100644 datafiles/uninstall.sh create mode 100644 node_cli/core/iptables.py diff --git a/cli/info.py b/cli/info.py deleted file mode 100644 index 9e37e0e1..00000000 --- a/cli/info.py +++ /dev/null @@ -1,5 +0,0 @@ -BUILD_DATETIME = '2021-03-02 11:39:19' -COMMIT = '012a875cd13faf8ee9e5ce6f9fdc84fc2eb694cf' -BRANCH = 'test-branch' -OS = 'Darwin-x86_64' -VERSION = '1.0.0' diff --git a/datafiles/dependencies.sh b/datafiles/dependencies.sh deleted file mode 100644 index 7a56e49e..00000000 --- a/datafiles/dependencies.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -sudo apt-get update -# sudo apt-get install \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# gnupg-agent \ -# software-properties-common -# -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo apt-get update -# sudo apt-get install docker-ce docker-ce-cli containerd.io - -# For skipping installation dialog -echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections -echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections -sudo apt install iptables-persistent -y - - -while ! (ps -ef | grep "[d]ocker" | awk {'print $2'}); -do - echo "Waiting for docker daemon file..." - sleep 5 -done diff --git a/datafiles/turn-off.sh b/datafiles/turn-off.sh deleted file mode 100644 index 5fd52584..00000000 --- a/datafiles/turn-off.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e - -source "$DATAFILES_FOLDER"/helper.sh - -remove_compose_containers -remove_dynamic_containers diff --git a/datafiles/turn-on.sh b/datafiles/turn-on.sh deleted file mode 100644 index d520e65e..00000000 --- a/datafiles/turn-on.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e - -CONFIG_DIR="$SKALE_DIR"/config -CONTRACTS_DIR="$SKALE_DIR"/contracts_info -NODE_DATA_DIR=$SKALE_DIR/node_data - -source "$DATAFILES_FOLDER"/helper.sh - -cd $CONFIG_DIR -up_compose diff --git a/datafiles/uninstall.sh b/datafiles/uninstall.sh deleted file mode 100644 index cac47792..00000000 --- a/datafiles/uninstall.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -export CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source $CURRENT_DIR/helper.sh - -remove_compose_containers -remove_dynamic_containers - -# todo: format disk setted in $NODE_DATA_DIR/disk_mountpoint.txt - -rm -rf /tmp/.skale -cp -r $SKALE_DIR /tmp/.skale -rm -rf $SKALE_DIR diff --git a/main.spec b/main.spec index 212df43e..3483c927 100644 --- a/main.spec +++ b/main.spec @@ -14,9 +14,7 @@ a = Analysis( ("./text.yml", "data"), ("./datafiles/configure-iptables.sh", "data/datafiles"), ("./datafiles/backup-install.sh", "data/datafiles"), - ("./datafiles/helper.sh", "data/datafiles"), - ("./datafiles/turn-off.sh", "data/datafiles"), - ("./datafiles/turn-on.sh", "data/datafiles") + ("./datafiles/helper.sh", "data/datafiles") ], hiddenimports=[], hookspath=[], diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8e42e4d9..3eb3f75a 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -55,6 +55,7 @@ DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +IPTABLES_DIR = '/etc/iptables/' FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) @@ -86,8 +87,6 @@ def _get_env(): CONFIGURE_IPTABLES_SCRIPT = os.path.join(DATAFILES_FOLDER, 'configure-iptables.sh') BACKUP_INSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'backup-install.sh') UNINSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'uninstall.sh') -TURN_OFF_SCRIPT = os.path.join(DATAFILES_FOLDER, 'turn-off.sh') -TURN_ON_SCRIPT = os.path.join(DATAFILES_FOLDER, 'turn-on.sh') REDIS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'redis-data') ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py new file mode 100644 index 00000000..8aa446d9 --- /dev/null +++ b/node_cli/core/iptables.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from pathlib import Path + +# from node_cli.configs import IPTABLES_DIR +IPTABLES_DIR = '/etc/iptables/' + + +logger = logging.getLogger(__name__) + +try: + import iptc +except (FileNotFoundError, AttributeError): + logger.warning('Unable to import iptc') + iptc = None + + +ALLOWED_INCOMING_TCP_PORTS = [ + '22', # ssh + '8080', # http + '443', # https + '53', # dns + '3009', # watchdog + '9100' # node exporter +] + +ALLOWED_INCOMING_UDP_PORTS = [ + '53' # dns +] + +LOOPBACK_RULE_D = { + 'protocol': 'tcp', + 'target': 'ACCEPT', + 'in_interface': 'lo' +} + + +ICMP_RULE_D = { + 'protocol': 'icmp', + 'target': 'ACCEPT', + 'icmp_type': 'destination-unreachable' +} + + +def configure_iptables(): + """ + This is the main function used for the initial setup of the firewall rules on the SKALE Node + host machine + """ + if not iptc: + raise Exception('Unable to import iptc package') + Path(IPTABLES_DIR).mkdir(parents=True, exist_ok=True) + + tb = iptc.Table(iptc.Table.FILTER) + input_chain = iptc.Chain(tb, 'INPUT') + + set_base_policies() + allow_loopback(input_chain) + allow_base_ports(input_chain) + drop_all_input(input_chain) + accept_icmp(input_chain) + + +def set_base_policies() -> None: + """Drop all incoming, allow all outcoming, drop all forwarding""" + logger.info('Setting base policies...') + iptc.easy.set_policy(iptc.Table.FILTER, 'INPUT', 'ACCEPT') + iptc.easy.set_policy(iptc.Table.FILTER, 'OUTPUT', 'ACCEPT') + iptc.easy.set_policy(iptc.Table.FILTER, 'FORWARD', 'DROP') + + +def allow_loopback(chain: iptc.Chain) -> None: + logger.info('Allowing loopback packages...') + rule = iptc.Rule() + rule.target = iptc.Target(rule, 'ACCEPT') + rule.in_interface = 'lo' + chain.insert_rule(rule) + + +def drop_all_input(chain: iptc.Chain) -> None: + """Drop all input connections by default (append in the end)""" + logger.info('Droping all input connections except specified...') + r = iptc.Rule() + t = iptc.Target(r, 'DROP') + r.target = t + chain.append_rule(r) + + +def allow_base_ports(chain: iptc.Chain) -> None: + logger.info('Allowing base ports...') + for port in ALLOWED_INCOMING_TCP_PORTS: + accept_incoming(chain, port, 'tcp') + for port in ALLOWED_INCOMING_UDP_PORTS: + accept_incoming(chain, port, 'udp') + + +def accept_incoming(chain, port, protocol) -> None: + rule = iptc.Rule() + rule.protocol = protocol + match = iptc.Match(rule, protocol) + match.dport = port + t = iptc.Target(rule, 'ACCEPT') + rule.target = t + rule.add_match(match) + safe_add_rule(chain, rule) + + +def accept_icmp(chain: iptc.Chain) -> None: + add_icmp_rule(chain, 'destination-unreachable') + add_icmp_rule(chain, 'source-quench') + add_icmp_rule(chain, 'time-exceeded') + + +def add_icmp_rule(chain: iptc.Chain, icmp_type: str) -> None: + rule = iptc.Rule() + rule.protocol = 'icmp' + match = iptc.Match(rule, 'icmp') + match.icmp_type = icmp_type + t = iptc.Target(rule, 'ACCEPT') + rule.target = t + rule.add_match(match) + safe_add_rule(chain, rule) + + +def safe_add_rule(chain: iptc.Chain, rule: iptc.Rule) -> None: + if rule not in chain.rules: + logger.debug(f'Adding rule: {rule_to_dict(rule)}, chain: {chain.name}') + chain.insert_rule(rule) + else: + logger.debug(f'Rule already present: {rule_to_dict(rule)}, chain: {chain.name}') + + +def rule_to_dict(rule): + return { + 'proto': rule.protocol, + 'src': rule.src, + 'dst': rule.dst, + 'in_interface': rule.in_interface, + 'out': rule.out_interface, + 'target': rule.target.name, + } + + +if __name__ == '__main__': + + import sys + from logging import Formatter, StreamHandler + + LOG_FORMAT_FILE = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' # noqa + + formatter = Formatter(LOG_FORMAT_FILE) + stream_handler = StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(logging.INFO) + logging.basicConfig(level=logging.DEBUG, handlers=[stream_handler]) + + configure_iptables() diff --git a/node_cli/core/node.py b/node_cli/core/node.py index d81b7ae2..958f43c4 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,13 +28,12 @@ import docker from node_cli.configs import ( - SKALE_DIR, UNINSTALL_SCRIPT, BACKUP_INSTALL_SCRIPT, DATAFILES_FOLDER, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TURN_OFF_SCRIPT, TURN_ON_SCRIPT, - TM_INIT_TIMEOUT + SKALE_DIR, DATAFILES_FOLDER, INIT_ENV_FILEPATH, + BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TM_INIT_TIMEOUT ) from node_cli.configs.cli_logger import LOG_DIRNAME -from node_cli.operations import update_op, init_op +from node_cli.operations import update_op, init_op, turn_off_op, turn_on_op, restore_op from node_cli.core.resources import update_resource_allocation from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import ( @@ -116,39 +115,21 @@ def restore(backup_path, env_filepath): if env_params is None: return save_env_params(env_filepath) - if not run_restore_script(backup_path, env_params): - return - time.sleep(RESTORE_SLEEP_TIMEOUT) - if not restore_mysql_backup(env_filepath): - print('WARNING: MySQL data restoring failed. ' - 'Check < skale logs cli > for more information') - logger.info('Generating resource allocation file ...') - update_resource_allocation(env_params['ENV_TYPE']) - print('Node is restored from backup') - - -def run_restore_script(backup_path, env_params) -> bool: env = { 'SKALE_DIR': SKALE_DIR, 'DATAFILES_FOLDER': DATAFILES_FOLDER, 'BACKUP_RUN': 'True', - 'BACKUP_PATH': backup_path, 'HOME_DIR': HOME_DIR, **env_params } - try: - run_cmd(['bash', BACKUP_INSTALL_SCRIPT], env=env) - except Exception: - logger.exception('Restore script process errored') - print_node_cmd_error() - return False - return True - - -def purge(): - # todo: check that node is installed - run_cmd(['sudo', 'bash', UNINSTALL_SCRIPT]) - print('Success') + restore_op(env, backup_path) + time.sleep(RESTORE_SLEEP_TIMEOUT) + if not restore_mysql_backup(env_filepath): + print('WARNING: MySQL data restoring failed. ' + 'Check < skale logs cli > for more information') + logger.info('Generating resource allocation file ...') + update_resource_allocation(env_params['ENV_TYPE']) + print('Node is restored from backup') def get_node_env(env_filepath, inited_node=False, sync_schains=None): @@ -266,49 +247,27 @@ def set_maintenance_mode_off(): error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def run_turn_off_script(): - print('Turing off the node...') - cmd_env = { - 'SKALE_DIR': SKALE_DIR, - 'DATAFILES_FOLDER': DATAFILES_FOLDER - } - try: - run_cmd(['bash', TURN_OFF_SCRIPT], env=cmd_env) - except Exception: - error_msg = 'Turning off failed' - logger.exception(error_msg) - error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - print('Node was successfully turned off') - - -def run_turn_on_script(sync_schains, env_filepath): - print('Turning on the node...') - env = get_node_env(env_filepath, inited_node=True, sync_schains=sync_schains) - try: - run_cmd(['bash', TURN_ON_SCRIPT], env=env) - except Exception: - error_msg = 'Turning on failed' - logger.exception(error_msg) - error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - print('Waiting for transaction manager initialization ...') - time.sleep(TM_INIT_TIMEOUT) - print('Node was successfully turned on') - - def turn_off(maintenance_on): if not is_node_inited(): print(TEXTS['node']['not_inited']) return if maintenance_on: set_maintenance_mode_on() - run_turn_off_script() + turn_off_op() def turn_on(maintenance_off, sync_schains, env_file): if not is_node_inited(): print(TEXTS['node']['not_inited']) return - run_turn_on_script(sync_schains, env_file) + env = get_node_env(env_file, inited_node=True, sync_schains=sync_schains) + turn_on_op(env) + logger.info('Waiting for transaction manager initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(): + print_node_cmd_error() + return + logger.info('Node turned on') if maintenance_off: set_maintenance_mode_off() diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 53078b23..5bc6cef3 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -19,5 +19,8 @@ from node_cli.operations.base import ( # noqa update as update_op, - init as init_op + init as init_op, + turn_off as turn_off_op, + turn_on as turn_on_op, + restore as restore_op ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 45975830..0ef8f4ec 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging + from node_cli.cli.info import VERSION from node_cli.core.host import prepare_host, link_env_file from node_cli.core.resources import update_resource_allocation @@ -32,6 +34,9 @@ from node_cli.utils.meta import update_meta +logger = logging.getLogger(__name__) + + def update(env_filepath: str, env: str) -> None: compose_rm(env) remove_dynamic_containers() @@ -85,13 +90,17 @@ def init(env_filepath: str, env: str) -> None: compose_up(env) -def backup_init(env): - pass +def turn_off(): + logger.info('Turning off the node...') + compose_rm() + remove_dynamic_containers() + logger.info('Node was successfully turned off') -def turn_off(env): - pass +def turn_on(env): + logger.info('Turning on the node...') + compose_up(env) -def turn_on(env): +def restore(env, backup_path): pass diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 8b3717f7..e9e14556 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -83,6 +83,7 @@ def configure_flask(): def configure_iptables(): + # todo!!! try: run_cmd(['bash', CONFIGURE_IPTABLES_SCRIPT]) except Exception: diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index debdd3d8..54cfa3ca 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -110,7 +110,7 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) -def compose_rm(env): +def compose_rm(env={}): logger.info(f'Removing {MAIN_COMPOSE_CONTAINERS} containers') run_cmd( cmd=f'docker-compose -f {COMPOSE_PATH} rm -s -f {MAIN_COMPOSE_CONTAINERS}'.split(), diff --git a/setup.py b/setup.py index fa9dcb61..50b7818b 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,8 @@ def find_version(*file_paths): "terminaltables==3.1.0", "requests==2.23.0", "GitPython==3.1.12", - "PyYAML==5.4.1" + "PyYAML==5.4.1", + "python-iptables==1.0.0" ], python_requires='>=3.6,<4', extras_require=extras_require, From eb6757dbfe198255926baf1d3af06895af031005 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 11 Mar 2021 18:44:11 +0200 Subject: [PATCH 036/140] SKALE-3649 Add new packages to checks --- node_cli/core/checks.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 442581e8..c9edb64e 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -133,9 +133,6 @@ class PackagesChecker(BaseChecker): def __init__(self, requirements: dict) -> None: self.requirements = requirements - def _compose_get_binary_cmd(self, binary_name: str) -> list: - return ['command', '-v', binary_name] - def docker(self) -> CheckResult: name = 'docker package' cmd = shutil.which('docker') @@ -170,31 +167,37 @@ def docker_compose(self) -> CheckResult: 'expected_version': expected_version, 'actual_version': actual_version } - info = f'Expected docker-compose version {expected_version}, actual {actual_version}' + info = f'Expected docker-compose version {expected_version}, actual {actual_version}' # noqa if actual_version < expected_version: return self._error(name=name, info=info) else: return self._ok(name=name, info=info) def iptables_persistent(self) -> CheckResult: - name = 'iptables-persistent' - dpkg_cmd_result = run_cmd( - ['dpkg', '-s', 'iptables-persistent'], check_code=False) - output = dpkg_cmd_result.stdout.decode('utf-8') - if dpkg_cmd_result.returncode == 0: - return self._ok(name=name, info=output) - else: - return self._error(name=name, info=output) + return self._check_apt_package('iptables-persistent') def lvm2(self) -> CheckResult: - name = 'lvm2' + return self._check_apt_package('lvm2') + + def btrfs_progs(self) -> CheckResult: + return self._check_apt_package('btrfs-progs') + + def lsof(self) -> CheckResult: + return self._check_apt_package('lsof') + + def psmisc(self) -> CheckResult: + return self._check_apt_package('psmisc') + + def _check_apt_package(self, package_name: str, + version: str = None) -> CheckResult: + # TODO: check versions dpkg_cmd_result = run_cmd( ['dpkg', '-s', 'lvm2'], check_code=False) output = dpkg_cmd_result.stdout.decode('utf-8') if dpkg_cmd_result.returncode == 0: - return self._ok(name=name, info=output) + return self._ok(name=package_name, info=output) else: - return self._error(name=name, info=output) + return self._error(name=package_name, info=output) class DockerChecker(BaseChecker): From 2a0e25c54d93e8b1b39aa9dd9544c3e8344a5114 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 12 Mar 2021 12:51:00 +0200 Subject: [PATCH 037/140] SKALE-2972 Allow conntrack, reduce log levels --- node_cli/core/iptables.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 8aa446d9..3be03fde 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -65,6 +65,7 @@ def configure_iptables(): This is the main function used for the initial setup of the firewall rules on the SKALE Node host machine """ + logger.info('Configuring iptables...') if not iptc: raise Exception('Unable to import iptc package') Path(IPTABLES_DIR).mkdir(parents=True, exist_ok=True) @@ -77,27 +78,40 @@ def configure_iptables(): allow_base_ports(input_chain) drop_all_input(input_chain) accept_icmp(input_chain) + allow_conntrack(input_chain) def set_base_policies() -> None: """Drop all incoming, allow all outcoming, drop all forwarding""" - logger.info('Setting base policies...') + logger.debug('Setting base policies...') iptc.easy.set_policy(iptc.Table.FILTER, 'INPUT', 'ACCEPT') iptc.easy.set_policy(iptc.Table.FILTER, 'OUTPUT', 'ACCEPT') iptc.easy.set_policy(iptc.Table.FILTER, 'FORWARD', 'DROP') def allow_loopback(chain: iptc.Chain) -> None: - logger.info('Allowing loopback packages...') + logger.debug('Allowing loopback packages...') rule = iptc.Rule() rule.target = iptc.Target(rule, 'ACCEPT') rule.in_interface = 'lo' chain.insert_rule(rule) +def allow_conntrack(chain: iptc.Chain) -> None: + logger.debug('Allowing conntrack...') + rule = iptc.Rule() + rule.protocol = "tcp" + rule.target = iptc.Target(rule, "ACCEPT") + match = iptc.Match(rule, "conntrack") + chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT") + match.ctstate = "RELATED,ESTABLISHED" + rule.add_match(match) + chain.insert_rule(rule) + + def drop_all_input(chain: iptc.Chain) -> None: """Drop all input connections by default (append in the end)""" - logger.info('Droping all input connections except specified...') + logger.debug('Droping all input connections except specified...') r = iptc.Rule() t = iptc.Target(r, 'DROP') r.target = t @@ -105,7 +119,7 @@ def drop_all_input(chain: iptc.Chain) -> None: def allow_base_ports(chain: iptc.Chain) -> None: - logger.info('Allowing base ports...') + logger.debug('Allowing base ports...') for port in ALLOWED_INCOMING_TCP_PORTS: accept_incoming(chain, port, 'tcp') for port in ALLOWED_INCOMING_UDP_PORTS: From a57c8ea6ede4f2bd176b9f0bf3cdd54c5a25eb11 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 12 Mar 2021 13:18:29 +0200 Subject: [PATCH 038/140] SKALE-2972 Remove datafiles --- datafiles/backup-install.sh | 30 ------ datafiles/configure-iptables.sh | 31 ------ datafiles/helper.sh | 163 -------------------------------- main.spec | 5 +- node_cli/configs/__init__.py | 7 -- node_cli/core/iptables.py | 35 +------ node_cli/core/node.py | 4 +- node_cli/operations/base.py | 3 +- node_cli/operations/common.py | 16 +--- 9 files changed, 8 insertions(+), 286 deletions(-) delete mode 100644 datafiles/backup-install.sh delete mode 100644 datafiles/configure-iptables.sh delete mode 100644 datafiles/helper.sh diff --git a/datafiles/backup-install.sh b/datafiles/backup-install.sh deleted file mode 100644 index d8ae74db..00000000 --- a/datafiles/backup-install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -e - -CONFIG_DIR="$SKALE_DIR"/config -CONTRACTS_DIR="$SKALE_DIR"/contracts_info -NODE_DATA_DIR=$SKALE_DIR/node_data - - -source "$DATAFILES_FOLDER"/helper.sh - -tar -zxvf $BACKUP_PATH -C $HOME_DIR - -echo "Creating .env symlink to $CONFIG_DIR/.env ..." -if [[ -f $CONFIG_DIR/.env ]]; then - rm "$CONFIG_DIR/.env" -fi - -ln -sf "$SKALE_DIR/.env" "$CONFIG_DIR/.env" - -cd $SKALE_DIR - -iptables_configure -docker_lvmpy_install - -cd $CONFIG_DIR -if [[ ! -z $CONTAINER_CONFIGS_DIR ]]; then - echo "Building containers ..." - SKALE_DIR=$SKALE_DIR docker-compose -f docker-compose.yml build -fi -up_compose diff --git a/datafiles/configure-iptables.sh b/datafiles/configure-iptables.sh deleted file mode 100644 index 1d08a9bc..00000000 --- a/datafiles/configure-iptables.sh +++ /dev/null @@ -1,31 +0,0 @@ -echo "Configuring iptables ..." -mkdir -p /etc/iptables/ -# Base policies (drop all incoming, allow all outcoming, drop all forwarding) -sudo iptables -P INPUT ACCEPT -sudo iptables -P OUTPUT ACCEPT -sudo iptables -P FORWARD DROP -# Allow conntrack established connections -sudo iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT -# Allow local loopback services -sudo iptables -A INPUT -i lo -j ACCEPT -# Allow ssh -sudo iptables -A INPUT -p tcp --dport 22 -j ACCEPT -# Allow http -sudo iptables -A INPUT -p tcp --dport 8080 -j ACCEPT -# Allow https -sudo iptables -A INPUT -p tcp --dport 443 -j ACCEPT -# Allow dns -sudo iptables -A INPUT -p tcp --dport 53 -j ACCEPT -sudo iptables -A INPUT -p udp --dport 53 -j ACCEPT -# Allow watchdog -sudo iptables -A INPUT -p tcp --dport 3009 -j ACCEPT -# Allow monitor node exporter -sudo iptables -A INPUT -p tcp --dport 9100 -j ACCEPT -# Drop all the rest -sudo iptables -A INPUT -p tcp -j DROP -sudo iptables -A INPUT -p udp -j DROP -# Allow pings -sudo iptables -I INPUT -p icmp --icmp-type destination-unreachable -j ACCEPT -sudo iptables -I INPUT -p icmp --icmp-type source-quench -j ACCEPT -sudo iptables -I INPUT -p icmp --icmp-type time-exceeded -j ACCEPT -sudo bash -c 'iptables-save > /etc/iptables/rules.v4' diff --git a/datafiles/helper.sh b/datafiles/helper.sh deleted file mode 100644 index 4250b753..00000000 --- a/datafiles/helper.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -export FLASK_SECRET_KEY_FILE=$NODE_DATA_DIR/flask_db_key.txt -export DISK_MOUNTPOINT_FILE=$NODE_DATA_DIR/disk_mountpoint.txt -export SGX_CERTIFICATES_DIR_NAME=sgx_certs - -export FILESTORAGE_INFO_FILE=$SKALE_DIR/config/filestorage_info.json -export FILESTORAGE_ARTIFACTS_FILE=$NODE_DATA_DIR/filestorage_artifacts.json - -export CONTRACTS_DIR="$SKALE_DIR"/contracts_info -export BACKUP_CONTRACTS_DIR="$SKALE_DIR"/.old_contracts_info - -export BASE_SERVICES="transaction-manager skale-admin skale-api mysql bounty nginx watchdog filebeat" -export NOTIFICATION_SERVICES="celery redis" - -remove_dynamic_containers () { - echo 'Removing schains containers ...' - SCHAIN_CONTAINERS="$(docker ps -a --format '{{.Names}}' | grep '^skale_schain_' | awk '{print $1}' | xargs)" - for CONTAINER in $SCHAIN_CONTAINERS; do - echo 'Stopping' $CONTAINER - docker stop $CONTAINER -t 40 - echo 'Removing' $CONTAINER - docker rm $CONTAINER - done - - echo 'Removing ima containers...' - IMA_CONTAINERS="$(docker ps -a --format '{{.Names}}' | grep '^skale_ima_' | awk '{print $1}' | xargs)" - for CONTAINER in $IMA_CONTAINERS; do - echo 'Stopping' $CONTAINER - docker stop $CONTAINER -t 40 - echo 'Removing' $CONTAINER - docker rm $CONTAINER - done -} - -remove_compose_containers () { - echo 'Removing node containers ...' - COMPOSE_PATH=$SKALE_DIR/config/docker-compose.yml - echo 'Removing api, bounty, admin containers ...' - DB_PORT=0 docker-compose -f $COMPOSE_PATH rm -s -f skale-api bounty skale-admin - echo 'Api, bounty, admin containers were removed. Sleeping ...' - sleep 7 - echo 'Removing transaction-manager, mysql and rest ...' - DB_PORT=0 docker-compose -f $COMPOSE_PATH rm -s -f - echo 'Done' -} - -download_contracts () { - echo "Downloading contracts ABI ..." - curl -L $MANAGER_CONTRACTS_ABI_URL > $CONTRACTS_DIR/manager.json - curl -L $IMA_CONTRACTS_ABI_URL > $CONTRACTS_DIR/ima.json -} - -download_filestorage_artifacts () { - echo "Downloading filestorage artifacts ..." - FS_ARTIFACTS_URL=$(cat $FILESTORAGE_INFO_FILE | sed -n 's/^[[:space:]]*"artifacts_url"[[:space:]]*:[[:space:]]*//p') - FS_ARTIFACTS_URL=$(echo "$FS_ARTIFACTS_URL" | sed -r 's/["]+//g') - curl -L $FS_ARTIFACTS_URL > $FILESTORAGE_ARTIFACTS_FILE -} - -backup_old_contracts () { - echo "Copying old contracts ABI ..." - cp -R $CONTRACTS_DIR $BACKUP_CONTRACTS_DIR -} - - -update_docker_lvmpy_sources () { - echo 'Updating docker-lvmpy sources ...' - cd docker-lvmpy - echo "Fetching changes ..." - git fetch - echo "Checkouting to $DOCKER_LVMPY_STREAM ..." - git checkout $DOCKER_LVMPY_STREAM - is_branch="$(git show-ref --verify refs/heads/$DOCKER_LVMPY_STREAM > /dev/null 2>&1; echo $?)" - if [[ $is_branch -eq 0 ]] ; then - echo "Pulling recent changes from $DOCKER_LVMPY_STREAM ..." - git pull - fi -} - -docker_lvmpy_install () { - echo 'Installing docker-lvmpy ...' - if [[ ! -d docker-lvmpy ]]; then - git clone "https://github.com/skalenetwork/docker-lvmpy.git" - fi - update_docker_lvmpy_sources - echo "Running install.sh script ..." - sudo -H PHYSICAL_VOLUME=$DISK_MOUNTPOINT VOLUME_GROUP=schains PATH=$PATH scripts/install.sh - cd - -} - -docker_lvmpy_update () { - update_docker_lvmpy_sources - echo "Running update.sh script ..." - sudo -H PHYSICAL_VOLUME=$DISK_MOUNTPOINT VOLUME_GROUP=schains PATH=$PATH scripts/update.sh - cd - -} - -iptables_configure() { - echo "Configuring iptables ..." - mkdir -p /etc/iptables/ - # Base policies (drop all incoming, allow all outcoming, drop all forwarding) - sudo iptables -P INPUT ACCEPT - sudo iptables -P OUTPUT ACCEPT - sudo iptables -P FORWARD DROP - # Allow conntrack established connections - sudo iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT - # Allow local loopback services - sudo iptables -A INPUT -i lo -j ACCEPT - # Allow ssh - sudo iptables -A INPUT -p tcp --dport 22 -j ACCEPT - # Allow http - sudo iptables -A INPUT -p tcp --dport 8080 -j ACCEPT - # Allow https - sudo iptables -A INPUT -p tcp --dport 443 -j ACCEPT - # Allow dns - sudo iptables -A INPUT -p tcp --dport 53 -j ACCEPT - sudo iptables -A INPUT -p udp --dport 53 -j ACCEPT - # Allow watchdog - sudo iptables -A INPUT -p tcp --dport 3009 -j ACCEPT - # Allow monitor node exporter - sudo iptables -A INPUT -p tcp --dport 9100 -j ACCEPT - # Drop all the rest - sudo iptables -A INPUT -p tcp -j DROP - sudo iptables -A INPUT -p udp -j DROP - # Allow pings - sudo iptables -I INPUT -p icmp --icmp-type destination-unreachable -j ACCEPT - sudo iptables -I INPUT -p icmp --icmp-type source-quench -j ACCEPT - sudo iptables -I INPUT -p icmp --icmp-type time-exceeded -j ACCEPT - sudo bash -c 'iptables-save > /etc/iptables/rules.v4' -} - -configure_flask () { - echo "Configuring flask secret key ..." - if [ -e $FLASK_SECRET_KEY_FILE ]; then - echo "File $FLASK_SECRET_KEY_FILE already exists!" - else - FLASK_SECRET_KEY=$(openssl rand -base64 32) - echo $FLASK_SECRET_KEY >> $FLASK_SECRET_KEY_FILE - fi - export FLASK_SECRET_KEY=$FLASK_SECRET_KEY -} - -configure_filebeat () { - echo "Configuring filebeat ..." - sudo cp $CONFIG_DIR/filebeat.yml $NODE_DATA_DIR/ - sudo chown root $NODE_DATA_DIR/filebeat.yml - sudo chmod go-w $NODE_DATA_DIR/filebeat.yml -} - -up_compose() { - if [[ "$MONITORING_CONTAINERS" == "True" ]]; then - echo "Running SKALE Node with monitoring containers..." - SKALE_DIR="$SKALE_DIR" docker-compose -f docker-compose.yml up -d - else - echo "Running SKALE Node with base set of containers..." - SKALE_DIR="$SKALE_DIR" docker-compose -f docker-compose.yml up -d $BASE_SERVICES - fi - if [[ ! -z "$TG_API_KEY" && ! -z "$TG_CHAT_ID" ]]; then - SKALE_DIR="$SKALE_DIR" docker-compose -f docker-compose.yml up -d $NOTIFICATION_SERVICES - echo "Running containers for telegram notifications..." - fi -} diff --git a/main.spec b/main.spec index 3483c927..6c7e25b2 100644 --- a/main.spec +++ b/main.spec @@ -11,10 +11,7 @@ a = Analysis( pathex=['.'], binaries=[], datas=[ - ("./text.yml", "data"), - ("./datafiles/configure-iptables.sh", "data/datafiles"), - ("./datafiles/backup-install.sh", "data/datafiles"), - ("./datafiles/helper.sh", "data/datafiles") + ("./text.yml", "data") ], hiddenimports=[], hookspath=[], diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 3eb3f75a..7a8dfdb0 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -80,13 +80,6 @@ def _get_env(): PROJECT_DIR = PARDIR TEXT_FILE = os.path.join(PROJECT_DIR, 'text.yml') -DATAFILES_FOLDER = os.path.join(PROJECT_DIR, 'datafiles') - -THIRDPARTY_FOLDER_PATH = os.path.join(DATAFILES_FOLDER, 'third_party') - -CONFIGURE_IPTABLES_SCRIPT = os.path.join(DATAFILES_FOLDER, 'configure-iptables.sh') -BACKUP_INSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'backup-install.sh') -UNINSTALL_SCRIPT = os.path.join(DATAFILES_FOLDER, 'uninstall.sh') REDIS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'redis-data') ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 3be03fde..c8bc202a 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -19,9 +19,7 @@ import logging from pathlib import Path - -# from node_cli.configs import IPTABLES_DIR -IPTABLES_DIR = '/etc/iptables/' +from node_cli.configs import IPTABLES_DIR logger = logging.getLogger(__name__) @@ -46,19 +44,6 @@ '53' # dns ] -LOOPBACK_RULE_D = { - 'protocol': 'tcp', - 'target': 'ACCEPT', - 'in_interface': 'lo' -} - - -ICMP_RULE_D = { - 'protocol': 'icmp', - 'target': 'ACCEPT', - 'icmp_type': 'destination-unreachable' -} - def configure_iptables(): """ @@ -90,6 +75,7 @@ def set_base_policies() -> None: def allow_loopback(chain: iptc.Chain) -> None: + """Allow local loopback services""" logger.debug('Allowing loopback packages...') rule = iptc.Rule() rule.target = iptc.Target(rule, 'ACCEPT') @@ -98,6 +84,7 @@ def allow_loopback(chain: iptc.Chain) -> None: def allow_conntrack(chain: iptc.Chain) -> None: + """Allow conntrack established connections""" logger.debug('Allowing conntrack...') rule = iptc.Rule() rule.protocol = "tcp" @@ -171,19 +158,3 @@ def rule_to_dict(rule): 'out': rule.out_interface, 'target': rule.target.name, } - - -if __name__ == '__main__': - - import sys - from logging import Formatter, StreamHandler - - LOG_FORMAT_FILE = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' # noqa - - formatter = Formatter(LOG_FORMAT_FILE) - stream_handler = StreamHandler(sys.stderr) - stream_handler.setFormatter(formatter) - stream_handler.setLevel(logging.INFO) - logging.basicConfig(level=logging.DEBUG, handlers=[stream_handler]) - - configure_iptables() diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 958f43c4..4317c977 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,7 +28,7 @@ import docker from node_cli.configs import ( - SKALE_DIR, DATAFILES_FOLDER, INIT_ENV_FILEPATH, + SKALE_DIR, INIT_ENV_FILEPATH, BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TM_INIT_TIMEOUT ) from node_cli.configs.cli_logger import LOG_DIRNAME @@ -117,7 +117,6 @@ def restore(backup_path, env_filepath): save_env_params(env_filepath) env = { 'SKALE_DIR': SKALE_DIR, - 'DATAFILES_FOLDER': DATAFILES_FOLDER, 'BACKUP_RUN': 'True', 'HOME_DIR': HOME_DIR, **env_params @@ -142,7 +141,6 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None): env_params = extract_env_params(INIT_ENV_FILEPATH) env = { 'SKALE_DIR': SKALE_DIR, - 'DATAFILES_FOLDER': DATAFILES_FOLDER, **env_params } if inited_node: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 0ef8f4ec..616e12e3 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -25,11 +25,12 @@ from node_cli.operations.common import ( backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, - configure_flask, configure_iptables + configure_flask ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install from node_cli.operations.skale_node import sync_skale_node +from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index e9e14556..5cb42daf 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -30,12 +30,9 @@ from node_cli.configs import ( CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, - FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE, - CONFIGURE_IPTABLES_SCRIPT + FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE ) from node_cli.utils.helper import read_json -from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.helper import error_exit, run_cmd logger = logging.getLogger(__name__) @@ -80,14 +77,3 @@ def configure_flask(): with open(FLASK_SECRET_KEY_FILE, 'w') as f: f.write(flask_secret_key) logger.info('Flask secret key generated and saved') - - -def configure_iptables(): - # todo!!! - try: - run_cmd(['bash', CONFIGURE_IPTABLES_SCRIPT]) - except Exception: - error_msg = 'iptables configure script failed' - logger.exception(error_msg) - error_exit(error_msg, exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - logger.info('iptables configured successfully') From b58ea6fa5431e0f118810e090237c9da966e4bf5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 12 Mar 2021 13:32:06 +0200 Subject: [PATCH 039/140] SKALE-2972 Move restore script --- node_cli/operations/base.py | 14 ++++++++++++-- node_cli/operations/common.py | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 616e12e3..2357b147 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -25,7 +25,7 @@ from node_cli.operations.common import ( backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, - configure_flask + configure_flask, unpack_backup_archive ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install from node_cli.operations.skale_node import sync_skale_node @@ -104,4 +104,14 @@ def turn_on(env): def restore(env, backup_path): - pass + unpack_backup_archive(backup_path) + link_env_file() + configure_iptables() + docker_lvmpy_install(env) + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'] + ) + update_resource_allocation(env_type=env['ENV_TYPE']) + compose_up(env) diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 5cb42daf..050e72bf 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -19,6 +19,7 @@ import os import stat +import tarfile import logging import shutil import secrets @@ -29,7 +30,7 @@ from node_cli.configs import ( CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, + MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, HOME_DIR, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE ) from node_cli.utils.helper import read_json @@ -77,3 +78,10 @@ def configure_flask(): with open(FLASK_SECRET_KEY_FILE, 'w') as f: f.write(flask_secret_key) logger.info('Flask secret key generated and saved') + + +def unpack_backup_archive(backup_path: str) -> None: + logger.info('Unpacking backup archive...') + tar = tarfile.open(backup_path) + tar.extractall(path=HOME_DIR) + tar.close() From 670eae3518f66a84bc9ce5d9f78c66ee2223f476 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Sun, 14 Mar 2021 23:57:05 +0200 Subject: [PATCH 040/140] SKALE-2972 Fix iptc, fix tests --- node_cli/core/iptables.py | 3 ++- tests/cli/node_test.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index c8bc202a..78965053 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -28,7 +28,8 @@ import iptc except (FileNotFoundError, AttributeError): logger.warning('Unable to import iptc') - iptc = None + from collections import namedtuple # hotfix for tests + iptc = namedtuple('iptc', ['Chain', 'Rule']) ALLOWED_INCOMING_TCP_PORTS = [ diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 4c0ec733..7a56270c 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -345,6 +345,8 @@ def test_restore(): backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.restore_op'), \ + mock.patch('node_cli.core.node.restore_mysql_backup'), \ mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock): result = run_command( restore_node, @@ -387,6 +389,7 @@ def test_turn_off_maintenance_on(): {'status': 'ok', 'payload': None} ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ + mock.patch('node_cli.core.node.turn_off_op'), \ mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -397,7 +400,7 @@ def test_turn_off_maintenance_on(): '--yes' ]) assert result.exit_code == 0 - assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\nTuring off the node...\nNode was successfully turned off\n' # noqa + assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa def test_turn_on_maintenance_off(): @@ -407,6 +410,7 @@ def test_turn_on_maintenance_off(): ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ + mock.patch('node_cli.core.node.turn_on_op'), \ mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', From d704b5f22bddca5f2588f0d2c7b89862ca43f5eb Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 15 Mar 2021 09:09:39 +0200 Subject: [PATCH 041/140] SKALE-3649 Improve logging --- datafiles/requirements.yaml | 55 ++++++++++++++++++++++++++++++ node_cli/cli/node.py | 8 ++--- node_cli/configs/__init__.py | 3 +- node_cli/core/checks.py | 15 ++++---- node_cli/core/configs_reader.py | 15 +++++--- node_cli/core/host.py | 6 ++-- node_cli/core/node.py | 8 ++++- node_cli/core/resources.py | 17 +++++---- node_cli/main.py | 3 +- node_cli/operations/base.py | 13 +++++-- node_cli/utils/helper.py | 10 +++--- node_cli/utils/print_formatters.py | 4 +-- tests/core_checks_test.py | 10 +++--- 13 files changed, 119 insertions(+), 48 deletions(-) create mode 100644 datafiles/requirements.yaml diff --git a/datafiles/requirements.yaml b/datafiles/requirements.yaml new file mode 100644 index 00000000..75a54ba7 --- /dev/null +++ b/datafiles/requirements.yaml @@ -0,0 +1,55 @@ +mainnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 1000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +testnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 1000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +qanet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 1000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +devnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 1000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index a61b8650..c2ff5cbd 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -32,7 +32,7 @@ from node_cli.core.host import run_preinstall_checks from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd -from node_cli.utils.print_formatters import print_requirements_check_result +from node_cli.utils.print_formatters import print_failed_requirements_checks TEXTS = safe_load_texts() @@ -248,9 +248,9 @@ def _set_domain_name(domain): help='Network to check' ) def check_requirements(network): - result = run_preinstall_checks(network) - if not result: + failed_checks = run_preinstall_checks(network) + if not failed_checks: print('Requirements checking succesfully finished!') else: print('Node is not fully meet the requirements!') - print_requirements_check_result(result) + print_failed_requirements_checks(failed_checks) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 641d2e40..a2c52e2b 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -35,7 +35,7 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') FILESTORAGE_INFO_FILE = os.path.join(CONTAINER_CONFIG_PATH, 'filestorage_info.json') FILESTORAGE_ARTIFACTS_FILE = os.path.join(NODE_DATA_PATH, 'filestorage_artifacts.json') -CONFIGS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'configs.yml') +NET_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'net_params.yaml') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' @@ -59,7 +59,6 @@ FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) -REQUIREMENTS_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'requirements.yaml') DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json' HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG') diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index c9edb64e..3992c3c5 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -31,7 +31,7 @@ import docker import yaml -from node_cli.configs import DOCKER_CONFIG_FILEPATH, REQUIREMENTS_PATH +from node_cli.configs import DOCKER_CONFIG_FILEPATH, NET_PARAMS_FILEPATH from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) @@ -41,10 +41,9 @@ ListChecks = List[CheckResult] -def get_requirements(network: str = 'mainnet'): - with open(REQUIREMENTS_PATH) as requirements_file: +def get_net_params(network: str = 'mainnet'): + with open(NET_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) - logger.info(ydata) return ydata[network] @@ -72,7 +71,7 @@ def __init__(self, requirements: dict) -> None: def cpu_total(self) -> CheckResult: name = 'cpu_total' actual = psutil.cpu_count(logical=True) - expected = self.requirements['hardware']['cpu_total'] + expected = self.requirements['server']['cpu_total'] info = f'Expected {expected} logical cores, actual {actual} cores' if actual < expected: return self._error(name=name, info=info) @@ -82,7 +81,7 @@ def cpu_total(self) -> CheckResult: def cpu_physical(self) -> CheckResult: name = 'cpu_physical' actual = psutil.cpu_count(logical=False) - expected = self.requirements['hardware']['cpu_physical'] + expected = self.requirements['server']['cpu_physical'] info = f'Expected {expected} physical cores, actual {actual} cores' if actual < expected: return self._error(name=name, info=info) @@ -93,7 +92,7 @@ def memory(self) -> CheckResult: name = 'memory' actual = psutil.virtual_memory().total, actual = actual[0] - expected = self.requirements['hardware']['memory'] + expected = self.requirements['server']['memory'] actual_gb = round(actual / 1024 ** 3, 2) expected_gb = round(expected / 1024 ** 3, 2) info = f'Expected RAM {expected_gb} GB, actual {actual_gb} GB' @@ -105,7 +104,7 @@ def memory(self) -> CheckResult: def swap(self) -> CheckResult: name = 'swap' actual = psutil.swap_memory().total - expected = self.requirements['hardware']['swap'] + expected = self.requirements['server']['swap'] actual_gb = round(actual / 1024 ** 3, 2) expected_gb = round(expected / 1024 ** 3, 2) info = f'Expected swap memory {expected_gb} GB, actual {actual_gb} GB' diff --git a/node_cli/core/configs_reader.py b/node_cli/core/configs_reader.py index 316099e2..ebb9dc4d 100644 --- a/node_cli/core/configs_reader.py +++ b/node_cli/core/configs_reader.py @@ -17,18 +17,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import yaml from node_cli.utils.helper import safe_load_yml -from node_cli.configs import CONFIGS_FILEPATH +from node_cli.configs import NET_PARAMS_FILEPATH -def read_node_configs(): - return safe_load_yml(CONFIGS_FILEPATH) +def read_node_params(): + return safe_load_yml(NET_PARAMS_FILEPATH) def get_config_env_section(env_type: str, section: str): - configs = read_node_configs() + configs = read_node_params() return configs[env_type][section] def get_config_env_schain_option(env_type: str, key: str): return get_config_env_section(env_type, 'schain')[key] + + +def get_net_params(network: str = 'mainnet'): + with open(NET_PARAMS_FILEPATH) as requirements_file: + ydata = yaml.load(requirements_file, Loader=yaml.Loader) + return ydata[network] diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 19537a03..4feecfe4 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -25,7 +25,7 @@ from node_cli.core.resources import update_resource_allocation from node_cli.core.checks import ( - get_requirements, DockerChecker, ListChecks, + get_net_params, DockerChecker, ListChecks, MachineChecker, PackagesChecker ) @@ -80,8 +80,8 @@ def prepare_host(env_filepath, disk_mountpoint, sgx_server_url, env_type, update_resource_allocation(env_type) -def run_preinstall_checks(network: str = 'mainnet') -> ListChecks: - requirements = get_requirements(network) +def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: + requirements = get_net_params(env_type) checkers = [ MachineChecker(requirements), PackagesChecker(requirements), diff --git a/node_cli/core/node.py b/node_cli/core/node.py index d81b7ae2..b0f02e0a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -100,7 +100,13 @@ def init(env_filepath): env = get_node_env(env_filepath) if env is None: return - init_op(env_filepath, env) + init_result = init_op(env_filepath, env) + if not init_result: + error_exit( + 'Init operation failed', + exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR + ) + return logger.info('Waiting for transaction manager initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 235c13be..98000a00 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -25,9 +25,9 @@ from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import ( - write_json, read_json, run_cmd, format_output, extract_env_params, safe_load_yml + write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) -from node_cli.core.configs_reader import get_config_env_schain_option +from node_cli.core.configs_reader import get_net_params from node_cli.configs import ALLOCATION_FILEPATH from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, @@ -188,7 +188,7 @@ def get_disk_alloc(): def get_static_disk_alloc(env_type: str): disk_size = get_disk_size() - env_disk_size = get_config_env_schain_option(env_type, 'disk_size_bytes') + env_disk_size = get_net_params(env_type)['server']['disk'] check_disk_size(disk_size, env_disk_size) free_space = calculate_free_disk_space(env_disk_size) return ResourceAlloc(free_space) @@ -199,16 +199,15 @@ def check_disk_size(disk_size: int, env_disk_size: int): raise Exception(f'Disk size: {disk_size}, required disk size: {env_disk_size}') -def get_disk_size(): +def get_disk_size() -> int: disk_path = get_disk_path() disk_size_cmd = construct_disk_size_cmd(disk_path) - res = run_cmd(disk_size_cmd) - stdout, _ = format_output(res) - return int(stdout) + output = run_cmd(disk_size_cmd).stdout.decode('utf-8') + return int(output) -def construct_disk_size_cmd(disk_path): - return f'sudo blockdev --getsize64 {disk_path}' +def construct_disk_size_cmd(disk_path: str) -> list: + return ['blockdev', '--getsize64', disk_path] def check_is_partition(disk_path): diff --git a/node_cli/main.py b/node_cli/main.py index faf69ec6..6a0abe83 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -110,4 +110,5 @@ def handle_exception(exc_type, exc_value, exc_traceback): print(f'Command execution failed with {err}. Recheck your inputs') traceback.print_exc() logger.exception(f'Command failed with {err}') - logger.debug(f'execution time: {time.time() - start_time} seconds') + finally: + logger.debug(f'execution time: {time.time() - start_time} seconds') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 45975830..2d7d3925 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -18,7 +18,9 @@ # along with this program. If not, see . from node_cli.cli.info import VERSION -from node_cli.core.host import prepare_host, link_env_file +from node_cli.core.host import ( + prepare_host, link_env_file, run_preinstall_checks +) from node_cli.core.resources import update_resource_allocation from node_cli.operations.common import ( @@ -30,6 +32,7 @@ from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta +from node_cli.utils.print_formatters import print_failed_requirements_checks def update(env_filepath: str, env: str) -> None: @@ -57,9 +60,12 @@ def update(env_filepath: str, env: str) -> None: compose_up(env) -def init(env_filepath: str, env: str) -> None: +def init(env_filepath: str, env: str) -> bool: sync_skale_node(env) - # todo: add hardware checks + failed_checks = run_preinstall_checks(env['ENV_TYPE']) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False prepare_host( env_filepath, env['DISK_MOUNTPOINT'], @@ -83,6 +89,7 @@ def init(env_filepath: str, env: str) -> None: ) update_resource_allocation(env_type=env['ENV_TYPE']) compose_up(env) + return True def backup_init(env): diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 0ce7698b..42835c11 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -94,9 +94,8 @@ def run_cmd(cmd, env={}, shell=False, secure=False, check_code=True): return res -def format_output(res): - return res.stdout.decode('UTF-8').rstrip(), \ - res.stderr.decode('UTF-8').rstrip() +def format_output(res: subprocess.CompletedProcess) -> str: + return res.stdout.decode('UTF-8').rstrip() def download_file(url, filepath): @@ -248,9 +247,8 @@ def download_dump(path, container_name=None): def init_default_logger(): f_handler = get_file_handler(LOG_FILEPATH, logging.INFO) debug_f_handler = get_file_handler(DEBUG_LOG_FILEPATH, logging.DEBUG) - stream_handler = get_stream_handler() - logging.basicConfig(level=logging.DEBUG, handlers=[ - f_handler, debug_f_handler, stream_handler]) + logging.basicConfig( + level=logging.DEBUG, handlers=[f_handler, debug_f_handler]) def get_stream_handler(): diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index fb46d578..a1b89101 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -290,9 +290,9 @@ def print_err_response(error_payload): print(f'You can find more info in {DEBUG_LOG_FILEPATH}') -def print_requirements_check_result(result: list) -> None: +def print_failed_requirements_checks(failed_checks: list) -> None: headers = ['Check', 'Info'] - rows = [[r.name, r.info] for r in result] + rows = [[r.name, r.info] for r in failed_checks] table = texttable.Texttable() table.add_rows([headers, *rows]) drawing = table.draw() diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index 2ec3b578..93ff8be2 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -10,7 +10,7 @@ @pytest.fixture def requirements_data(): return { - 'hardware': { + 'server': { 'cpu_total': 1, 'cpu_physical': 1, 'memory': 100, @@ -30,7 +30,7 @@ def test_checks_cpu_total(requirements_data): r = checker.cpu_total() assert r.name == 'cpu_total' assert r.status == 'ok' - requirements_data['hardware']['cpu_total'] = 10000 # too big + requirements_data['server']['cpu_total'] = 10000 # too big checker = MachineChecker(requirements_data) r = checker.cpu_total() assert r.name == 'cpu_total' @@ -43,7 +43,7 @@ def test_checks_cpu_physical(requirements_data): r = checker.cpu_physical() assert r.name == 'cpu_physical' assert r.status == 'ok' - requirements_data['hardware']['cpu_physical'] = 10000 # too big + requirements_data['server']['cpu_physical'] = 10000 # too big checker = MachineChecker(requirements_data) r = checker.cpu_physical() assert r.name == 'cpu_physical' @@ -56,7 +56,7 @@ def test_checks_memory(requirements_data): assert r.name == 'memory' assert r.status == 'ok' # too big - requirements_data['hardware']['memory'] = 10000000000000 + requirements_data['server']['memory'] = 10000000000000 checker = MachineChecker(requirements_data) r = checker.memory() assert r.name == 'memory' @@ -75,7 +75,7 @@ def test_checks_swap(requirements_data): assert r.name == 'swap' assert r.status == 'ok' # too big - requirements_data['hardware']['swap'] = 10000000000000 + requirements_data['server']['swap'] = 10000000000000 checker = MachineChecker(requirements_data) r = checker.swap() assert r.name == 'swap' From 35be820bf1f3ebe2569dd661baff2d949d72dc32 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 15 Mar 2021 10:57:46 +0200 Subject: [PATCH 042/140] SKALE-3649 Fix resource test --- node_cli/core/checks.py | 1 + node_cli/core/resources.py | 3 +- scripts/run_tests.sh | 2 +- tests/resources_test.py | 92 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 3992c3c5..b1b0609f 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -44,6 +44,7 @@ def get_net_params(network: str = 'mainnet'): with open(NET_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) + print('IVD', ydata) return ydata[network] diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 98000a00..e6503b16 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -56,7 +56,7 @@ def dict(self): return self.values -class SChainVolumeAlloc(): +class SChainVolumeAlloc: def __init__(self, disk_alloc: ResourceAlloc, proportions: dict): self.volume_alloc = {} disk_alloc_dict = disk_alloc.dict() @@ -191,6 +191,7 @@ def get_static_disk_alloc(env_type: str): env_disk_size = get_net_params(env_type)['server']['disk'] check_disk_size(disk_size, env_disk_size) free_space = calculate_free_disk_space(env_disk_size) + print(free_space, env_disk_size) return ResourceAlloc(free_space) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index a56118d1..ac6fc775 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -6,4 +6,4 @@ export HIDE_STREAM_LOG=true export ENV=dev # IVD ?? export HOME_DIR='tests/' -HOME_DIR='tests/' ENV=dev DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ +HOME_DIR='tests/' ENV=dev DOTENV_FILEPATH='tests/test-env' py.test tests/resources_test.py --ignore=tests/operations/ $@ diff --git a/tests/resources_test.py b/tests/resources_test.py index 73b70b52..afeccd07 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -3,8 +3,9 @@ import mock import pytest +import yaml -from node_cli.configs import ALLOCATION_FILEPATH +from node_cli.configs import ALLOCATION_FILEPATH, NET_PARAMS_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.resources import ( compose_resource_allocation_config, @@ -25,6 +26,91 @@ BIG_DISK_SIZE = NORMAL_DISK_SIZE * 100 +TEST_NET_PARAMS = """ +mainnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 2000000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +testnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +testnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +qanet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +devnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 80000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 +""" + + +@pytest.fixture +def net_params_file(): + with open(NET_PARAMS_FILEPATH, 'w') as f: + yaml.dump( + yaml.load(TEST_NET_PARAMS, Loader=yaml.Loader), + stream=f, + Dumper=yaml.Dumper + ) + yield NET_PARAMS_FILEPATH + os.remove(NET_PARAMS_FILEPATH) + + def disk_alloc_mock(env_type): return ResourceAlloc(128) @@ -90,7 +176,7 @@ def test_update_allocation_config(resource_alloc_config): assert json.load(jfile) != INITIAL_CONFIG -def test_get_static_disk_alloc_devnet(): +def test_get_static_disk_alloc_devnet(net_params_file): with mock.patch('node_cli.core.resources.get_disk_size', return_value=SMALL_DISK_SIZE): with pytest.raises(Exception): get_static_disk_alloc(DEFAULT_ENV_TYPE) @@ -111,7 +197,7 @@ def test_get_static_disk_alloc_devnet(): } -def test_get_static_disk_alloc_mainnet(): +def test_get_static_disk_alloc_mainnet(net_params_file): env_type = 'mainnet' with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): with pytest.raises(Exception): From 7d996784b0feb60715efb7335b9d5931b6568cb8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Mar 2021 11:42:35 +0200 Subject: [PATCH 043/140] SKALE-2972 Fix turn on test --- tests/cli/node_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 7a56270c..09944e69 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -411,6 +411,7 @@ def test_turn_on_maintenance_off(): with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ mock.patch('node_cli.core.node.turn_on_op'), \ + mock.patch('node_cli.core.node.is_base_containers_alive'), \ mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -424,7 +425,10 @@ def test_turn_on_maintenance_off(): ]) assert result.exit_code == 0 - assert result.output == 'Turning on the node...\nWaiting for transaction manager initialization ...\nNode was successfully turned on\nSetting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa + + print('result.output result.output') + print(result.output) + assert result.output == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa, tmp fix def test_set_domain_name(): From ba6a7e39fe9a385c74a96675bfa9b564451da972 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Mar 2021 11:52:06 +0200 Subject: [PATCH 044/140] SKALE-2972 Remove print --- tests/cli/node_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 09944e69..5fd1abef 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -425,9 +425,6 @@ def test_turn_on_maintenance_off(): ]) assert result.exit_code == 0 - - print('result.output result.output') - print(result.output) assert result.output == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa, tmp fix From 3387e3c45cec5f55ccb1a0bdb0bad337a9c6b3db Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 15 Mar 2021 18:09:18 +0200 Subject: [PATCH 045/140] SKALE-3649 Packages checking. Cli input validation --- node_cli/cli/node.py | 4 ++-- node_cli/core/checks.py | 48 +++++++++++++++++++++++++++++++---------- scripts/run_tests.sh | 3 +-- setup.py | 4 +++- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index c2ff5cbd..f83f61f9 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -31,6 +31,7 @@ ) from node_cli.core.host import run_preinstall_checks from node_cli.configs import DEFAULT_NODE_BASE_PORT +from node_cli.configs.env import ALLOWED_ENV_TYPES from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd from node_cli.utils.print_formatters import print_failed_requirements_checks @@ -243,8 +244,7 @@ def _set_domain_name(domain): @node.command(help='Check if node meet network requirements') @click.option( '--network', '-n', - type=str, - default='mainnet', + type=click.Choice(ALLOWED_ENV_TYPES), help='Network to check' ) def check_requirements(network): diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index b1b0609f..e0f8324c 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -30,6 +30,8 @@ import docker import yaml +from debian import debian_support +from packaging.version import parse as version_parse from node_cli.configs import DOCKER_CONFIG_FILEPATH, NET_PARAMS_FILEPATH from node_cli.utils.helper import run_cmd @@ -44,7 +46,6 @@ def get_net_params(network: str = 'mainnet'): with open(NET_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) - print('IVD', ydata) return ydata[network] @@ -141,11 +142,16 @@ def docker(self) -> CheckResult: return self._error(name=name, info=info) v_cmd_result = run_cmd(['docker', '-v'], check_code=False) - output = v_cmd_result.stdout.decode('utf-8') + actual_version = v_cmd_result.stdout.decode('utf-8').strip() + expected_version = self.requirements['packages']['docker'] + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } if v_cmd_result.returncode == 0: - return self._ok(name=name, info=output) + return self._ok(name=name, info=info) else: - return self._error(name=name, info=output) + return self._error(name=name, info=info) def docker_compose(self) -> CheckResult: name = 'docker-compose' @@ -155,7 +161,7 @@ def docker_compose(self) -> CheckResult: return self._error(name=name, info=info) v_cmd_result = run_cmd(['docker-compose', '-v'], check_code=False) - output = v_cmd_result.stdout.decode('utf-8') + output = v_cmd_result.stdout.decode('utf-8').rstrip() if v_cmd_result.returncode != 0: output = v_cmd_result.stdout.decode('utf-8') info = f'Checking docker-compose version failed with: {output}' @@ -163,12 +169,13 @@ def docker_compose(self) -> CheckResult: actual_version = output.split(',')[0].split()[-1].strip() expected_version = self.requirements['packages']['docker-compose'] + info = { 'expected_version': expected_version, 'actual_version': actual_version } info = f'Expected docker-compose version {expected_version}, actual {actual_version}' # noqa - if actual_version < expected_version: + if version_parse(actual_version) < version_parse(expected_version): return self._error(name=name, info=info) else: return self._ok(name=name, info=info) @@ -188,17 +195,36 @@ def lsof(self) -> CheckResult: def psmisc(self) -> CheckResult: return self._check_apt_package('psmisc') + def _version_from_dpkg_output(self, output: str) -> str: + v_line = next(filter( + lambda s: s.startswith('Version'), + output.split('\n') + )) + return v_line.split()[1] + def _check_apt_package(self, package_name: str, version: str = None) -> CheckResult: # TODO: check versions dpkg_cmd_result = run_cmd( - ['dpkg', '-s', 'lvm2'], check_code=False) - output = dpkg_cmd_result.stdout.decode('utf-8') - if dpkg_cmd_result.returncode == 0: - return self._ok(name=package_name, info=output) - else: + ['dpkg', '-s', package_name], check_code=False) + output = dpkg_cmd_result.stdout.decode('utf-8').strip() + if dpkg_cmd_result.returncode != 0: return self._error(name=package_name, info=output) + actual_version = self._version_from_dpkg_output(output) + expected_version = self.requirements['packages'][package_name] + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } + compare_result = debian_support.version_compare( + actual_version, actual_version + ) + if compare_result == -1: + return self._error(name=package_name, info=info) + else: + return self._ok(name=package_name, info=info) + class DockerChecker(BaseChecker): def __init__(self) -> None: diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index ac6fc775..193930c0 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,7 +3,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) export HIDE_STREAM_LOG=true -export ENV=dev # IVD ?? export HOME_DIR='tests/' -HOME_DIR='tests/' ENV=dev DOTENV_FILEPATH='tests/test-env' py.test tests/resources_test.py --ignore=tests/operations/ $@ +HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index fa9dcb61..d2ce3d74 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,9 @@ def find_version(*file_paths): "terminaltables==3.1.0", "requests==2.23.0", "GitPython==3.1.12", - "PyYAML==5.4.1" + "PyYAML==5.4.1", + "packaging==20.9", + "python-debian==0.1.39" ], python_requires='>=3.6,<4', extras_require=extras_require, From 03556dd96f68284731fff985cfde8eab3931c101 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 15 Mar 2021 19:05:35 +0200 Subject: [PATCH 046/140] SKALE-3649 Fix tests --- node_cli/core/checks.py | 5 ++- scripts/run_tests.sh | 2 +- tests/conftest.py | 93 +++++++++++++++++++++++++++++++++++++++ tests/core_checks_test.py | 55 +++++++++++------------ tests/resources_test.py | 88 +----------------------------------- 5 files changed, 123 insertions(+), 120 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index e0f8324c..5f8d718b 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -196,9 +196,10 @@ def psmisc(self) -> CheckResult: return self._check_apt_package('psmisc') def _version_from_dpkg_output(self, output: str) -> str: + info_lines = map(lambda s: s.strip(), output.split('\n')) v_line = next(filter( lambda s: s.startswith('Version'), - output.split('\n') + info_lines )) return v_line.split()[1] @@ -218,7 +219,7 @@ def _check_apt_package(self, package_name: str, 'actual_version': actual_version } compare_result = debian_support.version_compare( - actual_version, actual_version + actual_version, expected_version ) if compare_result == -1: return self._error(name=package_name, info=info) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 193930c0..2efba50f 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -5,4 +5,4 @@ PROJECT_DIR=$(dirname $DIR) export HIDE_STREAM_LOG=true export HOME_DIR='tests/' -HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test tests/ --ignore=tests/operations/ $@ +HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/tests/conftest.py b/tests/conftest.py index 8d419d67..73a65e4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,3 +16,96 @@ # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . + +import os + + +import pytest +import yaml + +from node_cli.configs import NET_PARAMS_FILEPATH + + +TEST_NET_PARAMS = """ +mainnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 2000000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +testnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +testnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +qanet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 200000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 + +devnet: + server: + cpu_total: 4 + cpu_physical: 4 + memory: 32 + swap: 16 + disk: 80000000000 + + packages: + docker: 1.1.3 + docker-compose: 1.1.3 + iptables-persistant: 1.1.3 + lvm2: 1.1.1 +""" + + +@pytest.fixture +def net_params_file(): + with open(NET_PARAMS_FILEPATH, 'w') as f: + yaml.dump( + yaml.load(TEST_NET_PARAMS, Loader=yaml.Loader), + stream=f, + Dumper=yaml.Dumper + ) + yield NET_PARAMS_FILEPATH + os.remove(NET_PARAMS_FILEPATH) diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index 93ff8be2..2871c46d 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -18,9 +18,10 @@ def requirements_data(): }, 'packages': { 'docker-compose': '1.27.4', - 'docker': None, - 'iptables_persistant': None, - 'lvm2': None + 'docker': '0.0.0', + 'iptables_persistant': '0.0.0', + 'lvm2': '0.0.0', + 'test-package': '2.2.2' } } @@ -109,44 +110,38 @@ def run_cmd_mock(*args, **kwargs): r.status == 'error' -def test_checks_iptables_persistent(requirements_data): +def test_checks_apt_package(requirements_data): checker = PackagesChecker(requirements_data) res_mock = mock.Mock() - res_mock.stdout = b'Test output' + res_mock.stdout = b"""Package: test-package + Version: 5.2.1-2 + """ def run_cmd_mock(*args, **kwargs): return res_mock res_mock.returncode = 0 + apt_package_name = 'test-package' with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.iptables_persistent() - r.name == 'iptables_persistent' - r.status == 'ok' - res_mock.returncode = 1 - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.iptables_persistent() - r.name == 'iptables_persistent' - r.status == 'ok' - - -def test_checks_lvm2(requirements_data): - checker = PackagesChecker(requirements_data) - res_mock = mock.Mock() - res_mock.stdout = b'Test output' + r = checker._check_apt_package(apt_package_name) + assert r.name == apt_package_name + assert r.status == 'ok' - def run_cmd_mock(*args, **kwargs): - return res_mock - - res_mock.returncode = 0 + res_mock.stdout = b"""Package: test-package + Version: 1.1.1 + """ with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.lvm2() - r.name == 'lvm2' - r.status == 'ok' - res_mock.returncode = 1 + r = checker._check_apt_package(apt_package_name) + assert r.name == 'test-package' + assert r.status == 'error' + + res_mock.stdout = b"""Package: test-package + Version: 2.2.2 + """ with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.lvm2() - r.name == 'lvm2' - r.status == 'ok' + r = checker._check_apt_package(apt_package_name) + assert r.name == 'test-package' + assert r.status == 'ok' @pytest.fixture diff --git a/tests/resources_test.py b/tests/resources_test.py index afeccd07..e84fd1c8 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -3,9 +3,8 @@ import mock import pytest -import yaml -from node_cli.configs import ALLOCATION_FILEPATH, NET_PARAMS_FILEPATH +from node_cli.configs import ALLOCATION_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.resources import ( compose_resource_allocation_config, @@ -26,91 +25,6 @@ BIG_DISK_SIZE = NORMAL_DISK_SIZE * 100 -TEST_NET_PARAMS = """ -mainnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 2000000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -testnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 200000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -testnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 200000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -qanet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 200000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -devnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 80000000000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 -""" - - -@pytest.fixture -def net_params_file(): - with open(NET_PARAMS_FILEPATH, 'w') as f: - yaml.dump( - yaml.load(TEST_NET_PARAMS, Loader=yaml.Loader), - stream=f, - Dumper=yaml.Dumper - ) - yield NET_PARAMS_FILEPATH - os.remove(NET_PARAMS_FILEPATH) - - def disk_alloc_mock(env_type): return ResourceAlloc(128) From 28679e6a773709078e3f977c57774765ed208458 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 15 Mar 2021 19:07:08 +0200 Subject: [PATCH 047/140] SKALE-3649 Removed redundant file --- datafiles/requirements.yaml | 55 ------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 datafiles/requirements.yaml diff --git a/datafiles/requirements.yaml b/datafiles/requirements.yaml deleted file mode 100644 index 75a54ba7..00000000 --- a/datafiles/requirements.yaml +++ /dev/null @@ -1,55 +0,0 @@ -mainnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 1000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -testnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 1000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -qanet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 1000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 - -devnet: - server: - cpu_total: 4 - cpu_physical: 4 - memory: 32 - swap: 16 - disk: 1000 - - packages: - docker: 1.1.3 - docker-compose: 1.1.3 - iptables-persistant: 1.1.3 - lvm2: 1.1.1 From 7d92b892a3c1e6735513ee4a51a8b75b2f95e237 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Mar 2021 19:43:49 +0200 Subject: [PATCH 048/140] SKALE-3978 Add user check, modify inited validations --- .gitignore | 3 +- README.md | 11 +++- node_cli/configs/__init__.py | 4 ++ node_cli/core/node.py | 61 +++++++++--------- node_cli/utils/exit_codes.py | 4 ++ node_cli/utils/helper.py | 11 ---- node_cli/utils/validations.py | 99 +++++++++++++++++++++++++++++ tests/etc/skale/.keep | 0 tests/utils/__init__.py | 0 tests/utils/validations_test.py | 107 ++++++++++++++++++++++++++++++++ 10 files changed, 254 insertions(+), 46 deletions(-) create mode 100644 node_cli/utils/validations.py create mode 100644 tests/etc/skale/.keep create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/validations_test.py diff --git a/.gitignore b/.gitignore index 1b69f910..707381fd 100644 --- a/.gitignore +++ b/.gitignore @@ -115,4 +115,5 @@ meta.json disk_mountpoint.txt sgx_server_url.txt -resource_allocation.json \ No newline at end of file +resource_allocation.json +conf.json \ No newline at end of file diff --git a/README.md b/README.md index 3773fdde..0207ba55 100644 --- a/README.md +++ b/README.md @@ -497,8 +497,15 @@ Exit codes conventions for SKALE CLI tools - `0` - Everything is OK - `1` - General error exit code -- `3` - Bad API response -- `4` - Script execution error +- `3` - Bad API response** +- `4` - Script execution error** +- `5` - Transaction error* +- `6` - Revert error* +- `7` - Bad user error** +- `8` - Node state error** + +`*` - `validator-cli` only +`**` - `node-cli` only ## Development diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8e42e4d9..b7fe89a9 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -119,3 +119,7 @@ def _get_env(): SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' + +GLOBAL_SKALE_DIR = '/etc/skale' +GLOBAL_SKALE_CONF_FILENAME = 'conf.json' +GLOBAL_SKALE_CONF_FILEPATH = os.path.join(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILENAME) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index d81b7ae2..39c3e1c6 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -37,13 +37,14 @@ from node_cli.operations import update_op, init_op from node_cli.core.resources import update_resource_allocation from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup -from node_cli.core.host import ( - is_node_inited, save_env_params, get_flask_secret_key) +from node_cli.core.host import save_env_params, get_flask_secret_key from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info from node_cli.utils.helper import error_exit, get_request, post_request from node_cli.utils.helper import run_cmd, extract_env_params from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.validations import check_not_inited, check_inited, check_user + logger = logging.getLogger(__name__) TEXTS = Texts() @@ -62,6 +63,24 @@ class NodeStatuses(Enum): NOT_CREATED = 5 +@check_not_inited +def init(env_filepath): + env = get_node_env(env_filepath) + if env is None: + return + init_op(env_filepath, env) + logger.info('Waiting for transaction manager initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(): + error_exit('Containers are not running', exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) + return + logger.info('Generating resource allocation file ...') + update_resource_allocation(env['ENV_TYPE']) + logger.info('Init procedure finished') + + +@check_inited +@check_user def register_node(name, p2p_ip, public_ip, port, domain_name, gas_limit=None, @@ -93,24 +112,7 @@ def register_node(name, p2p_ip, error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def init(env_filepath): - if is_node_inited(): - print(TEXTS['node']['already_inited']) - return - env = get_node_env(env_filepath) - if env is None: - return - init_op(env_filepath, env) - logger.info('Waiting for transaction manager initialization') - time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(): - error_exit('Containers are not running', exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) - return - logger.info('Generating resource allocation file ...') - update_resource_allocation(env['ENV_TYPE']) - logger.info('Init procedure finished') - - +@check_not_inited def restore(backup_path, env_filepath): env_params = extract_env_params(env_filepath) if env_params is None: @@ -172,10 +174,9 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None): return {k: v for k, v in env.items() if v != ''} +@check_inited +@check_user def update(env_filepath): - if not is_node_inited(): - print(TEXTS['node']['not_inited']) - return logger.info('Node update started') env = get_node_env(env_filepath, inited_node=True, sync_schains=False) update_op(env_filepath, env) @@ -295,19 +296,17 @@ def run_turn_on_script(sync_schains, env_filepath): print('Node was successfully turned on') +@check_inited +@check_user def turn_off(maintenance_on): - if not is_node_inited(): - print(TEXTS['node']['not_inited']) - return if maintenance_on: set_maintenance_mode_on() run_turn_off_script() +@check_inited +@check_user def turn_on(maintenance_off, sync_schains, env_file): - if not is_node_inited(): - print(TEXTS['node']['not_inited']) - return run_turn_on_script(sync_schains, env_file) if maintenance_off: set_maintenance_mode_off() @@ -344,10 +343,8 @@ def get_node_status(status): return TEXTS['node']['status'][node_status] +@check_inited def set_domain_name(domain_name): - if not is_node_inited(): - print(TEXTS['node']['not_inited']) - return print(f'Setting new domain name: {domain_name}') status, payload = post_request( blueprint=BLUEPRINT_NAME, diff --git a/node_cli/utils/exit_codes.py b/node_cli/utils/exit_codes.py index 959f3fd6..1ed02a3b 100644 --- a/node_cli/utils/exit_codes.py +++ b/node_cli/utils/exit_codes.py @@ -26,3 +26,7 @@ class CLIExitCodes(IntEnum): FAILURE = 1 BAD_API_RESPONSE = 3 SCRIPT_EXECUTION_ERROR = 4 + TRANSACTION_ERROR = 5 + REVERT_ERROR = 6 + BAD_USER_ERROR = 7 + NODE_STATE_ERROR = 8 diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 2613dab9..d619d4c9 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -27,7 +27,6 @@ import subprocess import urllib.request from subprocess import PIPE -from functools import wraps import logging from logging import Formatter, StreamHandler @@ -42,7 +41,6 @@ from node_cli.utils.print_formatters import print_err_response from node_cli.utils.exit_codes import CLIExitCodes - from node_cli.configs.env import ( absent_params as absent_env_params, get_params as get_env_params @@ -158,15 +156,6 @@ def safe_get_config(config, key): return None -def no_node(f): - @wraps(f) - def inner(*args, **kwargs): - # todo: check that node is not installed yet! - return f(*args, **kwargs) - - return inner - - def safe_load_texts(): with open(TEXT_FILE, 'r') as stream: try: diff --git a/node_cli/utils/validations.py b/node_cli/utils/validations.py new file mode 100644 index 00000000..5c06a9aa --- /dev/null +++ b/node_cli/utils/validations.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +from functools import wraps + +from node_cli.core.host import safe_mk_dirs, is_node_inited +from node_cli.utils.helper import write_json, read_json, error_exit +from node_cli.utils.texts import Texts +from node_cli.configs import GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR + +from node_cli.utils.exit_codes import CLIExitCodes + +TEXTS = Texts() + + +def read_g_config() -> dict: + """Read global SKALE config file, init if not exists""" + try: + return read_json(GLOBAL_SKALE_CONF_FILEPATH) + except FileNotFoundError: + return generate_g_config_file() + + +def generate_g_config_file() -> dict: + """Init global SKALE config file""" + safe_mk_dirs(GLOBAL_SKALE_DIR) + g_config = { + 'user': safe_get_user(), + 'home_dir': os.path.expanduser('~') + } + write_json(GLOBAL_SKALE_CONF_FILEPATH, g_config) + return g_config + + +def safe_get_user() -> str: + if os.environ.get('SUDO_USER'): + return os.environ['SUDO_USER'] + else: + return os.environ['USER'] + + +def is_user_valid(allow_root=True): + current_user = safe_get_user() + if current_user == 'root' and allow_root: + return True + g_conf_user = get_g_conf_user() + return current_user == g_conf_user + + +def get_g_conf_user(): + return read_g_config()['user'] + + +def check_not_inited(f): + @wraps(f) + def inner(*args, **kwargs): + if is_node_inited(): + error_exit(TEXTS['node']['already_inited'], exit_code=CLIExitCodes.NODE_STATE_ERROR) + return f(*args, **kwargs) + return inner + + +def check_inited(f): + @wraps(f) + def inner(*args, **kwargs): + if not is_node_inited(): + error_exit(TEXTS['node']['not_inited'], exit_code=CLIExitCodes.NODE_STATE_ERROR) + return f(*args, **kwargs) + return inner + + +def check_user(f): + @wraps(f) + def inner(*args, **kwargs): + if not is_user_valid(): + g_conf_user = get_g_conf_user() + current_user = safe_get_user() + error_msg = f'You couldn\'t execute this command from user {current_user}. \ +Allowed: {g_conf_user} or root.' + error_exit(error_msg, exit_code=CLIExitCodes.BAD_USER_ERROR) + return f(*args, **kwargs) + return inner diff --git a/tests/etc/skale/.keep b/tests/etc/skale/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/utils/validations_test.py b/tests/utils/validations_test.py new file mode 100644 index 00000000..b1bf177e --- /dev/null +++ b/tests/utils/validations_test.py @@ -0,0 +1,107 @@ +import os +import mock +import pytest + +from node_cli.utils.validations import ( + read_g_config, generate_g_config_file, safe_get_user, is_user_valid, get_g_conf_user, + check_not_inited, check_inited, check_user +) +from node_cli.utils.helper import write_json + + +TEST_GLOBAL_SKALE_DIR = os.path.join(os.environ.get('HOME_DIR'), 'etc', 'skale') +TEST_G_CONF_FP = os.path.join(TEST_GLOBAL_SKALE_DIR, 'conf.json') + + +def test_read_g_config(): + write_json(TEST_G_CONF_FP, {'test': 1}) + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ + mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): + g_config = read_g_config() + assert g_config['test'] == 1 + + +def test_generate_g_config_file(): + try: + os.remove(TEST_G_CONF_FP) + except OSError: + pass + + assert not os.path.exists(TEST_G_CONF_FP) + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ + mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): + generate_g_config_file() + assert os.path.exists(TEST_G_CONF_FP) + + g_config = read_g_config() + assert g_config['user'] == safe_get_user() + assert g_config['home_dir'] == os.path.expanduser('~') + + +def test_safe_get_user(): + sudo_user = os.environ.get('SUDO_USER') + if sudo_user: + del os.environ['SUDO_USER'] + os.environ['USER'] = 'test' + assert safe_get_user() == 'test' + if sudo_user: + os.environ['SUDO_USER'] = sudo_user + + +def test_is_user_valid(): + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): + generate_g_config_file() + assert is_user_valid() + + write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + assert not is_user_valid() + + sudo_user = os.environ.get('SUDO_USER') + os.environ['SUDO_USER'] = 'root' + assert is_user_valid() + assert not is_user_valid(allow_root=False) + + if sudo_user: + os.environ['SUDO_USER'] = sudo_user + else: + del os.environ['SUDO_USER'] + + +def test_get_g_conf_user(): + write_json(TEST_G_CONF_FP, {'user': 'test_get_g_conf_user'}) + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): + assert get_g_conf_user() == 'test_get_g_conf_user' + + +def test_check_not_inited(): + @check_not_inited + def requires_not_inited_node(): + pass + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + requires_not_inited_node() + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with pytest.raises(SystemExit): + requires_not_inited_node() + + +def test_check_inited(): + @check_inited + def requires_inited_node(): + pass + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + requires_inited_node() + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + with pytest.raises(SystemExit): + requires_inited_node() + + +def test_check_user(): + @check_user + def this_checks_user(): + pass + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): + generate_g_config_file() + this_checks_user() + write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + with pytest.raises(SystemExit): + this_checks_user() From 0002bbc91ad580aa8d409e92e7236bbd2c915f78 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Mar 2021 11:40:54 +0200 Subject: [PATCH 049/140] SKALE-3978 Fix node tests --- tests/cli/node_test.py | 58 ++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 4c0ec733..5e6eae48 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -50,11 +50,12 @@ def test_register_node(): requests.codes.ok, {'status': 'ok', 'payload': None} ) - result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - register_node, - ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080', '-d', 'skale.test']) + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + register_node, + ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080', '-d', 'skale.test']) assert result.exit_code == 0 assert result.output == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa @@ -64,11 +65,12 @@ def test_register_node_with_error(): requests.codes.ok, {'status': 'error', 'payload': ['Strange error']}, ) - result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - register_node, - ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test']) + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + register_node, + ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test']) assert result.exit_code == 3 assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa @@ -78,11 +80,12 @@ def test_register_node_with_prompted_ip(): requests.codes.ok, {'status': 'ok', 'payload': None} ) - result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - register_node, - ['--name', 'test-node', '--port', '8080', '-d', 'skale.test'], input='0.0.0.0\n') + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + register_node, + ['--name', 'test-node', '--port', '8080', '-d', 'skale.test'], input='0.0.0.0\n') assert result.exit_code == 0 assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa @@ -92,11 +95,12 @@ def test_register_node_with_default_port(): requests.codes.ok, {'status': 'ok', 'payload': None} ) - result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - register_node, - ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + register_node, + ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') assert result.exit_code == 0 assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa @@ -111,7 +115,7 @@ def test_init_node(caplog): # todo: write new init node test mock.patch('node_cli.core.node.init_op'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=False): + mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, @@ -154,15 +158,15 @@ def test_update_node_without_init(): mock.patch('node_cli.core.host.init_data_dir'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=False): + mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, update_node, params, input='/dev/sdp') - assert result.exit_code == 0 - assert result.output == "Node hasn't been inited before.\nYou should run < skale node init >\n" # noqa + assert result.exit_code == 8 + assert result.output == "Command failed with following errors:\n--------------------------------------------------\nNode hasn't been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n" # noqa def test_node_info_node_info(): @@ -387,7 +391,7 @@ def test_turn_off_maintenance_on(): {'status': 'ok', 'payload': None} ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=True): + mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -407,7 +411,7 @@ def test_turn_on_maintenance_off(): ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=True): + mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -429,7 +433,7 @@ def test_set_domain_name(): {'status': 'ok', 'payload': None} ) - with mock.patch('node_cli.core.node.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, From 7424bce4fdb0ca344587b5555fd39fa2630b2f2b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Mar 2021 15:36:45 +0200 Subject: [PATCH 050/140] SKALE-3978 Fix restore test --- tests/cli/node_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 5e6eae48..6457c487 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -349,7 +349,8 @@ def test_restore(): backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock): + mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock), \ + mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] From f30b40abd94851e1955f59b68cb6647c8fd52520 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Mar 2021 15:52:55 +0200 Subject: [PATCH 051/140] SKALE-3978 Update global config tests --- cli/info.py | 5 --- tests/cli/node_test.py | 16 ++++----- tests/conftest.py | 14 ++++++++ tests/utils/validations_test.py | 59 ++++++++++++++------------------- 4 files changed, 47 insertions(+), 47 deletions(-) delete mode 100644 cli/info.py diff --git a/cli/info.py b/cli/info.py deleted file mode 100644 index 9e37e0e1..00000000 --- a/cli/info.py +++ /dev/null @@ -1,5 +0,0 @@ -BUILD_DATETIME = '2021-03-02 11:39:19' -COMMIT = '012a875cd13faf8ee9e5ce6f9fdc84fc2eb694cf' -BRANCH = 'test-branch' -OS = 'Darwin-x86_64' -VERSION = '1.0.0' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 6457c487..473929fc 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -45,7 +45,7 @@ def disk_alloc_mock(env_type): return ResourceAlloc(128) -def test_register_node(): +def test_register_node(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -60,7 +60,7 @@ def test_register_node(): assert result.output == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_register_node_with_error(): +def test_register_node_with_error(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'error', 'payload': ['Strange error']}, @@ -75,7 +75,7 @@ def test_register_node_with_error(): assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa -def test_register_node_with_prompted_ip(): +def test_register_node_with_prompted_ip(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -90,7 +90,7 @@ def test_register_node_with_prompted_ip(): assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_register_node_with_default_port(): +def test_register_node_with_default_port(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -125,7 +125,7 @@ def test_init_node(caplog): # todo: write new init node test assert result.exit_code == 0 -def test_update_node(): +def test_update_node(mocked_g_config): os.makedirs(NODE_DATA_PATH, exist_ok=True) params = ['./tests/test-env', '--yes'] resp_mock = response_mock(requests.codes.created) @@ -340,7 +340,7 @@ def test_backup(): assert 'Backup archive successfully created: /tmp/skale-node-backup-' in result.output -def test_restore(): +def test_restore(mocked_g_config): Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) result = run_command( backup_node, @@ -373,7 +373,7 @@ def test_maintenance_on(): assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa -def test_maintenance_off(): +def test_maintenance_off(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -386,7 +386,7 @@ def test_maintenance_off(): assert result.output == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa -def test_turn_off_maintenance_on(): +def test_turn_off_maintenance_on(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} diff --git a/tests/conftest.py b/tests/conftest.py index 8d419d67..3a22d835 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,3 +16,17 @@ # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . + +import os +import mock +import pytest + +TEST_GLOBAL_SKALE_DIR = os.path.join(os.environ.get('HOME_DIR'), 'etc', 'skale') +TEST_G_CONF_FP = os.path.join(TEST_GLOBAL_SKALE_DIR, 'conf.json') + + +@pytest.fixture() +def mocked_g_config(): + with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ + mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): + yield diff --git a/tests/utils/validations_test.py b/tests/utils/validations_test.py index b1bf177e..f906f182 100644 --- a/tests/utils/validations_test.py +++ b/tests/utils/validations_test.py @@ -8,29 +8,23 @@ ) from node_cli.utils.helper import write_json +from tests.conftest import TEST_G_CONF_FP -TEST_GLOBAL_SKALE_DIR = os.path.join(os.environ.get('HOME_DIR'), 'etc', 'skale') -TEST_G_CONF_FP = os.path.join(TEST_GLOBAL_SKALE_DIR, 'conf.json') - -def test_read_g_config(): +def test_read_g_config(mocked_g_config): write_json(TEST_G_CONF_FP, {'test': 1}) - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ - mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): - g_config = read_g_config() + g_config = read_g_config() assert g_config['test'] == 1 -def test_generate_g_config_file(): +def test_generate_g_config_file(mocked_g_config): try: os.remove(TEST_G_CONF_FP) except OSError: pass assert not os.path.exists(TEST_G_CONF_FP) - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ - mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): - generate_g_config_file() + generate_g_config_file() assert os.path.exists(TEST_G_CONF_FP) g_config = read_g_config() @@ -48,29 +42,27 @@ def test_safe_get_user(): os.environ['SUDO_USER'] = sudo_user -def test_is_user_valid(): - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): - generate_g_config_file() - assert is_user_valid() +def test_is_user_valid(mocked_g_config): + generate_g_config_file() + assert is_user_valid() - write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) - assert not is_user_valid() + write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + assert not is_user_valid() - sudo_user = os.environ.get('SUDO_USER') - os.environ['SUDO_USER'] = 'root' - assert is_user_valid() - assert not is_user_valid(allow_root=False) + sudo_user = os.environ.get('SUDO_USER') + os.environ['SUDO_USER'] = 'root' + assert is_user_valid() + assert not is_user_valid(allow_root=False) - if sudo_user: - os.environ['SUDO_USER'] = sudo_user - else: - del os.environ['SUDO_USER'] + if sudo_user: + os.environ['SUDO_USER'] = sudo_user + else: + del os.environ['SUDO_USER'] -def test_get_g_conf_user(): +def test_get_g_conf_user(mocked_g_config): write_json(TEST_G_CONF_FP, {'user': 'test_get_g_conf_user'}) - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): - assert get_g_conf_user() == 'test_get_g_conf_user' + assert get_g_conf_user() == 'test_get_g_conf_user' def test_check_not_inited(): @@ -95,13 +87,12 @@ def requires_inited_node(): requires_inited_node() -def test_check_user(): +def test_check_user(mocked_g_config): @check_user def this_checks_user(): pass - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP): - generate_g_config_file() + generate_g_config_file() + this_checks_user() + write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + with pytest.raises(SystemExit): this_checks_user() - write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) - with pytest.raises(SystemExit): - this_checks_user() From 3cdb1bdee99d6d0a89074a999a642cb096366c4c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Mar 2021 15:56:39 +0200 Subject: [PATCH 052/140] SKALE-3978 Fix test_turn_on_maintenance_off test --- tests/cli/node_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 473929fc..d53c4a7f 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -405,7 +405,7 @@ def test_turn_off_maintenance_on(mocked_g_config): assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\nTuring off the node...\nNode was successfully turned off\n' # noqa -def test_turn_on_maintenance_off(): +def test_turn_on_maintenance_off(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} From 71a2d58e68b256d4b6bbded526bcae6ace7d52d8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Mar 2021 17:55:33 +0200 Subject: [PATCH 053/140] SKALE-2972 Update backup unpack, use env for restore --- node_cli/core/iptables.py | 2 +- node_cli/core/node.py | 14 +++++--------- node_cli/operations/common.py | 5 ++--- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 78965053..d5f80537 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -53,7 +53,7 @@ def configure_iptables(): """ logger.info('Configuring iptables...') if not iptc: - raise Exception('Unable to import iptc package') + raise ImportError('Unable to import iptc package') Path(IPTABLES_DIR).mkdir(parents=True, exist_ok=True) tb = iptc.Table(iptc.Table.FILTER) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 4317c977..26cd33fb 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -111,23 +111,19 @@ def init(env_filepath): def restore(backup_path, env_filepath): - env_params = extract_env_params(env_filepath) - if env_params is None: + env = get_node_env(env_filepath) + if env is None: return save_env_params(env_filepath) - env = { - 'SKALE_DIR': SKALE_DIR, - 'BACKUP_RUN': 'True', - 'HOME_DIR': HOME_DIR, - **env_params - } + env['SKALE_DIR'] = SKALE_DIR + env['BACKUP_RUN'] = 'True' # should be str restore_op(env, backup_path) time.sleep(RESTORE_SLEEP_TIMEOUT) if not restore_mysql_backup(env_filepath): print('WARNING: MySQL data restoring failed. ' 'Check < skale logs cli > for more information') logger.info('Generating resource allocation file ...') - update_resource_allocation(env_params['ENV_TYPE']) + update_resource_allocation(env['ENV_TYPE']) print('Node is restored from backup') diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 050e72bf..4f264739 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -82,6 +82,5 @@ def configure_flask(): def unpack_backup_archive(backup_path: str) -> None: logger.info('Unpacking backup archive...') - tar = tarfile.open(backup_path) - tar.extractall(path=HOME_DIR) - tar.close() + with tarfile.open(backup_path) as tar: + tar.extractall(path=HOME_DIR) From fc5c6fc6485b164238d98b6d89a2adfe9f2a1144 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Mar 2021 10:25:07 +0200 Subject: [PATCH 054/140] SKALE-3649 Rename net_params.yaml -> envirnment_params.yaml --- node_cli/configs/__init__.py | 2 +- node_cli/core/checks.py | 6 ++++-- node_cli/core/configs_reader.py | 6 +++--- tests/conftest.py | 8 ++++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index eb4abd30..55463cb5 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -35,7 +35,7 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') FILESTORAGE_INFO_FILE = os.path.join(CONTAINER_CONFIG_PATH, 'filestorage_info.json') FILESTORAGE_ARTIFACTS_FILE = os.path.join(NODE_DATA_PATH, 'filestorage_artifacts.json') -NET_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'net_params.yaml') +ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 5f8d718b..2c1112e3 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -33,7 +33,9 @@ from debian import debian_support from packaging.version import parse as version_parse -from node_cli.configs import DOCKER_CONFIG_FILEPATH, NET_PARAMS_FILEPATH +from node_cli.configs import ( + DOCKER_CONFIG_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH +) from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) @@ -44,7 +46,7 @@ def get_net_params(network: str = 'mainnet'): - with open(NET_PARAMS_FILEPATH) as requirements_file: + with open(ENVIRONMENT_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) return ydata[network] diff --git a/node_cli/core/configs_reader.py b/node_cli/core/configs_reader.py index ebb9dc4d..98f61d20 100644 --- a/node_cli/core/configs_reader.py +++ b/node_cli/core/configs_reader.py @@ -19,11 +19,11 @@ import yaml from node_cli.utils.helper import safe_load_yml -from node_cli.configs import NET_PARAMS_FILEPATH +from node_cli.configs import ENVIRONMENT_PARAMS_FILEPATH def read_node_params(): - return safe_load_yml(NET_PARAMS_FILEPATH) + return safe_load_yml(ENVIRONMENT_PARAMS_FILEPATH) def get_config_env_section(env_type: str, section: str): @@ -36,6 +36,6 @@ def get_config_env_schain_option(env_type: str, key: str): def get_net_params(network: str = 'mainnet'): - with open(NET_PARAMS_FILEPATH) as requirements_file: + with open(ENVIRONMENT_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) return ydata[network] diff --git a/tests/conftest.py b/tests/conftest.py index 73a65e4f..499b4779 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ import pytest import yaml -from node_cli.configs import NET_PARAMS_FILEPATH +from node_cli.configs import ENVIRONMENT_PARAMS_FILEPATH TEST_NET_PARAMS = """ @@ -101,11 +101,11 @@ @pytest.fixture def net_params_file(): - with open(NET_PARAMS_FILEPATH, 'w') as f: + with open(ENVIRONMENT_PARAMS_FILEPATH, 'w') as f: yaml.dump( yaml.load(TEST_NET_PARAMS, Loader=yaml.Loader), stream=f, Dumper=yaml.Dumper ) - yield NET_PARAMS_FILEPATH - os.remove(NET_PARAMS_FILEPATH) + yield ENVIRONMENT_PARAMS_FILEPATH + os.remove(ENVIRONMENT_PARAMS_FILEPATH) From 6176ab5689f2e65818f929066f24cdabb7a862c5 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Mar 2021 10:27:40 +0200 Subject: [PATCH 055/140] SKALE-3649 Refactor network contants in checks --- node_cli/core/checks.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 2c1112e3..44d78fc1 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -45,6 +45,11 @@ ListChecks = List[CheckResult] +NETWORK_CHECK_TIMEOUT = 4 +CLOUDFLARE_DNS_HOST = '1.1.1.1' +CLOUDFLARE_DNS_HOST_PORT = 443 + + def get_net_params(network: str = 'mainnet'): with open(ENVIRONMENT_PARAMS_FILEPATH) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) @@ -119,13 +124,10 @@ def swap(self) -> CheckResult: def network(self) -> CheckResult: name = 'network' - timeout = 4 - cloudflare_dns_host = '1.1.1.1' - cloudflare_dns_host_port = 443 try: - socket.setdefaulttimeout(timeout) + socket.setdefaulttimeout(NETWORK_CHECK_TIMEOUT) socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( - (cloudflare_dns_host, cloudflare_dns_host_port)) + (CLOUDFLARE_DNS_HOST, CLOUDFLARE_DNS_HOST_PORT)) return self._ok(name=name) except socket.error as err: info = f'Network checking returned error: {err}' From 181ed88785ed02a8b5c8457c0a29eb189ec0e888 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Mar 2021 11:13:13 +0200 Subject: [PATCH 056/140] SKALE-3649 Rename SCRIPT_EXECUTION_ERROR -> OPERATION_EXECUTION_ERROR --- node_cli/core/node.py | 7 +++++-- node_cli/utils/exit_codes.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 05b9b815..e6b8be9f 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -103,13 +103,16 @@ def init(env_filepath): if not init_result: error_exit( 'Init operation failed', - exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR + exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) return logger.info('Waiting for transaction manager initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): - error_exit('Containers are not running', exit_code=CLIExitCodes.SCRIPT_EXECUTION_ERROR) + error_exit( + 'Containers are not running', + exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR + ) return logger.info('Generating resource allocation file ...') update_resource_allocation(env['ENV_TYPE']) diff --git a/node_cli/utils/exit_codes.py b/node_cli/utils/exit_codes.py index 959f3fd6..3f638405 100644 --- a/node_cli/utils/exit_codes.py +++ b/node_cli/utils/exit_codes.py @@ -25,4 +25,4 @@ class CLIExitCodes(IntEnum): SUCCESS = 0 FAILURE = 1 BAD_API_RESPONSE = 3 - SCRIPT_EXECUTION_ERROR = 4 + OPERATION_EXECUTION_ERROR = 4 From 2821a252b0f509d1ceb22ded6f669c44e9170ce6 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Mar 2021 12:53:10 +0200 Subject: [PATCH 057/140] SKALE-3649 Use docker.py to check docker version --- node_cli/core/checks.py | 63 ++++++++++++++++++++++------------- tests/conftest.py | 7 ++-- tests/core_checks_test.py | 69 ++++++++++++++++++++++++++------------- 3 files changed, 91 insertions(+), 48 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 44d78fc1..b6641d00 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -78,7 +78,7 @@ def __init__(self, requirements: dict) -> None: self.requirements = requirements def cpu_total(self) -> CheckResult: - name = 'cpu_total' + name = 'cpu-total' actual = psutil.cpu_count(logical=True) expected = self.requirements['server']['cpu_total'] info = f'Expected {expected} logical cores, actual {actual} cores' @@ -88,7 +88,7 @@ def cpu_total(self) -> CheckResult: return self._ok(name=name, info=info) def cpu_physical(self) -> CheckResult: - name = 'cpu_physical' + name = 'cpu-physical' actual = psutil.cpu_count(logical=False) expected = self.requirements['server']['cpu_physical'] info = f'Expected {expected} physical cores, actual {actual} cores' @@ -138,25 +138,6 @@ class PackagesChecker(BaseChecker): def __init__(self, requirements: dict) -> None: self.requirements = requirements - def docker(self) -> CheckResult: - name = 'docker package' - cmd = shutil.which('docker') - if cmd is None: - info = 'No such command: "docker"' - return self._error(name=name, info=info) - - v_cmd_result = run_cmd(['docker', '-v'], check_code=False) - actual_version = v_cmd_result.stdout.decode('utf-8').strip() - expected_version = self.requirements['packages']['docker'] - info = { - 'expected_version': expected_version, - 'actual_version': actual_version - } - if v_cmd_result.returncode == 0: - return self._ok(name=name, info=info) - else: - return self._error(name=name, info=info) - def docker_compose(self) -> CheckResult: name = 'docker-compose' cmd = shutil.which('docker-compose') @@ -232,8 +213,44 @@ def _check_apt_package(self, package_name: str, class DockerChecker(BaseChecker): - def __init__(self) -> None: + def __init__(self, requirements: dict) -> None: self.docker_client = docker.from_env() + self.requirements = requirements + + def _check_docker_command(self) -> str: + return shutil.which('docker') + + def docker_engine(self) -> CheckResult: + name = 'docker-engine' + if self._check_docker_command() is None: + return self._error(name=name, info='No such command: "docker"') + + actual_version = self.docker_client.version()['Version'] + expected_version = self.requirements['docker']['docker-engine'] + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } + if version_parse(actual_version) < version_parse(expected_version): + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) + + def docker_api(self) -> CheckResult: + name = 'docker-api' + if self._check_docker_command() is None: + return self._error(name=name, info='No such command: "docker"') + + actual_version = self.docker_client.version()['ApiVersion'] + expected_version = self.requirements['docker']['docker-api'] + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } + if version_parse(actual_version) < version_parse(expected_version): + return self._error(name=name, info=info) + else: + return self._ok(name=name, info=info) def _get_docker_config(self) -> dict: if not os.path.isfile(DOCKER_CONFIG_FILEPATH): @@ -270,7 +287,7 @@ def keeping_containers_alive(self) -> CheckResult: return self._error(name=name, info=info) def docker_service_status(self) -> CheckResult: - name = 'docker service status' + name = 'docker-service-status' try: self.docker_client.containers.list() except Exception as err: diff --git a/tests/conftest.py b/tests/conftest.py index 499b4779..ea40b27f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,10 +92,13 @@ disk: 80000000000 packages: - docker: 1.1.3 - docker-compose: 1.1.3 iptables-persistant: 1.1.3 lvm2: 1.1.1 + docker-compose: 1.1.3 + + docker: + docker-api: 1.1.3 + docker-engine: 1.1.3 """ diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index 2871c46d..0b9dc6e7 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -18,10 +18,13 @@ def requirements_data(): }, 'packages': { 'docker-compose': '1.27.4', - 'docker': '0.0.0', 'iptables_persistant': '0.0.0', 'lvm2': '0.0.0', 'test-package': '2.2.2' + }, + 'docker': { + 'docker-engine': '0.0.0', + 'docker-api': '0.0.0', } } @@ -29,12 +32,12 @@ def requirements_data(): def test_checks_cpu_total(requirements_data): checker = MachineChecker(requirements_data) r = checker.cpu_total() - assert r.name == 'cpu_total' + assert r.name == 'cpu-total' assert r.status == 'ok' requirements_data['server']['cpu_total'] = 10000 # too big checker = MachineChecker(requirements_data) r = checker.cpu_total() - assert r.name == 'cpu_total' + assert r.name == 'cpu-total' assert r.status == 'error' assert checker.cpu_total().status == 'error' @@ -42,12 +45,12 @@ def test_checks_cpu_total(requirements_data): def test_checks_cpu_physical(requirements_data): checker = MachineChecker(requirements_data) r = checker.cpu_physical() - assert r.name == 'cpu_physical' + assert r.name == 'cpu-physical' assert r.status == 'ok' requirements_data['server']['cpu_physical'] = 10000 # too big checker = MachineChecker(requirements_data) r = checker.cpu_physical() - assert r.name == 'cpu_physical' + assert r.name == 'cpu-physical' assert r.status == 'error' @@ -90,24 +93,44 @@ def test_checks_network(requirements_data): assert r.name == 'network' -def test_checks_docker_version(requirements_data): - checker = PackagesChecker(requirements_data) - res_mock = mock.Mock() - res_mock.stdout = b'Test output' +def test_checks_docker_engine(requirements_data): + checker = DockerChecker(requirements_data) - def run_cmd_mock(*args, **kwargs): - return res_mock + r = checker.docker_engine() + assert r.name == 'docker-engine' + assert r.status == 'ok' - res_mock.returncode = 0 - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.docker() - r.name == 'docker' - r.status == 'ok' - res_mock.returncode = 1 - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker.docker() - r.name == 'docker' - r.status == 'error' + with mock.patch('shutil.which', return_value=None): + r = checker.docker_engine() + assert r.name == 'docker-engine' + assert r.status == 'error' + assert r.info == 'No such command: "docker"' + + requirements_data['docker']['docker-engine'] = '111.111.111' + r = checker.docker_engine() + assert r.name == 'docker-engine' + assert r.status == 'error' + assert r.info['expected_version'] == '111.111.111' + + +def test_checks_docker_api(requirements_data): + checker = DockerChecker(requirements_data) + + r = checker.docker_api() + assert r.name == 'docker-api' + assert r.status == 'ok' + + with mock.patch('shutil.which', return_value=None): + r = checker.docker_api() + assert r.name == 'docker-api' + assert r.status == 'error' + assert r.info == 'No such command: "docker"' + + requirements_data['docker']['docker-api'] = '111.111.111' + r = checker.docker_api() + assert r.name == 'docker-api' + assert r.status == 'error' + assert r.info['expected_version'] == '111.111.111' def test_checks_apt_package(requirements_data): @@ -187,7 +210,7 @@ def test_checks_docker_compose_invalid_version( def test_checks_docker_service_status(): - checker = DockerChecker() + checker = DockerChecker(requirements_data) checker.docker_client = mock.Mock() r = checker.docker_service_status() r.name == 'docker-compose' @@ -195,7 +218,7 @@ def test_checks_docker_service_status(): def test_checks_docker_config(): - checker = DockerChecker() + checker = DockerChecker(requirements_data) valid_config = { 'live-restore': True } From a58474398a7727cb008220f3fc93e55a84c4cf28 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Mar 2021 16:12:44 +0200 Subject: [PATCH 058/140] SKALE-3649 Fix build --- main.spec | 7 ++++++- node_cli/core/host.py | 2 +- scripts/build.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/main.spec b/main.spec index 6c7e25b2..e49ce75f 100644 --- a/main.spec +++ b/main.spec @@ -4,12 +4,17 @@ # if distutils.distutils_path.endswith('__init__.py'): # distutils.distutils_path = os.path.dirname(distutils.distutils_path) +import importlib.util + +libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin + + block_cipher = None a = Analysis( ['node_cli/main.py'], pathex=['.'], - binaries=[], + binaries=[(libxtwrapper_path, '.')], datas=[ ("./text.yml", "data") ], diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 4feecfe4..5eec2a19 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -85,7 +85,7 @@ def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: checkers = [ MachineChecker(requirements), PackagesChecker(requirements), - DockerChecker() + DockerChecker(requirements) ] result = [] for checker in checkers: diff --git a/scripts/build.sh b/scripts/build.sh index 89345775..432eb7d2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -40,7 +40,7 @@ echo "VERSION = '$VERSION'" >> $DIST_INFO_FILEPATH EXECUTABLE_NAME=skale-$VERSION-$OS -pyinstaller --onefile main.spec --hidden-import=eth_hash.backends.pysha3 +pyinstaller --onefile main.spec mv $PARENT_DIR/dist/main $PARENT_DIR/dist/$EXECUTABLE_NAME From 89a0af6066002bf9ca7e99754423d0252761459d Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Mar 2021 16:15:55 +0200 Subject: [PATCH 059/140] SKALE-3649 Refactor run checks command --- node_cli/cli/node.py | 12 +++--------- node_cli/core/node.py | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index f83f61f9..b50f6d9e 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -27,13 +27,11 @@ update, backup, set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info, - set_domain_name + set_domain_name, run_checks ) -from node_cli.core.host import run_preinstall_checks from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.configs.env import ALLOWED_ENV_TYPES from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd -from node_cli.utils.print_formatters import print_failed_requirements_checks TEXTS = safe_load_texts() @@ -245,12 +243,8 @@ def _set_domain_name(domain): @click.option( '--network', '-n', type=click.Choice(ALLOWED_ENV_TYPES), + default='mainnet', help='Network to check' ) def check_requirements(network): - failed_checks = run_preinstall_checks(network) - if not failed_checks: - print('Requirements checking succesfully finished!') - else: - print('Node is not fully meet the requirements!') - print_failed_requirements_checks(failed_checks) + run_checks(network) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index e6b8be9f..54969f15 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -37,8 +37,12 @@ from node_cli.core.resources import update_resource_allocation from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import ( - is_node_inited, save_env_params, get_flask_secret_key) -from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info + is_node_inited, save_env_params, + get_flask_secret_key, run_preinstall_checks +) +from node_cli.utils.print_formatters import ( + print_failed_requirements_checks, print_node_cmd_error, print_node_info +) from node_cli.utils.helper import error_exit, get_request, post_request from node_cli.utils.helper import run_cmd, extract_env_params from node_cli.utils.texts import Texts @@ -324,3 +328,16 @@ def set_domain_name(domain_name): print(msg) else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def run_checks(network: str) -> None: + if not is_node_inited(): + print(TEXTS['node']['not_inited']) + return + + failed_checks = run_preinstall_checks(network) + if not failed_checks: + print('Requirements checking succesfully finished!') + else: + print('Node is not fully meet the requirements!') + print_failed_requirements_checks(failed_checks) From 31c37bae9e9040232eb8b8e71d620162ee96222f Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Mar 2021 20:01:55 +0200 Subject: [PATCH 060/140] SKALE-3649 Improve error handling --- node_cli/core/checks.py | 29 +++++++++++++++++++---------- node_cli/core/host.py | 3 +++ node_cli/operations/base.py | 3 ++- node_cli/operations/skale_node.py | 21 +++++++++++++++------ 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index b6641d00..d9060af5 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -220,11 +220,23 @@ def __init__(self, requirements: dict) -> None: def _check_docker_command(self) -> str: return shutil.which('docker') + def _get_docker_version_info(self) -> dict: + try: + return self.docker_client.version() + except Exception as err: + logger.error(f'Request to docker api failed {err}') + def docker_engine(self) -> CheckResult: name = 'docker-engine' if self._check_docker_command() is None: return self._error(name=name, info='No such command: "docker"') + version_info = self._get_docker_version_info() + if not version_info: + return self._error( + name=name, + info='Docker api request failed. Is docker installed?' + ) actual_version = self.docker_client.version()['Version'] expected_version = self.requirements['docker']['docker-engine'] info = { @@ -241,7 +253,13 @@ def docker_api(self) -> CheckResult: if self._check_docker_command() is None: return self._error(name=name, info='No such command: "docker"') - actual_version = self.docker_client.version()['ApiVersion'] + version_info = self._get_docker_version_info() + if not version_info: + return self._error( + name=name, + info='Docker api request failed. Is docker installed?' + ) + actual_version = version_info['ApiVersion'] expected_version = self.requirements['docker']['docker-api'] info = { 'expected_version': expected_version, @@ -285,12 +303,3 @@ def keeping_containers_alive(self) -> CheckResult: return self._ok(name=name, info=info) else: return self._error(name=name, info=info) - - def docker_service_status(self) -> CheckResult: - name = 'docker-service-status' - try: - self.docker_client.containers.list() - except Exception as err: - info = err - return self._error(name=name, info=info) - return self._ok(name=name) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 5eec2a19..efa9c228 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -81,6 +81,7 @@ def prepare_host(env_filepath, disk_mountpoint, sgx_server_url, env_type, def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: + logger.info('Checking that host meets requirements ...') requirements = get_net_params(env_type) checkers = [ MachineChecker(requirements), @@ -90,6 +91,8 @@ def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: result = [] for checker in checkers: result.extend(filter(lambda r: r.status == 'error', checker.check())) + if result: + logger.info('Host is not fully meet the requirements') return result diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index ae8cb1d9..fa21c760 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -30,7 +30,7 @@ configure_flask, unpack_backup_archive ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install -from node_cli.operations.skale_node import sync_skale_node +from node_cli.operations.skale_node import sync_skale_node, update_images from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers @@ -94,6 +94,7 @@ def init(env_filepath: str, env: str) -> bool: env['DOCKER_LVMPY_STREAM'] ) update_resource_allocation(env_type=env['ENV_TYPE']) + update_images() compose_up(env) return True diff --git a/node_cli/operations/skale_node.py b/node_cli/operations/skale_node.py index a03b7bea..0946a74b 100644 --- a/node_cli/operations/skale_node.py +++ b/node_cli/operations/skale_node.py @@ -28,20 +28,29 @@ logger = logging.getLogger(__name__) -def sync_skale_node(env): +def update_images(env: dict) -> None: + if 'CONTAINER_CONFIGS_DIR' in env: + compose_build() + else: + compose_pull() + + +def sync_skale_node(env: dict) -> None: if 'CONTAINER_CONFIGS_DIR' in env: sync_skale_node_dev(env) else: sync_skale_node_git(env) -def sync_skale_node_git(env): - sync_repo(SKALE_NODE_REPO_URL, CONTAINER_CONFIG_PATH, env["CONTAINER_CONFIGS_STREAM"]) - compose_pull() +def sync_skale_node_git(env: dict) -> None: + sync_repo( + SKALE_NODE_REPO_URL, + CONTAINER_CONFIG_PATH, + env["CONTAINER_CONFIGS_STREAM"] + ) -def sync_skale_node_dev(env): +def sync_skale_node_dev(env: dict) -> None: logger.info(f'Syncing {CONTAINER_CONFIG_PATH} with {env["CONTAINER_CONFIGS_DIR"]}') run_cmd(f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/ {CONTAINER_CONFIG_PATH}'.split()) run_cmd(f'rsync -r {env["CONTAINER_CONFIGS_DIR"]}/.git {CONTAINER_CONFIG_PATH}'.split()) - compose_build() From b0310897898b23d8f6dc87d6ed91d07639c6156d Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 19 Mar 2021 13:41:43 +0200 Subject: [PATCH 061/140] SKALE-3649 Fix iptables rules --- node_cli/cli/node.py | 12 +++++- node_cli/configs/__init__.py | 1 + node_cli/core/iptables.py | 75 +++++++++++++++++++++++++----------- node_cli/core/node.py | 14 ++++++- node_cli/core/resources.py | 1 - node_cli/operations/base.py | 3 +- tests/core_checks_test.py | 8 ---- 7 files changed, 79 insertions(+), 35 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index b50f6d9e..f18d92e8 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -23,7 +23,9 @@ import click from node_cli.core.node import ( - get_node_signature, init, restore, register_node as register, + configure_firewall_rules, + get_node_signature, init, restore, + register_node as register, update, backup, set_maintenance_mode_on, set_maintenance_mode_off, turn_off, turn_on, get_node_info, @@ -248,3 +250,11 @@ def _set_domain_name(domain): ) def check_requirements(network): run_checks(network) + + +@node.command(help='Reconfigure iptables rules') +@click.option('--yes', is_flag=True, callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to reconfigure firewall rules?') +def configure_firewall(): + configure_firewall_rules() diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 55463cb5..334cc0f0 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -56,6 +56,7 @@ DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') IPTABLES_DIR = '/etc/iptables/' +IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index d5f80537..47683d87 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -18,18 +18,22 @@ # along with this program. If not, see . import logging +import sys from pathlib import Path -from node_cli.configs import IPTABLES_DIR +from node_cli.configs import IPTABLES_DIR, IPTABLES_RULES_STATE_FILEPATH +from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) try: import iptc -except (FileNotFoundError, AttributeError): - logger.warning('Unable to import iptc') - from collections import namedtuple # hotfix for tests - iptc = namedtuple('iptc', ['Chain', 'Rule']) +except (FileNotFoundError, AttributeError) as err: + if "pytest" in sys.modules: + from collections import namedtuple # hotfix for tests + iptc = namedtuple('iptc', ['Chain', 'Rule']) + else: + logger.error(f'Unable to import iptc due to an error {err}') ALLOWED_INCOMING_TCP_PORTS = [ @@ -61,10 +65,19 @@ def configure_iptables(): set_base_policies() allow_loopback(input_chain) - allow_base_ports(input_chain) - drop_all_input(input_chain) accept_icmp(input_chain) allow_conntrack(input_chain) + allow_base_ports(input_chain) + drop_all_tcp(input_chain) + drop_all_udp(input_chain) + save_iptables_rules_state() + + +def save_iptables_rules_state(): + res = run_cmd(['iptables-save']) + plain_rules = res.stdout.decode('utf-8').rstrip() + with open(IPTABLES_RULES_STATE_FILEPATH, 'w') as state_file: + state_file.write(plain_rules) def set_base_policies() -> None: @@ -81,29 +94,44 @@ def allow_loopback(chain: iptc.Chain) -> None: rule = iptc.Rule() rule.target = iptc.Target(rule, 'ACCEPT') rule.in_interface = 'lo' - chain.insert_rule(rule) + ensure_rule(chain, rule) def allow_conntrack(chain: iptc.Chain) -> None: """Allow conntrack established connections""" logger.debug('Allowing conntrack...') rule = iptc.Rule() - rule.protocol = "tcp" - rule.target = iptc.Target(rule, "ACCEPT") - match = iptc.Match(rule, "conntrack") - chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT") - match.ctstate = "RELATED,ESTABLISHED" + # rule.protocol = 'tcp' + rule.target = iptc.Target(rule, 'ACCEPT') + match = iptc.Match(rule, 'conntrack') + chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), 'INPUT') + match.ctstate = 'RELATED,ESTABLISHED' rule.add_match(match) - chain.insert_rule(rule) + ensure_rule(chain, rule) + + +def drop_all_tcp(chain: iptc.Chain) -> None: + """Drop the rest of tcp connections """ + logger.debug('Adding drop tcp rule ...') + r = iptc.Rule() + t = iptc.Target(r, 'DROP') + r.target = t + r.protocol = 'tcp' + # tcp_p = iptc.Match(r, 'tcp') + # r.add_match(tcp_p) + ensure_rule(chain, r) -def drop_all_input(chain: iptc.Chain) -> None: - """Drop all input connections by default (append in the end)""" - logger.debug('Droping all input connections except specified...') +def drop_all_udp(chain: iptc.Chain) -> None: + """Drop the rest of udp connections """ + logger.debug('Adding drop udp rule ...') r = iptc.Rule() t = iptc.Target(r, 'DROP') r.target = t - chain.append_rule(r) + r.protocol = 'udp' + # upd_p = iptc.Match(r, 'udp') + # r.add_match(udp_p) + ensure_rule(chain, r) def allow_base_ports(chain: iptc.Chain) -> None: @@ -122,7 +150,7 @@ def accept_incoming(chain, port, protocol) -> None: t = iptc.Target(rule, 'ACCEPT') rule.target = t rule.add_match(match) - safe_add_rule(chain, rule) + ensure_rule(chain, rule) def accept_icmp(chain: iptc.Chain) -> None: @@ -139,13 +167,16 @@ def add_icmp_rule(chain: iptc.Chain, icmp_type: str) -> None: t = iptc.Target(rule, 'ACCEPT') rule.target = t rule.add_match(match) - safe_add_rule(chain, rule) + ensure_rule(chain, rule) -def safe_add_rule(chain: iptc.Chain, rule: iptc.Rule) -> None: +def ensure_rule(chain: iptc.Chain, rule: iptc.Rule, position='bottom') -> None: if rule not in chain.rules: logger.debug(f'Adding rule: {rule_to_dict(rule)}, chain: {chain.name}') - chain.insert_rule(rule) + if position == 'top': + chain.insert_rule(rule) + else: + chain.append_rule(rule) else: logger.debug(f'Rule already present: {rule_to_dict(rule)}, chain: {chain.name}') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 54969f15..3c11872b 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -33,13 +33,17 @@ ) from node_cli.configs.cli_logger import LOG_DIRNAME -from node_cli.operations import update_op, init_op, turn_off_op, turn_on_op, restore_op -from node_cli.core.resources import update_resource_allocation +from node_cli.core.iptables import configure_iptables from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import ( is_node_inited, save_env_params, get_flask_secret_key, run_preinstall_checks ) +from node_cli.core.resources import update_resource_allocation +from node_cli.operations import ( + update_op, + init_op, turn_off_op, turn_on_op, restore_op +) from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, print_node_info ) @@ -341,3 +345,9 @@ def run_checks(network: str) -> None: else: print('Node is not fully meet the requirements!') print_failed_requirements_checks(failed_checks) + + +def configure_firewall_rules() -> None: + print('Configuring firewall ...') + configure_iptables() + print('Done') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index e6503b16..5e6ffacb 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -191,7 +191,6 @@ def get_static_disk_alloc(env_type: str): env_disk_size = get_net_params(env_type)['server']['disk'] check_disk_size(disk_size, env_disk_size) free_space = calculate_free_disk_space(env_disk_size) - print(free_space, env_disk_size) return ResourceAlloc(free_space) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index fa21c760..24722978 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -63,6 +63,7 @@ def update(env_filepath: str, env: str) -> None: env['CONTAINER_CONFIGS_STREAM'], env['DOCKER_LVMPY_STREAM'] ) + update_images(env) compose_up(env) @@ -94,7 +95,7 @@ def init(env_filepath: str, env: str) -> bool: env['DOCKER_LVMPY_STREAM'] ) update_resource_allocation(env_type=env['ENV_TYPE']) - update_images() + update_images(env) compose_up(env) return True diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index 0b9dc6e7..0950486f 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -209,14 +209,6 @@ def test_checks_docker_compose_invalid_version( r.status == 'ok' -def test_checks_docker_service_status(): - checker = DockerChecker(requirements_data) - checker.docker_client = mock.Mock() - r = checker.docker_service_status() - r.name == 'docker-compose' - r.status == 'ok' - - def test_checks_docker_config(): checker = DockerChecker(requirements_data) valid_config = { From 003cb8a4a96d74c56e7845650f0f7e833b2a186a Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 19 Mar 2021 13:49:06 +0200 Subject: [PATCH 062/140] SKALE-3649 Remove unnecessary comments --- node_cli/core/iptables.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 47683d87..809e801d 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -101,7 +101,6 @@ def allow_conntrack(chain: iptc.Chain) -> None: """Allow conntrack established connections""" logger.debug('Allowing conntrack...') rule = iptc.Rule() - # rule.protocol = 'tcp' rule.target = iptc.Target(rule, 'ACCEPT') match = iptc.Match(rule, 'conntrack') chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), 'INPUT') @@ -111,14 +110,12 @@ def allow_conntrack(chain: iptc.Chain) -> None: def drop_all_tcp(chain: iptc.Chain) -> None: - """Drop the rest of tcp connections """ + """Drop the rest of tcp connections""" logger.debug('Adding drop tcp rule ...') r = iptc.Rule() t = iptc.Target(r, 'DROP') r.target = t r.protocol = 'tcp' - # tcp_p = iptc.Match(r, 'tcp') - # r.add_match(tcp_p) ensure_rule(chain, r) @@ -129,8 +126,6 @@ def drop_all_udp(chain: iptc.Chain) -> None: t = iptc.Target(r, 'DROP') r.target = t r.protocol = 'udp' - # upd_p = iptc.Match(r, 'udp') - # r.add_match(udp_p) ensure_rule(chain, r) From 81cd149719781880b0ba2d322f4b4d2f2226c91f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Mar 2021 05:00:32 +0000 Subject: [PATCH 063/140] Update isort requirement from <5.4.3,>=4.2.15 to >=4.2.15,<5.8.1 Updates the requirements on [isort](https://github.com/pycqa/isort) to permit the latest version. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/4.2.15...5.8.0) Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50b7818b..95cdbb2a 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def find_version(*file_paths): extras_require = { 'linter': [ "flake8==3.7.9", - "isort>=4.2.15,<5.4.3", + "isort>=4.2.15,<5.8.1", ], 'dev': [ "bumpversion==0.6.0", From 0c688846ec0a042eec8344f0f2e1d1b66415d15f Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 29 Mar 2021 11:09:22 +0300 Subject: [PATCH 064/140] SKALE-3649 Remove redundant inserting rule capabilities --- node_cli/core/iptables.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 809e801d..cebd3c00 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -165,13 +165,10 @@ def add_icmp_rule(chain: iptc.Chain, icmp_type: str) -> None: ensure_rule(chain, rule) -def ensure_rule(chain: iptc.Chain, rule: iptc.Rule, position='bottom') -> None: +def ensure_rule(chain: iptc.Chain, rule: iptc.Rule) -> None: if rule not in chain.rules: logger.debug(f'Adding rule: {rule_to_dict(rule)}, chain: {chain.name}') - if position == 'top': - chain.insert_rule(rule) - else: - chain.append_rule(rule) + chain.append_rule(rule) else: logger.debug(f'Rule already present: {rule_to_dict(rule)}, chain: {chain.name}') From 30bc1dd0fd83c1ed7121a4ce6bb33fbb0d04e4e3 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 30 Mar 2021 18:47:18 +0300 Subject: [PATCH 065/140] SKALE-3978 Restructure code - WIP --- .../utils/{validations.py => decorators.py} | 48 ++------------ node_cli/utils/global_config.py | 62 +++++++++++++++++++ tests/utils/validations_test.py | 8 +-- 3 files changed, 70 insertions(+), 48 deletions(-) rename node_cli/utils/{validations.py => decorators.py} (60%) create mode 100644 node_cli/utils/global_config.py diff --git a/node_cli/utils/validations.py b/node_cli/utils/decorators.py similarity index 60% rename from node_cli/utils/validations.py rename to node_cli/utils/decorators.py index 5c06a9aa..c4bf8ade 100644 --- a/node_cli/utils/validations.py +++ b/node_cli/utils/decorators.py @@ -17,57 +17,17 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os from functools import wraps -from node_cli.core.host import safe_mk_dirs, is_node_inited -from node_cli.utils.helper import write_json, read_json, error_exit +from node_cli.core.host import is_node_inited +from node_cli.utils.helper import error_exit from node_cli.utils.texts import Texts -from node_cli.configs import GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR - from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.validations import is_user_valid, get_g_conf_user, get_system_user TEXTS = Texts() -def read_g_config() -> dict: - """Read global SKALE config file, init if not exists""" - try: - return read_json(GLOBAL_SKALE_CONF_FILEPATH) - except FileNotFoundError: - return generate_g_config_file() - - -def generate_g_config_file() -> dict: - """Init global SKALE config file""" - safe_mk_dirs(GLOBAL_SKALE_DIR) - g_config = { - 'user': safe_get_user(), - 'home_dir': os.path.expanduser('~') - } - write_json(GLOBAL_SKALE_CONF_FILEPATH, g_config) - return g_config - - -def safe_get_user() -> str: - if os.environ.get('SUDO_USER'): - return os.environ['SUDO_USER'] - else: - return os.environ['USER'] - - -def is_user_valid(allow_root=True): - current_user = safe_get_user() - if current_user == 'root' and allow_root: - return True - g_conf_user = get_g_conf_user() - return current_user == g_conf_user - - -def get_g_conf_user(): - return read_g_config()['user'] - - def check_not_inited(f): @wraps(f) def inner(*args, **kwargs): @@ -91,7 +51,7 @@ def check_user(f): def inner(*args, **kwargs): if not is_user_valid(): g_conf_user = get_g_conf_user() - current_user = safe_get_user() + current_user = get_system_user() error_msg = f'You couldn\'t execute this command from user {current_user}. \ Allowed: {g_conf_user} or root.' error_exit(error_msg, exit_code=CLIExitCodes.BAD_USER_ERROR) diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py new file mode 100644 index 00000000..13cbb0ab --- /dev/null +++ b/node_cli/utils/global_config.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from node_cli.core.host import safe_mk_dirs +from node_cli.utils.helper import write_json, read_json +from node_cli.configs import GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR + + +def read_g_config() -> dict: + """Read global SKALE config file, init if not exists""" + try: + return read_json(GLOBAL_SKALE_CONF_FILEPATH) + except FileNotFoundError: + return generate_g_config_file() + + +def generate_g_config_file() -> dict: + """Init global SKALE config file""" + safe_mk_dirs(GLOBAL_SKALE_DIR) + g_config = { + 'user': get_system_user(), + 'home_dir': os.path.expanduser('~') + } + write_json(GLOBAL_SKALE_CONF_FILEPATH, g_config) + return g_config + + +def get_system_user() -> str: + if os.getenv('SUDO_USER'): + return os.environ['SUDO_USER'] + else: + return os.environ['USER'] + + +def is_user_valid(allow_root=True): + current_user = get_system_user() + if current_user == 'root' and allow_root: + return True + g_conf_user = get_g_conf_user() + return current_user == g_conf_user + + +def get_g_conf_user(): + return read_g_config()['user'] diff --git a/tests/utils/validations_test.py b/tests/utils/validations_test.py index f906f182..970224cd 100644 --- a/tests/utils/validations_test.py +++ b/tests/utils/validations_test.py @@ -3,7 +3,7 @@ import pytest from node_cli.utils.validations import ( - read_g_config, generate_g_config_file, safe_get_user, is_user_valid, get_g_conf_user, + read_g_config, generate_g_config_file, get_system_user, is_user_valid, get_g_conf_user, check_not_inited, check_inited, check_user ) from node_cli.utils.helper import write_json @@ -28,16 +28,16 @@ def test_generate_g_config_file(mocked_g_config): assert os.path.exists(TEST_G_CONF_FP) g_config = read_g_config() - assert g_config['user'] == safe_get_user() + assert g_config['user'] == get_system_user() assert g_config['home_dir'] == os.path.expanduser('~') -def test_safe_get_user(): +def test_get_system_user(): sudo_user = os.environ.get('SUDO_USER') if sudo_user: del os.environ['SUDO_USER'] os.environ['USER'] = 'test' - assert safe_get_user() == 'test' + assert get_system_user() == 'test' if sudo_user: os.environ['SUDO_USER'] = sudo_user From cc0c84fc08e0748b0ccaa5b051e775fbc8c54e76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Apr 2021 05:00:34 +0000 Subject: [PATCH 066/140] Bump pytest from 5.4.3 to 6.2.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.3 to 6.2.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.3...6.2.3) Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fc8e71d0..d296bb7d 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def find_version(*file_paths): ], 'dev': [ "bumpversion==0.6.0", - "pytest==5.4.3", + "pytest==6.2.3", "pytest-cov==2.9.0", "twine==2.0.0", "mock==4.0.2" From 717fabe8f37cd5376bc32b5500333be4bcb43492 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 15 Apr 2021 15:33:06 +0300 Subject: [PATCH 067/140] SKALE-3978 Refactor code --- node_cli/core/node.py | 2 +- node_cli/utils/decorators.py | 2 +- node_cli/utils/global_config.py | 4 ++++ tests/cli/node_test.py | 20 ++++++++++---------- tests/conftest.py | 6 ++++-- tests/utils/validations_test.py | 14 +++++++------- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 39c3e1c6..7cdf7836 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -43,7 +43,7 @@ from node_cli.utils.helper import run_cmd, extract_env_params from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.validations import check_not_inited, check_inited, check_user +from node_cli.utils.decorators import check_not_inited, check_inited, check_user logger = logging.getLogger(__name__) diff --git a/node_cli/utils/decorators.py b/node_cli/utils/decorators.py index c4bf8ade..14c587a1 100644 --- a/node_cli/utils/decorators.py +++ b/node_cli/utils/decorators.py @@ -23,7 +23,7 @@ from node_cli.utils.helper import error_exit from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.validations import is_user_valid, get_g_conf_user, get_system_user +from node_cli.utils.global_config import is_user_valid, get_g_conf_user, get_system_user TEXTS = Texts() diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 13cbb0ab..604fa71e 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -60,3 +60,7 @@ def is_user_valid(allow_root=True): def get_g_conf_user(): return read_g_config()['user'] + + +def get_g_conf_home(): + return read_g_config()['home_dir'] diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index d53c4a7f..abfeba6c 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -50,7 +50,7 @@ def test_register_node(mocked_g_config): requests.codes.ok, {'status': 'ok', 'payload': None} ) - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -65,7 +65,7 @@ def test_register_node_with_error(mocked_g_config): requests.codes.ok, {'status': 'error', 'payload': ['Strange error']}, ) - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -80,7 +80,7 @@ def test_register_node_with_prompted_ip(mocked_g_config): requests.codes.ok, {'status': 'ok', 'payload': None} ) - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -95,7 +95,7 @@ def test_register_node_with_default_port(mocked_g_config): requests.codes.ok, {'status': 'ok', 'payload': None} ) - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -115,7 +115,7 @@ def test_init_node(caplog): # todo: write new init node test mock.patch('node_cli.core.node.init_op'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, @@ -158,7 +158,7 @@ def test_update_node_without_init(): mock.patch('node_cli.core.host.init_data_dir'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command_mock( 'node_cli.utils.helper.post_request', resp_mock, @@ -350,7 +350,7 @@ def test_restore(mocked_g_config): 'Backup archive successfully created: ', '').replace('\n', '') with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.resources.get_static_disk_alloc', new=disk_alloc_mock), \ - mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] @@ -392,7 +392,7 @@ def test_turn_off_maintenance_on(mocked_g_config): {'status': 'ok', 'payload': None} ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -412,7 +412,7 @@ def test_turn_on_maintenance_off(mocked_g_config): ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ - mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -434,7 +434,7 @@ def test_set_domain_name(): {'status': 'ok', 'payload': None} ) - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, diff --git a/tests/conftest.py b/tests/conftest.py index 3a22d835..87304696 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,8 @@ @pytest.fixture() def mocked_g_config(): - with mock.patch('node_cli.utils.validations.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP), \ - mock.patch('node_cli.utils.validations.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): + with mock.patch( + 'node_cli.utils.global_config.GLOBAL_SKALE_CONF_FILEPATH', + new=TEST_G_CONF_FP + ), mock.patch('node_cli.utils.global_config.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): yield diff --git a/tests/utils/validations_test.py b/tests/utils/validations_test.py index 970224cd..88b3a74f 100644 --- a/tests/utils/validations_test.py +++ b/tests/utils/validations_test.py @@ -2,10 +2,10 @@ import mock import pytest -from node_cli.utils.validations import ( - read_g_config, generate_g_config_file, get_system_user, is_user_valid, get_g_conf_user, - check_not_inited, check_inited, check_user +from node_cli.utils.global_config import ( + read_g_config, generate_g_config_file, get_system_user, is_user_valid, get_g_conf_user ) +from node_cli.utils.decorators import check_not_inited, check_inited, check_user from node_cli.utils.helper import write_json from tests.conftest import TEST_G_CONF_FP @@ -69,9 +69,9 @@ def test_check_not_inited(): @check_not_inited def requires_not_inited_node(): pass - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + with mock.patch('node_cli.core.host.is_node_inited', return_value=False): requires_not_inited_node() - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.core.host.is_node_inited', return_value=True): with pytest.raises(SystemExit): requires_not_inited_node() @@ -80,9 +80,9 @@ def test_check_inited(): @check_inited def requires_inited_node(): pass - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=True): + with mock.patch('node_cli.core.host.is_node_inited', return_value=True): requires_inited_node() - with mock.patch('node_cli.utils.validations.is_node_inited', return_value=False): + with mock.patch('node_cli.core.host.is_node_inited', return_value=False): with pytest.raises(SystemExit): requires_inited_node() From 8ec45cbde235c196c8d39a4b4173f576659042df Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 15 Apr 2021 19:45:30 +0300 Subject: [PATCH 068/140] SKALE-3978 Fix run tests --- scripts/run_tests.sh | 4 +--- tests/conftest.py | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 2efba50f..0f197d59 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -2,7 +2,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -export HIDE_STREAM_LOG=true -export HOME_DIR='tests/' -HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +HIDE_STREAM_LOG=true HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/tests/conftest.py b/tests/conftest.py index 9aa79155..3863065c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,10 +23,13 @@ import yaml from node_cli.configs import ENVIRONMENT_PARAMS_FILEPATH +from node_cli.utils.global_config import generate_g_config_file + TEST_GLOBAL_SKALE_DIR = os.path.join(os.environ.get('HOME_DIR'), 'etc', 'skale') TEST_G_CONF_FP = os.path.join(TEST_GLOBAL_SKALE_DIR, 'conf.json') + TEST_NET_PARAMS = """ mainnet: server: @@ -121,4 +124,5 @@ def mocked_g_config(): 'node_cli.utils.global_config.GLOBAL_SKALE_CONF_FILEPATH', new=TEST_G_CONF_FP ), mock.patch('node_cli.utils.global_config.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): + generate_g_config_file() yield From 9e70f81b701441bcfb2d7d22fa362b026fff165e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Apr 2021 08:51:12 +0000 Subject: [PATCH 069/140] Bump gitpython from 3.1.12 to 3.1.14 Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.12 to 3.1.14. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/main/CHANGES) - [Commits](https://github.com/gitpython-developers/GitPython/compare/3.1.12...3.1.14) Signed-off-by: dependabot[bot] --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d296bb7d..4371c400 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def find_version(*file_paths): extras_require = { 'linter': [ "flake8==3.7.9", - "isort>=4.2.15,<5.4.3", + "isort>=4.2.15,<5.8.1", ], 'dev': [ "bumpversion==0.6.0", @@ -59,7 +59,7 @@ def find_version(*file_paths): "python-dotenv==0.13.0", "terminaltables==3.1.0", "requests==2.23.0", - "GitPython==3.1.12", + "GitPython==3.1.14", "PyYAML==5.4.1", "packaging==20.9", "python-debian==0.1.39", From 5ccd72bea346892746c5a9074159973c42f9b3b9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 16 Apr 2021 15:38:50 +0300 Subject: [PATCH 070/140] SKALE-3979 Fix tests --- tests/utils/decorators_test.py | 41 +++++++++++++++++++ ...idations_test.py => global_config_test.py} | 38 +---------------- 2 files changed, 42 insertions(+), 37 deletions(-) create mode 100644 tests/utils/decorators_test.py rename tests/utils/{validations_test.py => global_config_test.py} (60%) diff --git a/tests/utils/decorators_test.py b/tests/utils/decorators_test.py new file mode 100644 index 00000000..cb099bc8 --- /dev/null +++ b/tests/utils/decorators_test.py @@ -0,0 +1,41 @@ +import mock +import pytest + +from node_cli.utils.global_config import generate_g_config_file +from node_cli.utils.decorators import check_not_inited, check_inited, check_user +from node_cli.utils.helper import write_json + +from tests.conftest import TEST_G_CONF_FP + + +def test_check_not_inited(): + @check_not_inited + def requires_not_inited_node(): + pass + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + requires_not_inited_node() + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): + with pytest.raises(SystemExit): + requires_not_inited_node() + + +def test_check_inited(): + @check_inited + def requires_inited_node(): + pass + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): + requires_inited_node() + with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + with pytest.raises(SystemExit): + requires_inited_node() + + +def test_check_user(mocked_g_config): + @check_user + def this_checks_user(): + pass + generate_g_config_file() + this_checks_user() + write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + with pytest.raises(SystemExit): + this_checks_user() diff --git a/tests/utils/validations_test.py b/tests/utils/global_config_test.py similarity index 60% rename from tests/utils/validations_test.py rename to tests/utils/global_config_test.py index 88b3a74f..b3452721 100644 --- a/tests/utils/validations_test.py +++ b/tests/utils/global_config_test.py @@ -1,11 +1,8 @@ -import os -import mock -import pytest +import os from node_cli.utils.global_config import ( read_g_config, generate_g_config_file, get_system_user, is_user_valid, get_g_conf_user ) -from node_cli.utils.decorators import check_not_inited, check_inited, check_user from node_cli.utils.helper import write_json from tests.conftest import TEST_G_CONF_FP @@ -63,36 +60,3 @@ def test_is_user_valid(mocked_g_config): def test_get_g_conf_user(mocked_g_config): write_json(TEST_G_CONF_FP, {'user': 'test_get_g_conf_user'}) assert get_g_conf_user() == 'test_get_g_conf_user' - - -def test_check_not_inited(): - @check_not_inited - def requires_not_inited_node(): - pass - with mock.patch('node_cli.core.host.is_node_inited', return_value=False): - requires_not_inited_node() - with mock.patch('node_cli.core.host.is_node_inited', return_value=True): - with pytest.raises(SystemExit): - requires_not_inited_node() - - -def test_check_inited(): - @check_inited - def requires_inited_node(): - pass - with mock.patch('node_cli.core.host.is_node_inited', return_value=True): - requires_inited_node() - with mock.patch('node_cli.core.host.is_node_inited', return_value=False): - with pytest.raises(SystemExit): - requires_inited_node() - - -def test_check_user(mocked_g_config): - @check_user - def this_checks_user(): - pass - generate_g_config_file() - this_checks_user() - write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) - with pytest.raises(SystemExit): - this_checks_user() From c4f64d8adeee0dea79880744dd336a1720124579 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 19 Apr 2021 20:01:20 +0300 Subject: [PATCH 071/140] SKALE-3978 Init global config before vars init --- node_cli/configs/__init__.py | 18 ++++++---- node_cli/utils/decorators.py | 4 +-- node_cli/utils/global_config.py | 59 +++++++++++++------------------ node_cli/utils/helper.py | 33 ++++++++++++++++- scripts/run_tests.sh | 2 +- tests/conftest.py | 18 ++++------ tests/etc/skale/.keep | 0 tests/utils/decorators_test.py | 7 ++-- tests/utils/global_config_test.py | 29 +++++++-------- 9 files changed, 93 insertions(+), 77 deletions(-) delete mode 100644 tests/etc/skale/.keep diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 29fcabae..98f406c8 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -19,10 +19,18 @@ import os import sys -from pathlib import Path +from node_cli.utils.global_config import read_g_config -HOME_DIR = os.getenv('HOME_DIR') or str(Path.home()) -SKALE_DIR = os.path.join(HOME_DIR, '.skale') + +GLOBAL_SKALE_DIR = os.getenv('GLOBAL_SKALE_DIR') or '/etc/skale' +GLOBAL_SKALE_CONF_FILENAME = 'conf.json' +GLOBAL_SKALE_CONF_FILEPATH = os.path.join(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILENAME) +GLOBAL_CONFIG = read_g_config(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + +G_CONF_USER = GLOBAL_CONFIG['user'] +G_CONF_HOME = GLOBAL_CONFIG['home_dir'] + +SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data') CONTAINER_CONFIG_PATH = os.path.join(SKALE_DIR, 'config') @@ -115,7 +123,3 @@ def _get_env(): SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' - -GLOBAL_SKALE_DIR = '/etc/skale' -GLOBAL_SKALE_CONF_FILENAME = 'conf.json' -GLOBAL_SKALE_CONF_FILEPATH = os.path.join(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILENAME) diff --git a/node_cli/utils/decorators.py b/node_cli/utils/decorators.py index 14c587a1..a79e99bf 100644 --- a/node_cli/utils/decorators.py +++ b/node_cli/utils/decorators.py @@ -20,10 +20,10 @@ from functools import wraps from node_cli.core.host import is_node_inited -from node_cli.utils.helper import error_exit +from node_cli.utils.helper import error_exit, is_user_valid, get_g_conf_user, get_system_user from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.global_config import is_user_valid, get_g_conf_user, get_system_user + TEXTS = Texts() diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 604fa71e..4badacde 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -18,49 +18,40 @@ # along with this program. If not, see . import os +import json +import logging -from node_cli.core.host import safe_mk_dirs -from node_cli.utils.helper import write_json, read_json -from node_cli.configs import GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR +logger = logging.getLogger(__name__) -def read_g_config() -> dict: +def get_system_user() -> str: + user = os.getenv('SUDO_USER', os.getenv('USER')) + if not user: + raise ValueError('SUDO_USER or USER env variable should be set') + return user + + +def read_g_config(g_skale_dir: str, g_skale_conf_filepath: str) -> dict: """Read global SKALE config file, init if not exists""" - try: - return read_json(GLOBAL_SKALE_CONF_FILEPATH) - except FileNotFoundError: - return generate_g_config_file() + if not os.path.isfile(g_skale_conf_filepath): + return generate_g_config_file(g_skale_dir, g_skale_conf_filepath) + with open(g_skale_conf_filepath, encoding='utf-8') as data_file: + return json.loads(data_file.read()) -def generate_g_config_file() -> dict: +def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict: """Init global SKALE config file""" - safe_mk_dirs(GLOBAL_SKALE_DIR) + logger.info('Generating global SKALE config file...') + os.makedirs(g_skale_dir, exist_ok=True) + + user = os.environ['USER'] + user = os.getenv('SUDO_USER', user) + g_config = { 'user': get_system_user(), 'home_dir': os.path.expanduser('~') } - write_json(GLOBAL_SKALE_CONF_FILEPATH, g_config) + logger.info(f'{g_skale_conf_filepath} content: {g_config}') + with open(g_skale_conf_filepath, 'w') as outfile: + json.dump(g_config, outfile, indent=4) return g_config - - -def get_system_user() -> str: - if os.getenv('SUDO_USER'): - return os.environ['SUDO_USER'] - else: - return os.environ['USER'] - - -def is_user_valid(allow_root=True): - current_user = get_system_user() - if current_user == 'root' and allow_root: - return True - g_conf_user = get_g_conf_user() - return current_user == g_conf_user - - -def get_g_conf_user(): - return read_g_config()['user'] - - -def get_g_conf_home(): - return read_g_config()['home_dir'] diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 0d39e0ed..42b19ae8 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -45,12 +45,16 @@ absent_params as absent_env_params, get_params as get_env_params ) -from node_cli.configs import TEXT_FILE, ADMIN_HOST, ADMIN_PORT, HIDE_STREAM_LOG +from node_cli.configs import ( + TEXT_FILE, ADMIN_HOST, ADMIN_PORT, HIDE_STREAM_LOG, GLOBAL_SKALE_DIR, + GLOBAL_SKALE_CONF_FILEPATH +) from node_cli.configs.cli_logger import ( LOG_BACKUP_COUNT, LOG_FILE_SIZE_BYTES, STREAM_LOG_FORMAT, LOG_FILEPATH, DEBUG_LOG_FILEPATH, FILE_LOG_FORMAT ) from node_cli.configs.routes import get_route +from node_cli.utils.global_config import read_g_config logger = logging.getLogger(__name__) @@ -315,3 +319,30 @@ def wrapper(*args, **kwargs): logging.getLogger('').addHandler(get_stream_handler()) return func(*args, **kwargs) return wrapper + + +def get_system_user() -> str: + user = os.getenv('SUDO_USER', os.getenv('USER')) + if not user: + raise ValueError('SUDO_USER or USER env variable should be set') + return user + + +def is_user_valid(allow_root=True): + current_user = get_system_user() + if current_user == 'root' and allow_root: + return True + g_conf_user = get_g_conf_user() + return current_user == g_conf_user + + +def get_g_conf(): + return read_g_config(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + + +def get_g_conf_user(): + return get_g_conf()['user'] + + +def get_g_conf_home(): + return get_g_conf()['home_dir'] diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0f197d59..39dce203 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true HOME_DIR='tests/' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +HIDE_STREAM_LOG=true HOME_DIR='tests/' GLOBAL_SKALE_DIR='tests/etc/skale' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/tests/conftest.py b/tests/conftest.py index 3863065c..6eed6800 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,18 +18,15 @@ # along with this program. If not, see . import os -import mock import pytest import yaml -from node_cli.configs import ENVIRONMENT_PARAMS_FILEPATH +from node_cli.configs import ( + ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH +) from node_cli.utils.global_config import generate_g_config_file -TEST_GLOBAL_SKALE_DIR = os.path.join(os.environ.get('HOME_DIR'), 'etc', 'skale') -TEST_G_CONF_FP = os.path.join(TEST_GLOBAL_SKALE_DIR, 'conf.json') - - TEST_NET_PARAMS = """ mainnet: server: @@ -120,9 +117,6 @@ def net_params_file(): @pytest.fixture() def mocked_g_config(): - with mock.patch( - 'node_cli.utils.global_config.GLOBAL_SKALE_CONF_FILEPATH', - new=TEST_G_CONF_FP - ), mock.patch('node_cli.utils.global_config.GLOBAL_SKALE_DIR', new=TEST_GLOBAL_SKALE_DIR): - generate_g_config_file() - yield + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + yield + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) diff --git a/tests/etc/skale/.keep b/tests/etc/skale/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/utils/decorators_test.py b/tests/utils/decorators_test.py index cb099bc8..e67e1f04 100644 --- a/tests/utils/decorators_test.py +++ b/tests/utils/decorators_test.py @@ -4,8 +4,7 @@ from node_cli.utils.global_config import generate_g_config_file from node_cli.utils.decorators import check_not_inited, check_inited, check_user from node_cli.utils.helper import write_json - -from tests.conftest import TEST_G_CONF_FP +from node_cli.configs import GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH def test_check_not_inited(): @@ -34,8 +33,8 @@ def test_check_user(mocked_g_config): @check_user def this_checks_user(): pass - generate_g_config_file() + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) this_checks_user() - write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + write_json(GLOBAL_SKALE_CONF_FILEPATH, {'user': 'skaletest'}) with pytest.raises(SystemExit): this_checks_user() diff --git a/tests/utils/global_config_test.py b/tests/utils/global_config_test.py index b3452721..6d118791 100644 --- a/tests/utils/global_config_test.py +++ b/tests/utils/global_config_test.py @@ -1,30 +1,27 @@ import os -from node_cli.utils.global_config import ( - read_g_config, generate_g_config_file, get_system_user, is_user_valid, get_g_conf_user -) -from node_cli.utils.helper import write_json - -from tests.conftest import TEST_G_CONF_FP +from node_cli.utils.global_config import read_g_config, generate_g_config_file +from node_cli.utils.helper import write_json, get_system_user, is_user_valid, get_g_conf_user +from node_cli.configs import GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH def test_read_g_config(mocked_g_config): - write_json(TEST_G_CONF_FP, {'test': 1}) - g_config = read_g_config() + write_json(GLOBAL_SKALE_CONF_FILEPATH, {'test': 1}) + g_config = read_g_config(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) assert g_config['test'] == 1 def test_generate_g_config_file(mocked_g_config): try: - os.remove(TEST_G_CONF_FP) + os.remove(GLOBAL_SKALE_CONF_FILEPATH) except OSError: pass - assert not os.path.exists(TEST_G_CONF_FP) - generate_g_config_file() - assert os.path.exists(TEST_G_CONF_FP) + assert not os.path.exists(GLOBAL_SKALE_CONF_FILEPATH) + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + assert os.path.exists(GLOBAL_SKALE_CONF_FILEPATH) - g_config = read_g_config() + g_config = read_g_config(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) assert g_config['user'] == get_system_user() assert g_config['home_dir'] == os.path.expanduser('~') @@ -40,10 +37,10 @@ def test_get_system_user(): def test_is_user_valid(mocked_g_config): - generate_g_config_file() + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) assert is_user_valid() - write_json(TEST_G_CONF_FP, {'user': 'skaletest'}) + write_json(GLOBAL_SKALE_CONF_FILEPATH, {'user': 'skaletest'}) assert not is_user_valid() sudo_user = os.environ.get('SUDO_USER') @@ -58,5 +55,5 @@ def test_is_user_valid(mocked_g_config): def test_get_g_conf_user(mocked_g_config): - write_json(TEST_G_CONF_FP, {'user': 'test_get_g_conf_user'}) + write_json(GLOBAL_SKALE_CONF_FILEPATH, {'user': 'test_get_g_conf_user'}) assert get_g_conf_user() == 'test_get_g_conf_user' From 2ad513dd72d86375e3f4c2c965d61ab96eb64b1b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 20 Apr 2021 14:52:06 +0300 Subject: [PATCH 072/140] SKALE-3978 Fix tests --- node_cli/core/node.py | 4 ++-- node_cli/operations/common.py | 4 ++-- tests/conftest.py | 10 ++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 193f09df..bed449ba 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -29,7 +29,7 @@ from node_cli.configs import ( SKALE_DIR, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, HOME_DIR, RESTORE_SLEEP_TIMEOUT, TM_INIT_TIMEOUT + BACKUP_ARCHIVE_NAME, G_CONF_HOME, RESTORE_SLEEP_TIMEOUT, TM_INIT_TIMEOUT ) from node_cli.configs.cli_logger import LOG_DIRNAME @@ -217,7 +217,7 @@ def create_backup_archive(backup_filepath): print('Creating backup archive...') log_skale_path = os.path.join('.skale', LOG_DIRNAME) cmd = shlex.split( - f'tar -zcvf {backup_filepath} -C {HOME_DIR} ' + f'tar -zcvf {backup_filepath} -C {G_CONF_HOME} ' f'--exclude {log_skale_path} .skale' ) try: diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 4f264739..5fd6bef7 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -30,7 +30,7 @@ from node_cli.configs import ( CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, HOME_DIR, + MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, G_CONF_HOME, FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE ) from node_cli.utils.helper import read_json @@ -83,4 +83,4 @@ def configure_flask(): def unpack_backup_archive(backup_path: str) -> None: logger.info('Unpacking backup archive...') with tarfile.open(backup_path) as tar: - tar.extractall(path=HOME_DIR) + tar.extractall(path=G_CONF_HOME) diff --git a/tests/conftest.py b/tests/conftest.py index 6eed6800..58b3fbb1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,8 +18,9 @@ # along with this program. If not, see . import os -import pytest +import mock import yaml +import pytest from node_cli.configs import ( ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH @@ -117,6 +118,7 @@ def net_params_file(): @pytest.fixture() def mocked_g_config(): - generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) - yield - generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + with mock.patch('os.path.expanduser', return_value='tests/'): + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) + yield + generate_g_config_file(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) From 314af3fa379dabecdf44f1e3daef648808d87984 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 20 Apr 2021 15:10:19 +0300 Subject: [PATCH 073/140] SKALE-3978 Fix tests --- node_cli/configs/__init__.py | 2 +- scripts/run_tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 98f406c8..3ff124b0 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -28,7 +28,7 @@ GLOBAL_CONFIG = read_g_config(GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH) G_CONF_USER = GLOBAL_CONFIG['user'] -G_CONF_HOME = GLOBAL_CONFIG['home_dir'] +G_CONF_HOME = os.getenv('TEST_HOME_DIR') or GLOBAL_CONFIG['home_dir'] SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 39dce203..864d83e8 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true HOME_DIR='tests/' GLOBAL_SKALE_DIR='tests/etc/skale' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +HIDE_STREAM_LOG=true TEST_HOME_DIR='tests/' GLOBAL_SKALE_DIR='tests/etc/skale' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ From b8b6fb4bba263ad8796348736f32642885b47741 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 20 Apr 2021 17:23:52 +0300 Subject: [PATCH 074/140] Fix no alloc test --- tests/cli/node_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 6c8b830a..01602f8a 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -101,7 +101,7 @@ def test_register_node_with_default_port(resource_alloc, mocked_g_config): assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa -def test_register_with_no_alloc(resource_alloc, mocked_g_config): +def test_register_with_no_alloc(mocked_g_config): resp_mock = response_mock( requests.codes.ok, {'status': 'ok', 'payload': None} @@ -111,9 +111,9 @@ def test_register_with_no_alloc(resource_alloc, mocked_g_config): resp_mock, register_node, ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') - assert result.exit_code == 0 + assert result.exit_code == 8 print(repr(result.output)) - assert result.output == "Enter node public IP: 0.0.0.0\nNode hasn't been inited before.\nYou should run < skale node init >\n" # noqa + assert result.output == "Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn't been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n" # noqa def test_init_node(caplog): # todo: write new init node test From e8b35819bbff5e653b23cfb9df0dd55444492218 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 23 Apr 2021 17:09:36 +0300 Subject: [PATCH 075/140] SKALE-3947,SKALE-3838,SKALE-3947 Local logs command, fix global config, comment out checks --- node_cli/cli/logs.py | 8 +-- node_cli/core/logs.py | 70 +++++++++++++++++++++++++ node_cli/operations/base.py | 12 ++--- node_cli/utils/docker_utils.py | 21 +++++++- node_cli/utils/global_config.py | 6 +-- scripts/run_tests.sh | 2 +- setup.py | 3 +- tests/core/logs_test.py | 92 +++++++++++++++++++++++++++++++++ 8 files changed, 195 insertions(+), 19 deletions(-) create mode 100644 node_cli/core/logs.py create mode 100644 tests/core/logs_test.py diff --git a/node_cli/cli/logs.py b/node_cli/cli/logs.py index 88bb87b3..7472bcbf 100644 --- a/node_cli/cli/logs.py +++ b/node_cli/cli/logs.py @@ -20,7 +20,7 @@ import sys import click -from node_cli.utils.helper import download_dump +from node_cli.core.logs import create_logs_dump from node_cli.configs.cli_logger import LOG_FILEPATH, DEBUG_LOG_FILEPATH from node_cli.utils.exit_codes import CLIExitCodes @@ -52,8 +52,8 @@ def cli(debug): ) @click.argument('path') def dump(container, path): - res = download_dump(path, container) + res = create_logs_dump(path, container) if res: - print(f'File {res} downloaded') + print(f'Logs dump created: {res}') else: - sys.exit(CLIExitCodes.BAD_API_RESPONSE) + sys.exit(CLIExitCodes.OPERATION_EXECUTION_ERROR) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py new file mode 100644 index 00000000..919528cb --- /dev/null +++ b/node_cli/core/logs.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2021 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import shlex +import shutil +import datetime + +from node_cli.core.host import safe_mk_dirs +from node_cli.utils.helper import run_cmd +from node_cli.utils.docker_utils import ( + save_container_logs, get_containers +) +from node_cli.configs import REMOVED_CONTAINERS_FOLDER_PATH +from node_cli.configs.cli_logger import LOG_DATA_PATH + + +def create_logs_dump(path, filter_container=None): + dump_folder_path, dump_folder_name = create_dump_dir() + + containers_logs_path = os.path.join(dump_folder_path, 'containers') + cli_logs_path = os.path.join(dump_folder_path, 'cli') + removed_containers_logs_path = os.path.join(dump_folder_path, 'removed_containers') + archive_path = os.path.join(path, f'{dump_folder_name}.tar.gz') + + if filter_container: + containers = get_containers(filter_container) + else: + containers = get_containers('skale') + + for container in containers: + log_filepath = os.path.join(containers_logs_path, f'{container.name}.log') + save_container_logs(container, log_filepath, tail='all') + + shutil.copytree(LOG_DATA_PATH, cli_logs_path) + shutil.copytree(REMOVED_CONTAINERS_FOLDER_PATH, removed_containers_logs_path) + create_archive(archive_path, dump_folder_path) + if not os.path.isfile(archive_path): + return None + return archive_path + + +def create_dump_dir(): + time = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S") + folder_name = f'skale-logs-dump-{time}' + folder_path = os.path.join('/tmp', folder_name) + containers_path = os.path.join(folder_path, 'containers') + safe_mk_dirs(containers_path) + return folder_path, folder_name + + +def create_archive(archive_path, source_path): + cmd = shlex.split(f'tar -czvf {archive_path} -C {source_path} .') + run_cmd(cmd) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 24722978..2644524d 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -20,7 +20,7 @@ import logging from node_cli.cli.info import VERSION -from node_cli.core.host import ( +from node_cli.core.host import ( # noqa - tmp! prepare_host, link_env_file, run_preinstall_checks ) from node_cli.core.resources import update_resource_allocation @@ -35,7 +35,7 @@ from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta -from node_cli.utils.print_formatters import print_failed_requirements_checks +from node_cli.utils.print_formatters import print_failed_requirements_checks # noqa - tmp! logger = logging.getLogger(__name__) @@ -69,10 +69,10 @@ def update(env_filepath: str, env: str) -> None: def init(env_filepath: str, env: str) -> bool: sync_skale_node(env) - failed_checks = run_preinstall_checks(env['ENV_TYPE']) - if failed_checks: - print_failed_requirements_checks(failed_checks) - return False + # failed_checks = run_preinstall_checks(env['ENV_TYPE']) + # if failed_checks: + # print_failed_requirements_checks(failed_checks) + # return False prepare_host( env_filepath, env['DISK_MOUNTPOINT'], diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 475a5d68..e05bd3b2 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -50,6 +50,14 @@ def docker_client() -> DockerClient: return docker.from_env() +def get_sanitized_container_name(container_info: dict) -> str: + return container_info['Names'][0].replace('/', '', 1) + + +def get_containers(container_name_filter=None, _all=True) -> list: + return docker_client().containers.list(all=_all, filters={'name': container_name_filter}) + + def get_all_schain_containers(_all=True) -> list: return docker_client().containers.list(all=_all, filters={'name': 'skale_schain_*'}) @@ -98,11 +106,20 @@ def safe_rm(container: Container, stop_timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kw def backup_container_logs(container: Container, tail=DOCKER_DEFAULT_TAIL_LINES) -> None: logger.info(f'Going to backup container logs: {container.name}') logs_backup_filepath = get_logs_backup_filepath(container) - with open(logs_backup_filepath, "wb") as out: - out.write(container.logs(tail=tail)) + save_container_logs(container, logs_backup_filepath, tail) logger.debug(f'Old container logs saved to {logs_backup_filepath}, tail: {tail}') +def save_container_logs( + container: Container, + log_filepath: str, + tail=DOCKER_DEFAULT_TAIL_LINES + ) -> None: + with open(log_filepath, "wb") as out: + out.write(container.logs(tail=tail)) + logger.debug(f'Logs from {container.name} saved to {log_filepath}') + + def get_logs_backup_filepath(container: Container) -> str: container_index = sum(1 for f in os.listdir(REMOVED_CONTAINERS_FOLDER_PATH) if f.startswith(f'{container.name}-')) diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 4badacde..b0ae607d 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -25,7 +25,7 @@ def get_system_user() -> str: - user = os.getenv('SUDO_USER', os.getenv('USER')) + user = os.getenv('USER', os.getenv('SUDO_USER')) if not user: raise ValueError('SUDO_USER or USER env variable should be set') return user @@ -43,10 +43,6 @@ def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict """Init global SKALE config file""" logger.info('Generating global SKALE config file...') os.makedirs(g_skale_dir, exist_ok=True) - - user = os.environ['USER'] - user = os.getenv('SUDO_USER', user) - g_config = { 'user': get_system_user(), 'home_dir': os.path.expanduser('~') diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 864d83e8..9788f98f 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR='tests/' GLOBAL_SKALE_DIR='tests/etc/skale' DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +HIDE_STREAM_LOG=true TEST_HOME_DIR="$DIR/tests/" GLOBAL_SKALE_DIR="$DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index 4371c400..be5de1fb 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,8 @@ def find_version(*file_paths): "pytest==6.2.3", "pytest-cov==2.9.0", "twine==2.0.0", - "mock==4.0.2" + "mock==4.0.2", + "freezegun==0.3.15" ] } diff --git a/tests/core/logs_test.py b/tests/core/logs_test.py new file mode 100644 index 00000000..80b5a956 --- /dev/null +++ b/tests/core/logs_test.py @@ -0,0 +1,92 @@ +import os +import shlex +import shutil +from datetime import datetime + +import pytest +import freezegun + +from node_cli.core.logs import create_dump_dir, create_logs_dump +from node_cli.configs import G_CONF_HOME +from node_cli.utils.docker_utils import docker_client +from node_cli.utils.helper import run_cmd +from node_cli.core.host import safe_mk_dirs + + +CURRENT_TIMESTAMP = 1594903080 +CURRENT_DATETIME = datetime.utcfromtimestamp(CURRENT_TIMESTAMP) +TEST_DUMP_DIR_PATH = '/tmp/skale-logs-dump-2020-07-16-12:38:00' + +TEST_IMAGE = 'alpine' +TEST_SKALE_NAME = 'skale_cli_test_container' +TEST_ENTRYPOINT = 'echo Hello, SKALE!' + +TEST_ARCHIVE_FOLDER_NAME = 'skale-logs-dump-2020-07-16-12:38:00' +TEST_ARCHIVE_FOLDER_PATH = os.path.join(G_CONF_HOME, f'{TEST_ARCHIVE_FOLDER_NAME}') +TEST_ARCHIVE_PATH = os.path.join(G_CONF_HOME, f'{TEST_ARCHIVE_FOLDER_NAME}.tar.gz') + + +def _backup_cleanup(): + shutil.rmtree(TEST_DUMP_DIR_PATH, ignore_errors=True) + shutil.rmtree(TEST_ARCHIVE_FOLDER_PATH, ignore_errors=True) + if os.path.exists(TEST_ARCHIVE_PATH): + os.remove(TEST_ARCHIVE_PATH) + + +@pytest.fixture +def backup_func(): + _backup_cleanup() + yield + _backup_cleanup() + + +@pytest.fixture +def skale_container(): + client = docker_client() + container = client.containers.run( + image=TEST_IMAGE, + name=TEST_SKALE_NAME, + detach=True, + entrypoint=TEST_ENTRYPOINT + ) + yield + container.remove(force=True) + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_create_dump_dir(mocked_g_config, backup_func): + folder_path, folder_name = create_dump_dir() + assert folder_path == TEST_DUMP_DIR_PATH + assert folder_name == 'skale-logs-dump-2020-07-16-12:38:00' + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_create_logs_dump(backup_func, skale_container): + archive_path = create_logs_dump(G_CONF_HOME) + + test_container_log_path = os.path.join( + TEST_DUMP_DIR_PATH, 'containers', f'{TEST_SKALE_NAME}.log' + ) + with open(test_container_log_path) as data_file: + content = data_file.read() + assert content == 'Hello, SKALE!\n' + + safe_mk_dirs(TEST_ARCHIVE_FOLDER_PATH) + cmd = shlex.split(f'tar xf {archive_path} -C {TEST_ARCHIVE_FOLDER_PATH}') + run_cmd(cmd) + + assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'removed_containers')) + assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'cli')) + assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'containers')) + + assert os.path.isfile(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'cli', 'debug-node-cli.log')) + assert os.path.isfile(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'cli', 'node-cli.log')) + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_create_logs_dump_one_container(backup_func, skale_container): + create_logs_dump(G_CONF_HOME, filter_container='abc') + test_container_log_path = os.path.join( + TEST_DUMP_DIR_PATH, 'containers', f'{TEST_SKALE_NAME}.log' + ) + assert not os.path.isfile(test_container_log_path) From 47450ea767f4028a13dff36d5316d85ac587ccc4 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 23 Apr 2021 17:28:12 +0300 Subject: [PATCH 076/140] SKALE-3947 Fix tests path --- scripts/run_tests.sh | 2 +- tests/core/{logs_test.py => core_logs_test.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/core/{logs_test.py => core_logs_test.py} (100%) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 9788f98f..12cce17a 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR="$DIR/tests/" GLOBAL_SKALE_DIR="$DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +HIDE_STREAM_LOG=true TEST_HOME_DIR="$PROJECT_DIR/tests/" GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/tests/core/logs_test.py b/tests/core/core_logs_test.py similarity index 100% rename from tests/core/logs_test.py rename to tests/core/core_logs_test.py From 2e625f6312d8d232d885dec53c5eac262236f8f7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 23 Apr 2021 18:52:54 +0300 Subject: [PATCH 077/140] SKALE-3947 Fix all tests --- node_cli/core/logs.py | 2 +- tests/cli/logs_test.py | 32 +++++++++----------------------- tests/cli/node_test.py | 8 ++++---- tests/cli/schains_test.py | 5 +++-- tests/cli/validate_test.py | 10 +++++++--- tests/cli/wallet_test.py | 3 ++- tests/core/core_logs_test.py | 6 +++--- 7 files changed, 29 insertions(+), 37 deletions(-) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 919528cb..e1b77fce 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -57,7 +57,7 @@ def create_logs_dump(path, filter_container=None): def create_dump_dir(): - time = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S") + time = datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S") folder_name = f'skale-logs-dump-{time}' folder_path = os.path.join('/tmp', folder_name) containers_path = os.path.join(folder_path, 'containers') diff --git a/tests/cli/logs_test.py b/tests/cli/logs_test.py index 53675bbd..463cb1a8 100644 --- a/tests/cli/logs_test.py +++ b/tests/cli/logs_test.py @@ -17,31 +17,17 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import freezegun -import os - -import mock -import requests - -from io import BytesIO -from tests.helper import response_mock, run_command from node_cli.cli.logs import dump +from node_cli.configs import G_CONF_HOME +from tests.helper import run_command +from tests.core.core_logs_test import backup_func, CURRENT_DATETIME, TEST_ARCHIVE_PATH # noqa -def test_dump(): - archive_filename = 'skale-logs-dump-2019-10-08-17:40:00.tar.gz' - resp_mock = response_mock( - requests.codes.ok, - headers={ - 'Content-Disposition': f'attachment; filename="{archive_filename}"' - }, - raw=BytesIO() - ) - with mock.patch('requests.get') as req_get_mock: - req_get_mock.return_value.__enter__.return_value = resp_mock - result = run_command(dump, ['.']) - assert result.exit_code == 0 - assert result.output == f'File {archive_filename} downloaded\n' - if os.path.exists(archive_filename): - os.remove(archive_filename) +@freezegun.freeze_time(CURRENT_DATETIME) +def test_dump(backup_func): # noqa + result = run_command(dump, [G_CONF_HOME]) + assert result.exit_code == 0 + assert result.output == f'Logs dump created: {TEST_ARCHIVE_PATH}\n' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 01602f8a..dc087a4b 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -24,7 +24,7 @@ import requests import logging -from node_cli.configs import NODE_DATA_PATH, SKALE_DIR +from node_cli.configs import NODE_DATA_PATH, SKALE_DIR, G_CONF_HOME from node_cli.cli.node import (init_node, node_info, register_node, signature, update_node, backup_node, restore_node, set_node_in_maintenance, @@ -68,7 +68,7 @@ def test_register_node_with_error(resource_alloc, mocked_g_config): register_node, ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test']) assert result.exit_code == 3 - assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa def test_register_node_with_prompted_ip(resource_alloc, mocked_g_config): @@ -113,7 +113,7 @@ def test_register_with_no_alloc(mocked_g_config): ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') assert result.exit_code == 8 print(repr(result.output)) - assert result.output == "Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn't been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n" # noqa + assert result.output == f'Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn\'t been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa def test_init_node(caplog): # todo: write new init node test @@ -177,7 +177,7 @@ def test_update_node_without_init(): params, input='/dev/sdp') assert result.exit_code == 8 - assert result.output == "Command failed with following errors:\n--------------------------------------------------\nNode hasn't been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n" # noqa + assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nNode hasn\'t been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa def test_node_info_node_info(): diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 6d5d9895..5889ba29 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -22,6 +22,7 @@ import requests +from node_cli.configs import G_CONF_HOME from tests.helper import response_mock, run_command_mock from node_cli.cli.schains import (get_schain_config, ls, dkg, show_rules, repair, info_) @@ -173,7 +174,7 @@ def test_repair(): ['test-schain', '--yes']) print(repr(result.output)) assert result.exit_code == 3 - assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa def test_info(): @@ -200,5 +201,5 @@ def test_info(): ) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, info_, ['schain not found']) - assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa assert result.exit_code == 3 diff --git a/tests/cli/validate_test.py b/tests/cli/validate_test.py index 183c8be0..7a595b87 100644 --- a/tests/cli/validate_test.py +++ b/tests/cli/validate_test.py @@ -4,7 +4,7 @@ import pytest -from node_cli.configs import (CONTRACTS_PATH, +from node_cli.configs import (CONTRACTS_PATH, G_CONF_HOME, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH) from node_cli.cli.validate import abi from tests.helper import run_command @@ -53,11 +53,15 @@ def test_validate_abi(contract_valid_abi_files): def test_validate_abi_invalid_file(contract_abi_file_invalid): result = run_command(abi) - assert result.output == 'Some files do not exist or are incorrect\n Filepath Status Msg \n-----------------------------------------------------------------------------------\ntests/.skale/contracts_info/manager.json error Failed to load abi file as json\ntests/.skale/contracts_info/ima.json ok \n' # noqa + assert 'Some files do not exist or are incorrect' in result.output + assert f'{G_CONF_HOME}.skale/contracts_info/manager.json error Failed to load abi file as json' in result.output # noqa + assert f'{G_CONF_HOME}.skale/contracts_info/ima.json ok' in result.output assert result.exit_code == 0 def test_validate_abi_empty_file(contract_abi_file_empty): result = run_command(abi) - assert result.output == 'Some files do not exist or are incorrect\n Filepath Status Msg \n----------------------------------------------------------------\ntests/.skale/contracts_info/manager.json error No such file\ntests/.skale/contracts_info/ima.json ok \n' # noqa + assert 'Some files do not exist or are incorrect' in result.output + assert f'{G_CONF_HOME}.skale/contracts_info/manager.json error No such file' in result.output # noqa + assert f'{G_CONF_HOME}.skale/contracts_info/ima.json ok' in result.output assert result.exit_code == 0 diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index b803eda7..489ec07b 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -22,6 +22,7 @@ from mock import MagicMock, Mock +from node_cli.configs import G_CONF_HOME from node_cli.cli.wallet import wallet_info, send from tests.helper import run_command_mock, response_mock @@ -88,4 +89,4 @@ def test_wallet_send_with_error(): send, ['0x00000000000000000000000000000000', '10', '--yes']) assert result.exit_code == 3 - assert result.output == 'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in tests/.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa diff --git a/tests/core/core_logs_test.py b/tests/core/core_logs_test.py index 80b5a956..a6060e53 100644 --- a/tests/core/core_logs_test.py +++ b/tests/core/core_logs_test.py @@ -15,13 +15,13 @@ CURRENT_TIMESTAMP = 1594903080 CURRENT_DATETIME = datetime.utcfromtimestamp(CURRENT_TIMESTAMP) -TEST_DUMP_DIR_PATH = '/tmp/skale-logs-dump-2020-07-16-12:38:00' +TEST_DUMP_DIR_PATH = '/tmp/skale-logs-dump-2020-07-16--12-38-00' TEST_IMAGE = 'alpine' TEST_SKALE_NAME = 'skale_cli_test_container' TEST_ENTRYPOINT = 'echo Hello, SKALE!' -TEST_ARCHIVE_FOLDER_NAME = 'skale-logs-dump-2020-07-16-12:38:00' +TEST_ARCHIVE_FOLDER_NAME = 'skale-logs-dump-2020-07-16--12-38-00' TEST_ARCHIVE_FOLDER_PATH = os.path.join(G_CONF_HOME, f'{TEST_ARCHIVE_FOLDER_NAME}') TEST_ARCHIVE_PATH = os.path.join(G_CONF_HOME, f'{TEST_ARCHIVE_FOLDER_NAME}.tar.gz') @@ -57,7 +57,7 @@ def skale_container(): def test_create_dump_dir(mocked_g_config, backup_func): folder_path, folder_name = create_dump_dir() assert folder_path == TEST_DUMP_DIR_PATH - assert folder_name == 'skale-logs-dump-2020-07-16-12:38:00' + assert folder_name == 'skale-logs-dump-2020-07-16--12-38-00' @freezegun.freeze_time(CURRENT_DATETIME) From d54e219522faed07dc95db1b305fc0cb8000fe9e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 23 Apr 2021 18:59:12 +0300 Subject: [PATCH 078/140] SKALE-3947 Add removed containers to git --- tests/.skale/node_data/log/.removed_containers/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/.skale/node_data/log/.removed_containers/.keep diff --git a/tests/.skale/node_data/log/.removed_containers/.keep b/tests/.skale/node_data/log/.removed_containers/.keep new file mode 100644 index 00000000..e69de29b From f94d3b3ec57f871257c06d64b6540ff9bcc73926 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 23 Apr 2021 19:42:58 +0300 Subject: [PATCH 079/140] SKALE-3947 Remove logs dump from routes --- node_cli/configs/routes.py | 1 - tests/routes_test.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index fe73140b..19e7847e 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -25,7 +25,6 @@ ROUTES = { 'v1': { - 'logs': ['dump'], 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'signature', 'send-tg-notification', 'exit/start', 'exit/status', 'set-domain-name'], 'health': ['containers', 'schains', 'sgx'], diff --git a/tests/routes_test.py b/tests/routes_test.py index 533a968d..3cd2416e 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -4,8 +4,6 @@ ALL_V1_ROUTES = [ - '/api/v1/logs/dump', - '/api/v1/node/info', '/api/v1/node/register', '/api/v1/node/maintenance-on', @@ -36,8 +34,8 @@ def test_route_exists(): - assert route_exists('logs', 'dump', 'v1') - assert not route_exists('logshms', 'dumb', 'v1') + assert route_exists('node', 'signature', 'v1') + assert not route_exists('snode', 'mignature', 'v1') def test_get_route(): From 8858952dec9d5e0da62c3768bea6f188e662b308 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 26 Apr 2021 16:39:22 +0300 Subject: [PATCH 080/140] SKALE-3947 Add cleanup for logs cmd --- node_cli/core/logs.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index e1b77fce..4d96d342 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -18,8 +18,8 @@ # along with this program. If not, see . import os -import shlex import shutil +import logging import datetime from node_cli.core.host import safe_mk_dirs @@ -31,6 +31,9 @@ from node_cli.configs.cli_logger import LOG_DATA_PATH +logger = logging.getLogger(__name__) + + def create_logs_dump(path, filter_container=None): dump_folder_path, dump_folder_name = create_dump_dir() @@ -51,6 +54,7 @@ def create_logs_dump(path, filter_container=None): shutil.copytree(LOG_DATA_PATH, cli_logs_path) shutil.copytree(REMOVED_CONTAINERS_FOLDER_PATH, removed_containers_logs_path) create_archive(archive_path, dump_folder_path) + rm_dump_dir(dump_folder_path) if not os.path.isfile(archive_path): return None return archive_path @@ -61,10 +65,15 @@ def create_dump_dir(): folder_name = f'skale-logs-dump-{time}' folder_path = os.path.join('/tmp', folder_name) containers_path = os.path.join(folder_path, 'containers') + logging.debug(f'Creating tmp dir for logs dump: {folder_path}') safe_mk_dirs(containers_path) return folder_path, folder_name +def rm_dump_dir(dump_folder_path: str) -> None: + logger.debug(f'Going to remove tmp dir with logs dump: {dump_folder_path}') + shutil.rmtree(dump_folder_path) + + def create_archive(archive_path, source_path): - cmd = shlex.split(f'tar -czvf {archive_path} -C {source_path} .') - run_cmd(cmd) + run_cmd(['tar', '-czvf', archive_path, '-C', source_path, '.']) From dec8546937f4305e861ac545149e06513b779963 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 26 Apr 2021 18:15:51 +0300 Subject: [PATCH 081/140] SKALE-3947 Change logs dump tmp dir --- node_cli/configs/__init__.py | 1 + node_cli/core/host.py | 6 ++++-- node_cli/core/logs.py | 4 ++-- tests/.skale/.tmp/.keep | 0 tests/core/core_logs_test.py | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 tests/.skale/.tmp/.keep diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 3ff124b0..eae13d15 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -37,6 +37,7 @@ CONTRACTS_PATH = os.path.join(SKALE_DIR, 'contracts_info') BACKUP_CONTRACTS_PATH = os.path.join(SKALE_DIR, '.old_contracts_info') INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') +SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 29abecd7..0112a17d 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -35,7 +35,8 @@ ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, - IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH + IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, + SKALE_TMP_DIR ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -105,7 +106,8 @@ def make_dirs(): SKALE_DIR, NODE_DATA_PATH, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, - SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH + SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, + SKALE_TMP_DIR ): safe_mk_dirs(dir_path) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 4d96d342..34a240df 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -27,7 +27,7 @@ from node_cli.utils.docker_utils import ( save_container_logs, get_containers ) -from node_cli.configs import REMOVED_CONTAINERS_FOLDER_PATH +from node_cli.configs import REMOVED_CONTAINERS_FOLDER_PATH, SKALE_TMP_DIR from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -63,7 +63,7 @@ def create_logs_dump(path, filter_container=None): def create_dump_dir(): time = datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S") folder_name = f'skale-logs-dump-{time}' - folder_path = os.path.join('/tmp', folder_name) + folder_path = os.path.join(SKALE_TMP_DIR, folder_name) containers_path = os.path.join(folder_path, 'containers') logging.debug(f'Creating tmp dir for logs dump: {folder_path}') safe_mk_dirs(containers_path) diff --git a/tests/.skale/.tmp/.keep b/tests/.skale/.tmp/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/core/core_logs_test.py b/tests/core/core_logs_test.py index a6060e53..ca6fc285 100644 --- a/tests/core/core_logs_test.py +++ b/tests/core/core_logs_test.py @@ -7,7 +7,7 @@ import freezegun from node_cli.core.logs import create_dump_dir, create_logs_dump -from node_cli.configs import G_CONF_HOME +from node_cli.configs import G_CONF_HOME, SKALE_TMP_DIR from node_cli.utils.docker_utils import docker_client from node_cli.utils.helper import run_cmd from node_cli.core.host import safe_mk_dirs @@ -15,7 +15,7 @@ CURRENT_TIMESTAMP = 1594903080 CURRENT_DATETIME = datetime.utcfromtimestamp(CURRENT_TIMESTAMP) -TEST_DUMP_DIR_PATH = '/tmp/skale-logs-dump-2020-07-16--12-38-00' +TEST_DUMP_DIR_PATH = os.path.join(SKALE_TMP_DIR, 'skale-logs-dump-2020-07-16--12-38-00') TEST_IMAGE = 'alpine' TEST_SKALE_NAME = 'skale_cli_test_container' From f498b02d0edeb9f640101778e1b78f40f3a4b43e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 26 Apr 2021 18:45:22 +0300 Subject: [PATCH 082/140] SKALE-3947 Fix tests --- tests/core/core_logs_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/core/core_logs_test.py b/tests/core/core_logs_test.py index ca6fc285..d2fbd122 100644 --- a/tests/core/core_logs_test.py +++ b/tests/core/core_logs_test.py @@ -63,18 +63,17 @@ def test_create_dump_dir(mocked_g_config, backup_func): @freezegun.freeze_time(CURRENT_DATETIME) def test_create_logs_dump(backup_func, skale_container): archive_path = create_logs_dump(G_CONF_HOME) + safe_mk_dirs(TEST_ARCHIVE_FOLDER_PATH) + cmd = shlex.split(f'tar xf {archive_path} -C {TEST_ARCHIVE_FOLDER_PATH}') + run_cmd(cmd) test_container_log_path = os.path.join( - TEST_DUMP_DIR_PATH, 'containers', f'{TEST_SKALE_NAME}.log' + TEST_ARCHIVE_FOLDER_PATH, 'containers', f'{TEST_SKALE_NAME}.log' ) with open(test_container_log_path) as data_file: content = data_file.read() assert content == 'Hello, SKALE!\n' - safe_mk_dirs(TEST_ARCHIVE_FOLDER_PATH) - cmd = shlex.split(f'tar xf {archive_path} -C {TEST_ARCHIVE_FOLDER_PATH}') - run_cmd(cmd) - assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'removed_containers')) assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'cli')) assert os.path.exists(os.path.join(TEST_ARCHIVE_FOLDER_PATH, 'containers')) From 9937644d31abadc2a5cdaefba987fd3709d0e254 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 27 Apr 2021 15:20:18 +0300 Subject: [PATCH 083/140] SKALE-3947 Fix wrong user from env --- node_cli/utils/global_config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index b0ae607d..1f5c2ccf 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import os +import sys import json import logging @@ -25,7 +26,7 @@ def get_system_user() -> str: - user = os.getenv('USER', os.getenv('SUDO_USER')) + user = os.getenv('SUDO_USER', os.getenv('USER')) if not user: raise ValueError('SUDO_USER or USER env variable should be set') return user @@ -41,13 +42,18 @@ def read_g_config(g_skale_dir: str, g_skale_conf_filepath: str) -> dict: def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict: """Init global SKALE config file""" - logger.info('Generating global SKALE config file...') + print('Generating global SKALE config file...') os.makedirs(g_skale_dir, exist_ok=True) g_config = { 'user': get_system_user(), 'home_dir': os.path.expanduser('~') } - logger.info(f'{g_skale_conf_filepath} content: {g_config}') - with open(g_skale_conf_filepath, 'w') as outfile: - json.dump(g_config, outfile, indent=4) + print(f'{g_skale_conf_filepath} content: {g_config}') + try: + with open(g_skale_conf_filepath, 'w') as outfile: + json.dump(g_config, outfile, indent=4) + except PermissionError as e: + logger.exception(e) + print('No permissions to write into /etc directory') + sys.exit(7) return g_config From eea9a225e609c31cdfc92a7cd76f73f0c18931e8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 27 Apr 2021 16:37:04 +0300 Subject: [PATCH 084/140] SKALE-3947 Fix get system user cmd --- node_cli/utils/global_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 1f5c2ccf..339760e7 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -19,6 +19,7 @@ import os import sys +import pwd import json import logging @@ -26,10 +27,11 @@ def get_system_user() -> str: - user = os.getenv('SUDO_USER', os.getenv('USER')) - if not user: - raise ValueError('SUDO_USER or USER env variable should be set') - return user + return pwd.getpwuid(os.getuid())[0] + + +def get_home_dir() -> str: + return os.path.expanduser('~') def read_g_config(g_skale_dir: str, g_skale_conf_filepath: str) -> dict: @@ -46,7 +48,7 @@ def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict os.makedirs(g_skale_dir, exist_ok=True) g_config = { 'user': get_system_user(), - 'home_dir': os.path.expanduser('~') + 'home_dir': get_home_dir() } print(f'{g_skale_conf_filepath} content: {g_config}') try: From 20b8485c1f26b4e2df778dc2be9d5dfc0858e8b6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 27 Apr 2021 16:48:34 +0300 Subject: [PATCH 085/140] SKALE-3947 Fix tests --- node_cli/utils/helper.py | 6 ++---- tests/utils/global_config_test.py | 13 ++++--------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 42b19ae8..56d4c596 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -20,6 +20,7 @@ import os import re import sys +import pwd import json import yaml import shutil @@ -322,10 +323,7 @@ def wrapper(*args, **kwargs): def get_system_user() -> str: - user = os.getenv('SUDO_USER', os.getenv('USER')) - if not user: - raise ValueError('SUDO_USER or USER env variable should be set') - return user + return pwd.getpwuid(os.getuid())[0] def is_user_valid(allow_root=True): diff --git a/tests/utils/global_config_test.py b/tests/utils/global_config_test.py index 6d118791..1cdd8001 100644 --- a/tests/utils/global_config_test.py +++ b/tests/utils/global_config_test.py @@ -1,5 +1,6 @@ import os +import mock from node_cli.utils.global_config import read_g_config, generate_g_config_file from node_cli.utils.helper import write_json, get_system_user, is_user_valid, get_g_conf_user from node_cli.configs import GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH @@ -43,15 +44,9 @@ def test_is_user_valid(mocked_g_config): write_json(GLOBAL_SKALE_CONF_FILEPATH, {'user': 'skaletest'}) assert not is_user_valid() - sudo_user = os.environ.get('SUDO_USER') - os.environ['SUDO_USER'] = 'root' - assert is_user_valid() - assert not is_user_valid(allow_root=False) - - if sudo_user: - os.environ['SUDO_USER'] = sudo_user - else: - del os.environ['SUDO_USER'] + with mock.patch('os.getuid', return_value=0): + assert is_user_valid() + assert not is_user_valid(allow_root=False) def test_get_g_conf_user(mocked_g_config): From c808b64c0924676b9fe94788a195c67de0c62666 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 27 Apr 2021 17:07:54 +0300 Subject: [PATCH 086/140] SKALE-3947 Fix tests --- tests/utils/global_config_test.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/utils/global_config_test.py b/tests/utils/global_config_test.py index 1cdd8001..58f73665 100644 --- a/tests/utils/global_config_test.py +++ b/tests/utils/global_config_test.py @@ -28,13 +28,10 @@ def test_generate_g_config_file(mocked_g_config): def test_get_system_user(): - sudo_user = os.environ.get('SUDO_USER') - if sudo_user: - del os.environ['SUDO_USER'] - os.environ['USER'] = 'test' - assert get_system_user() == 'test' - if sudo_user: - os.environ['SUDO_USER'] = sudo_user + with mock.patch('os.getuid', return_value=0): + assert get_system_user() == 'root' + with mock.patch('pwd.getpwuid', return_value=['test']): + assert get_system_user() == 'test' def test_is_user_valid(mocked_g_config): From 3df3507bc1a7faab95152bc41a77d1203c0460d4 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 28 Apr 2021 15:06:37 +0300 Subject: [PATCH 087/140] SKALE-3947 Change get system user logic --- node_cli/utils/decorators.py | 3 ++- node_cli/utils/global_config.py | 3 +-- node_cli/utils/helper.py | 7 +------ tests/utils/global_config_test.py | 13 +++++++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/node_cli/utils/decorators.py b/node_cli/utils/decorators.py index a79e99bf..95c822e8 100644 --- a/node_cli/utils/decorators.py +++ b/node_cli/utils/decorators.py @@ -20,7 +20,8 @@ from functools import wraps from node_cli.core.host import is_node_inited -from node_cli.utils.helper import error_exit, is_user_valid, get_g_conf_user, get_system_user +from node_cli.utils.global_config import get_system_user +from node_cli.utils.helper import error_exit, is_user_valid, get_g_conf_user from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 339760e7..4a1c4eaa 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -19,7 +19,6 @@ import os import sys -import pwd import json import logging @@ -27,7 +26,7 @@ def get_system_user() -> str: - return pwd.getpwuid(os.getuid())[0] + return 'root' if get_home_dir() == '/root' else os.getenv('SUDO_USER', os.getenv('USER')) def get_home_dir() -> str: diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 56d4c596..b96c7926 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -20,7 +20,6 @@ import os import re import sys -import pwd import json import yaml import shutil @@ -55,7 +54,7 @@ LOG_FILEPATH, DEBUG_LOG_FILEPATH, FILE_LOG_FORMAT ) from node_cli.configs.routes import get_route -from node_cli.utils.global_config import read_g_config +from node_cli.utils.global_config import read_g_config, get_system_user logger = logging.getLogger(__name__) @@ -322,10 +321,6 @@ def wrapper(*args, **kwargs): return wrapper -def get_system_user() -> str: - return pwd.getpwuid(os.getuid())[0] - - def is_user_valid(allow_root=True): current_user = get_system_user() if current_user == 'root' and allow_root: diff --git a/tests/utils/global_config_test.py b/tests/utils/global_config_test.py index 58f73665..e159d51f 100644 --- a/tests/utils/global_config_test.py +++ b/tests/utils/global_config_test.py @@ -28,10 +28,15 @@ def test_generate_g_config_file(mocked_g_config): def test_get_system_user(): - with mock.patch('os.getuid', return_value=0): + with mock.patch('os.path.expanduser', return_value='/root'): assert get_system_user() == 'root' - with mock.patch('pwd.getpwuid', return_value=['test']): - assert get_system_user() == 'test' + sudo_user = os.environ.get('SUDO_USER') + if sudo_user: + del os.environ['SUDO_USER'] + os.environ['USER'] = 'test' + assert get_system_user() == 'test' + if sudo_user: + os.environ['SUDO_USER'] = sudo_user def test_is_user_valid(mocked_g_config): @@ -41,7 +46,7 @@ def test_is_user_valid(mocked_g_config): write_json(GLOBAL_SKALE_CONF_FILEPATH, {'user': 'skaletest'}) assert not is_user_valid() - with mock.patch('os.getuid', return_value=0): + with mock.patch('os.path.expanduser', return_value='/root'): assert is_user_valid() assert not is_user_valid(allow_root=False) From d2c571a0d946fbc681564cc7332eaffe172d7679 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 6 May 2021 14:11:11 +0300 Subject: [PATCH 088/140] SKALE-4103 Add filestorage dir --- node_cli/configs/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index eae13d15..2e6d9699 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -30,6 +30,9 @@ G_CONF_USER = GLOBAL_CONFIG['user'] G_CONF_HOME = os.getenv('TEST_HOME_DIR') or GLOBAL_CONFIG['home_dir'] +SKALE_DATA_DIR = '/var/lib/skale' +FILESTORAGE_DIR = os.path.join(SKALE_DATA_DIR, 'filestorage') + SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data') From 9b09b56fee54db8c6bf1781df2045d3e369b41c0 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 7 May 2021 11:12:43 +0300 Subject: [PATCH 089/140] SKALE-4103 Create mapping directory --- node_cli/configs/__init__.py | 4 ++-- node_cli/core/host.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 2e6d9699..660ca634 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -30,8 +30,8 @@ G_CONF_USER = GLOBAL_CONFIG['user'] G_CONF_HOME = os.getenv('TEST_HOME_DIR') or GLOBAL_CONFIG['home_dir'] -SKALE_DATA_DIR = '/var/lib/skale' -FILESTORAGE_DIR = os.path.join(SKALE_DATA_DIR, 'filestorage') +SKALE_STATE_DIR = '/var/lib/skale' +FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 0112a17d..e262ac4a 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,7 +36,7 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_TMP_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -107,7 +107,7 @@ def make_dirs(): CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, - SKALE_TMP_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING ): safe_mk_dirs(dir_path) From 08ec20ef562d34d4f8517d0f9b9a8b6cd34d73d5 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 11 May 2021 20:49:04 +0300 Subject: [PATCH 090/140] SKALE-4103 Pass FILESTORAGE_MAPPING to .env --- node_cli/operations/docker_lvmpy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index a15df8de..bedb645d 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -22,7 +22,7 @@ from node_cli.utils.helper import run_cmd from node_cli.utils.git_utils import sync_repo -from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL +from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING logger = logging.getLogger(__name__) @@ -30,6 +30,7 @@ def update_docker_lvmpy_env(env): env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] env['VOLUME_GROUP'] = 'schains' + env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING env['PATH'] = os.environ.get('PATH', None) return env From f5659eb150a5337622e7aa918a9c5be85ea837c5 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Wed, 12 May 2021 18:46:32 +0300 Subject: [PATCH 091/140] SKALE-4103 Add SCHAINS_MNT_DIR --- node_cli/configs/__init__.py | 1 + node_cli/operations/docker_lvmpy.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 660ca634..5ec0e775 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -32,6 +32,7 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') +SCHAINS_MNT_DIR = '/mnt' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index bedb645d..5dadde09 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -22,7 +22,7 @@ from node_cli.utils.helper import run_cmd from node_cli.utils.git_utils import sync_repo -from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING +from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, SCHAINS_MNT_DIR logger = logging.getLogger(__name__) @@ -31,6 +31,7 @@ def update_docker_lvmpy_env(env): env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING + env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR env['PATH'] = os.environ.get('PATH', None) return env From aa67ea75f94fe63e829dc958ae397a109a051674 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 13 May 2021 14:33:05 +0300 Subject: [PATCH 092/140] SKALE-4103 Fix flake, fix logger --- node_cli/core/logs.py | 2 +- node_cli/operations/docker_lvmpy.py | 9 +++++---- tests/cli/logs_test.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 34a240df..d76a2207 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -65,7 +65,7 @@ def create_dump_dir(): folder_name = f'skale-logs-dump-{time}' folder_path = os.path.join(SKALE_TMP_DIR, folder_name) containers_path = os.path.join(folder_path, 'containers') - logging.debug(f'Creating tmp dir for logs dump: {folder_path}') + logger.debug(f'Creating tmp dir for logs dump: {folder_path}') safe_mk_dirs(containers_path) return folder_path, folder_name diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 5dadde09..64168257 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -22,7 +22,8 @@ from node_cli.utils.helper import run_cmd from node_cli.utils.git_utils import sync_repo -from node_cli.configs import DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, SCHAINS_MNT_DIR +from node_cli.configs import (DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, SCHAINS_MNT_DIR) logger = logging.getLogger(__name__) @@ -42,13 +43,13 @@ def sync_docker_lvmpy_repo(env): def docker_lvmpy_update(env): sync_docker_lvmpy_repo(env) - logging.info('Running docker-lvmpy update script') + logger.info('Running docker-lvmpy update script') update_docker_lvmpy_env(env) run_cmd( cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(), env=env ) - logging.info('docker-lvmpy update done') + logger.info('docker-lvmpy update done') def docker_lvmpy_install(env): @@ -58,4 +59,4 @@ def docker_lvmpy_install(env): cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), env=env ) - logging.info('docker-lvmpy installed') + logger.info('docker-lvmpy installed') diff --git a/tests/cli/logs_test.py b/tests/cli/logs_test.py index 463cb1a8..df08d33b 100644 --- a/tests/cli/logs_test.py +++ b/tests/cli/logs_test.py @@ -26,8 +26,8 @@ from tests.core.core_logs_test import backup_func, CURRENT_DATETIME, TEST_ARCHIVE_PATH # noqa -@freezegun.freeze_time(CURRENT_DATETIME) -def test_dump(backup_func): # noqa +@freezegun.freeze_time(CURRENT_DATETIME) # noqa +def test_dump(backup_func): result = run_command(dump, [G_CONF_HOME]) assert result.exit_code == 0 assert result.output == f'Logs dump created: {TEST_ARCHIVE_PATH}\n' From c4ed42ab46130e48a09fdaf8f6b31a85a7bc9f02 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 13 May 2021 14:35:28 +0300 Subject: [PATCH 093/140] SKALE-4103 Fix flake --- tests/cli/logs_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cli/logs_test.py b/tests/cli/logs_test.py index df08d33b..3ba87431 100644 --- a/tests/cli/logs_test.py +++ b/tests/cli/logs_test.py @@ -26,8 +26,8 @@ from tests.core.core_logs_test import backup_func, CURRENT_DATETIME, TEST_ARCHIVE_PATH # noqa -@freezegun.freeze_time(CURRENT_DATETIME) # noqa -def test_dump(backup_func): +@freezegun.freeze_time(CURRENT_DATETIME) +def test_dump(backup_func): # noqa result = run_command(dump, [G_CONF_HOME]) assert result.exit_code == 0 assert result.output == f'Logs dump created: {TEST_ARCHIVE_PATH}\n' From e9a0db9e117e759b93cf2cfa21e34f7d58dd2d6e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 14 May 2021 17:06:07 +0300 Subject: [PATCH 094/140] SKALE-4103 Add sleep after test container run --- tests/core/core_logs_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/core_logs_test.py b/tests/core/core_logs_test.py index d2fbd122..fd1a63d5 100644 --- a/tests/core/core_logs_test.py +++ b/tests/core/core_logs_test.py @@ -1,4 +1,5 @@ import os +import time import shlex import shutil from datetime import datetime @@ -49,6 +50,7 @@ def skale_container(): detach=True, entrypoint=TEST_ENTRYPOINT ) + time.sleep(10) yield container.remove(force=True) From b6a90e8e22a0981bb6bd1d70daad63e978bf3153 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 15:39:47 +0300 Subject: [PATCH 095/140] SKALE-4198 Drop backuping/restoring mysql database --- node_cli/cli/node.py | 5 ++--- node_cli/configs/__init__.py | 6 ------ node_cli/core/host.py | 4 ++-- node_cli/core/node.py | 11 +---------- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index f18d92e8..34cbc425 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -163,9 +163,8 @@ def signature(validator_id): @click.option('--no-database', is_flag=True, help="Skip mysql backup") @streamed_cmd -def backup_node(backup_folder_path, env_file, no_database): - backup_mysql = True if not no_database else False - backup(backup_folder_path, env_file, backup_mysql) +def backup_node(backup_folder_path): + backup(backup_folder_path) @node.command('restore', help="Restore SKALE node on another machine") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 5ec0e775..e7cd7d81 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -111,12 +111,6 @@ def _get_env(): DEFAULT_NODE_BASE_PORT = 10000 BACKUP_ARCHIVE_NAME = 'skale-node-backup' -MYSQL_BACKUP_FILE_NAME = 'backup.sql' -MYSQL_BACKUP_FOLDER = os.path.join(SKALE_DIR, NODE_DATA_PATH, '.mysql-backup') -MYSQL_BACKUP_CONTAINER_FOLDER = '/mysql-backup' -MYSQL_BACKUP_PATH = os.path.join(MYSQL_BACKUP_FOLDER, MYSQL_BACKUP_FILE_NAME) -MYSQL_BACKUP_CONTAINER_PATH = os.path.join(MYSQL_BACKUP_CONTAINER_FOLDER, - MYSQL_BACKUP_FILE_NAME) TM_INIT_TIMEOUT = 20 RESTORE_SLEEP_TIMEOUT = 20 diff --git a/node_cli/core/host.py b/node_cli/core/host.py index e262ac4a..86633855 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -34,7 +34,7 @@ SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, - MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, + REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, SKALE_TMP_DIR, FILESTORAGE_MAPPING ) @@ -105,7 +105,7 @@ def make_dirs(): for dir_path in ( SKALE_DIR, NODE_DATA_PATH, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, - MYSQL_BACKUP_FOLDER, REMOVED_CONTAINERS_FOLDER_PATH, + REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, SKALE_TMP_DIR, FILESTORAGE_MAPPING ): diff --git a/node_cli/core/node.py b/node_cli/core/node.py index e96a939e..4b7ad400 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -34,7 +34,6 @@ from node_cli.configs.cli_logger import LOG_DIRNAME from node_cli.core.iptables import configure_iptables -from node_cli.core.mysql_backup import create_mysql_backup, restore_mysql_backup from node_cli.core.host import ( is_node_inited, save_env_params, get_flask_secret_key, run_preinstall_checks @@ -144,9 +143,6 @@ def restore(backup_path, env_filepath): env['BACKUP_RUN'] = 'True' # should be str restore_op(env, backup_path) time.sleep(RESTORE_SLEEP_TIMEOUT) - if not restore_mysql_backup(env_filepath): - print('WARNING: MySQL data restoring failed. ' - 'Check < skale logs cli > for more information') logger.info('Generating resource allocation file ...') update_resource_allocation(env['ENV_TYPE']) print('Node is restored from backup') @@ -199,12 +195,7 @@ def get_node_signature(validator_id): return payload -def backup(path, env_filepath, mysql_backup=True): - if mysql_backup: - if not create_mysql_backup(env_filepath): - print('Something went wrong while trying to create MySQL backup, ' - 'check out < skale logs cli > output') - return +def backup(path): backup_filepath = get_backup_filepath(path) create_backup_archive(backup_filepath) From ee4a9c0a798e99c534a02a2fc9d95ec5ec1eaed4 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 15:46:13 +0300 Subject: [PATCH 096/140] SKALE-4198 Remove backup parameters --- node_cli/cli/node.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 34cbc425..a3d94f20 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -159,9 +159,6 @@ def signature(validator_id): @node.command('backup', help="Generate backup file to restore SKALE node on another machine") @click.argument('backup_folder_path') -@click.argument('env_file') -@click.option('--no-database', is_flag=True, - help="Skip mysql backup") @streamed_cmd def backup_node(backup_folder_path): backup(backup_folder_path) From 45c99e9d59267a9d76099af98e80fd377d6c5f09 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 16:35:55 +0300 Subject: [PATCH 097/140] SKALE-4198 Delete mysql_backup.py file --- node_cli/core/mysql_backup.py | 57 ----------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 node_cli/core/mysql_backup.py diff --git a/node_cli/core/mysql_backup.py b/node_cli/core/mysql_backup.py deleted file mode 100644 index 80db564b..00000000 --- a/node_cli/core/mysql_backup.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import subprocess -import shlex - -from node_cli.configs import MYSQL_BACKUP_CONTAINER_PATH, MYSQL_BACKUP_PATH -from node_cli.utils.helper import run_cmd, extract_env_params - - -logger = logging.getLogger(__name__) - - -def run_mysql_cmd(cmd, env_filepath): - mysql_creds_str = mysql_creds_for_cmd(env_filepath) - cmd_str = f'docker exec -t skale_mysql bash -c "{cmd} {mysql_creds_str}"' - cmd = shlex.split(cmd_str) - return run_cmd(cmd, secure=True) - - -def mysql_creds_for_cmd(env_filepath: str) -> str: - """Returns string with user and password flags for MySQL CLI. - - :param env_filepath: Path to the environment params file - :type address: str - :returns: Formatted string - :rtype: str - """ - env_params = extract_env_params(env_filepath) - return f'-u \'{env_params["DB_USER"]}\' -p\'{env_params["DB_PASSWORD"]}\'' - - -def create_mysql_backup(env_filepath: str) -> bool: - try: - print('Creating MySQL backup...') - run_mysql_cmd( - f'mysqldump --all-databases --single-transaction --no-tablespaces ' - f'--quick --lock-tables=false > {MYSQL_BACKUP_CONTAINER_PATH}', - env_filepath - ) - print(f'MySQL backup successfully created: {MYSQL_BACKUP_PATH}') - return True - except subprocess.CalledProcessError as e: - logger.error(e) - return False - - -def restore_mysql_backup(env_filepath: str) -> bool: - try: - print('Restoring MySQL from backup...') - run_mysql_cmd( - f'mysql < {MYSQL_BACKUP_CONTAINER_PATH}', - env_filepath - ) - print(f'MySQL DB was successfully restored from backup: {MYSQL_BACKUP_PATH}') - return True - except subprocess.CalledProcessError: - logger.exception('MySQL restore command failed') - return False From 741eb00c48b8f4e89555c93fea9a9d2175edd51d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 16:37:25 +0300 Subject: [PATCH 098/140] SKALE-4198 Update tests --- tests/cli/node_test.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index dc087a4b..f42d92e0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -339,16 +339,12 @@ def test_node_signature(): def test_backup(): Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - with mock.patch('node_cli.core.mysql_backup.run_mysql_cmd'): - result = run_command( - backup_node, - [ - '/tmp', - './tests/test-env' - ] - ) - assert result.exit_code == 0 - assert 'Backup archive successfully created: /tmp/skale-node-backup-' in result.output + result = run_command( + backup_node, + ['/tmp'] + ) + assert result.exit_code == 0 + assert 'Backup archive successfully created: /tmp/skale-node-backup-' in result.output def test_restore(mocked_g_config): @@ -361,7 +357,6 @@ def test_restore(mocked_g_config): 'Backup archive successfully created: ', '').replace('\n', '') with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.restore_op'), \ - mock.patch('node_cli.core.node.restore_mysql_backup'), \ mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( From c1853ad6ae5efa19dc0e531a3c7d006ec3503956 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 16:47:11 +0300 Subject: [PATCH 099/140] SKALE-4198 Remove mysql from base containers --- node_cli/utils/docker_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index e05bd3b2..fc317731 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -37,7 +37,7 @@ IMA_REMOVE_TIMEOUT = 20 MAIN_COMPOSE_CONTAINERS = 'skale-api bounty skale-admin' -BASE_COMPOSE_SERVICES = 'transaction-manager skale-admin skale-api mysql bounty nginx watchdog filebeat' # noqa +BASE_COMPOSE_SERVICES = 'transaction-manager skale-admin skale-api bounty nginx watchdog filebeat' # noqa MONITORING_COMPOSE_SERVICES = 'node-exporter advisor' NOTIFICATION_COMPOSE_SERVICES = 'celery redis' COMPOSE_TIMEOUT = 10 From 6ed8f351e5a8d397c95f6a68b9dd0affe6d49fd4 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 17:04:45 +0300 Subject: [PATCH 100/140] SKALE-4198 Clean mysql remains --- node_cli/configs/env.py | 4 ---- node_cli/utils/helper.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/node_cli/configs/env.py b/node_cli/configs/env.py index 8b311db5..91215c52 100644 --- a/node_cli/configs/env.py +++ b/node_cli/configs/env.py @@ -14,10 +14,6 @@ 'IMA_ENDPOINT': '', 'CONTAINER_CONFIGS_STREAM': '', 'ENDPOINT': '', - 'DB_USER': 'root', - 'DB_PASSWORD': '', - 'DB_ROOT_PASSWORD': '', - 'DB_PORT': '3306', 'MANAGER_CONTRACTS_ABI_URL': '', 'IMA_CONTRACTS_ABI_URL': '', 'FILEBEAT_HOST': '', diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index b96c7926..41262f69 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -131,8 +131,6 @@ def get_username(): def extract_env_params(env_filepath): env_params = get_env_params(env_filepath) - if not env_params.get('DB_ROOT_PASSWORD'): - env_params['DB_ROOT_PASSWORD'] = env_params['DB_PASSWORD'] absent_params = ', '.join(absent_env_params(env_params)) if absent_params: From f46c66cbc407e80bc3e25f5bb7a4523667a6ee1d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 May 2021 17:32:34 +0300 Subject: [PATCH 101/140] SKALE-4198 Update readme --- README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 0207ba55..f36077e6 100644 --- a/README.md +++ b/README.md @@ -117,10 +117,7 @@ You should specify the following environment variables: - `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses - `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses - `FILEBEAT_URL` - URL to the Filebeat log server -- `DB_USER`' - MySQL user for local node database -- `DB_PASSWORD` - Password for root user of node internal database - (equal to user password by default) -- `DB_PORT` - Port for node internal database (default is `3306`) + Optional variables: @@ -152,12 +149,7 @@ skale node backup [BACKUP_FOLDER_PATH] [ENV_FILE] Arguments: - `BACKUP_FOLDER_PATH` - path to the folder where the backup file will be saved -- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command) -` - -Optional arguments: -- `--no-database` - skip mysql database backup (in case if mysql container is not started) #### Node Registration From 989affe3968e54a92d1e37e263b34ab1ad3f4045 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 25 May 2021 19:06:27 +0300 Subject: [PATCH 102/140] SKALE-4212 Add 80 port to iptables --- node_cli/core/iptables.py | 1 + 1 file changed, 1 insertion(+) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index cebd3c00..f41bd9a0 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -37,6 +37,7 @@ ALLOWED_INCOMING_TCP_PORTS = [ + '80', # filestorage '22', # ssh '8080', # http '443', # https From 4bd9d26c5c2c8ac3c71bec9e87ff88415ca4f88d Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Wed, 26 May 2021 15:54:48 +0300 Subject: [PATCH 103/140] SKALE-4212 Insert base ports, configure firewall with update --- node_cli/core/iptables.py | 9 ++++++--- node_cli/core/node.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index f41bd9a0..76ba81d0 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -146,7 +146,7 @@ def accept_incoming(chain, port, protocol) -> None: t = iptc.Target(rule, 'ACCEPT') rule.target = t rule.add_match(match) - ensure_rule(chain, rule) + ensure_rule(chain, rule, insert=True) def accept_icmp(chain: iptc.Chain) -> None: @@ -166,10 +166,13 @@ def add_icmp_rule(chain: iptc.Chain, icmp_type: str) -> None: ensure_rule(chain, rule) -def ensure_rule(chain: iptc.Chain, rule: iptc.Rule) -> None: +def ensure_rule(chain: iptc.Chain, rule: iptc.Rule, insert=False) -> None: if rule not in chain.rules: logger.debug(f'Adding rule: {rule_to_dict(rule)}, chain: {chain.name}') - chain.append_rule(rule) + if insert: + chain.insert_rule(rule) + else: + chain.append_rule(rule) else: logger.debug(f'Rule already present: {rule_to_dict(rule)}, chain: {chain.name}') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index e96a939e..9439ca42 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -176,6 +176,7 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None): @check_user def update(env_filepath): logger.info('Node update started') + configure_firewall_rules() env = get_node_env(env_filepath, inited_node=True, sync_schains=False) update_op(env_filepath, env) logger.info('Waiting for transaction manager initialization') From 057702151d7f34d020915ef040d411a5627c33e0 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 27 May 2021 14:46:40 +0300 Subject: [PATCH 104/140] SKALE-4212 Generate nginx conf from template --- node_cli/configs/__init__.py | 2 ++ node_cli/core/nginx.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 node_cli/core/nginx.py diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 5ec0e775..6e7ead9f 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -49,6 +49,8 @@ FILESTORAGE_INFO_FILE = os.path.join(CONTAINER_CONFIG_PATH, 'filestorage_info.json') FILESTORAGE_ARTIFACTS_FILE = os.path.join(NODE_DATA_PATH, 'filestorage_artifacts.json') ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') +NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') +NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py new file mode 100644 index 00000000..d3d9b60f --- /dev/null +++ b/node_cli/core/nginx.py @@ -0,0 +1,26 @@ +import logging +import os.path + +from node_cli.configs import NODE_CERTS_PATH, NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH +from node_cli.utils.helper import process_template + + +logger = logging.getLogger(__name__) + +SSL_KEY_NAME = 'ssl_key' +SSL_CRT_NAME = 'ssl_cert' + + +def generate_nginx_config(): + ssl_on = check_ssl_certs() + template_data = { + 'ssl': ssl_on, + } + logger.info(f'Processing nginx template. ssl: {ssl_on}') + process_template(NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, template_data) + + +def check_ssl_certs(): + crt_path = os.path.join(NODE_CERTS_PATH, SSL_CRT_NAME) + key_path = os.path.join(NODE_CERTS_PATH, SSL_KEY_NAME) + return os.path.exists(crt_path) and os.path.exists(key_path) From 8b23caa70b41c93995b4fe0a7b739db51c348457 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 27 May 2021 17:51:09 +0300 Subject: [PATCH 105/140] SKALE-4205 Update leveldb config structure --- node_cli/core/resources.py | 9 +- tests/.skale/config/environment_params.yaml | 14 +- tests/.skale/config/schain_allocation.yml | 170 +++++++++++--------- tests/resources_test.py | 29 ++-- 4 files changed, 118 insertions(+), 104 deletions(-) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index e1d72d37..df5f93d2 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -75,8 +75,8 @@ def compose_resource_allocation_config(env_type): 'cpu_shares': schain_cpu_alloc.dict(), 'mem': schain_mem_alloc.dict(), 'disk': schain_allocation_data[env_type]['disk'], - 'volume_limits': schain_allocation_data[env_type]['volume'], - 'storage_limit': compose_storage_limit(schain_allocation_data[env_type]['leveldb']) + 'volume_limits': schain_allocation_data[env_type]['volume_limits'], + 'leveldb_limits': schain_allocation_data[env_type]['leveldb_limits'] }, 'ima': { 'cpu_shares': ima_cpu_alloc.dict(), @@ -186,8 +186,3 @@ def get_allocation_option_name(schain): def get_disk_path(): with open(DISK_MOUNTPOINT_FILEPATH) as f: return f.read().strip() - - -def compose_storage_limit(leveldb_lim): - """Helper function was the backward compatibility with old skale-admin""" - return {k: leveldb_lim[k]['evm_storage_part'] for k in leveldb_lim.keys()} diff --git a/tests/.skale/config/environment_params.yaml b/tests/.skale/config/environment_params.yaml index e078465e..d36a1c4d 100644 --- a/tests/.skale/config/environment_params.yaml +++ b/tests/.skale/config/environment_params.yaml @@ -6,14 +6,14 @@ common: mem: skaled: 0.7 ima: 0.3 - volume: + volume_limits: max_consensus_storage_bytes: 0.3 max_skaled_leveldb_storage_bytes: 0.3 max_file_storage_bytes: 0.3 max_reserved_storage_bytes: 0.1 - leveldb_storage: - evm_storage_part: 0.6 - logs_storage_part: 0.4 + leveldb_limits: + contract_storage: 0.6 + db_storage: 0.4 base_rotate_after_block_divider: 61035.1 envs: mainnet: @@ -22,7 +22,7 @@ envs: cpu_physical: 8 memory: 32000000000 swap: 16000000000 - disk: 2000000000000 + disk: 1900000000000 packages: docker: 20.10 @@ -83,3 +83,7 @@ envs: btrfs-progs: 4.15.1 lsof: 0.0.0 psmisc: 0.0.0 + + docker: + docker-api: 0.0.0 + docker-engine: 0.0.0 \ No newline at end of file diff --git a/tests/.skale/config/schain_allocation.yml b/tests/.skale/config/schain_allocation.yml index b56a0e15..716eef0f 100644 --- a/tests/.skale/config/schain_allocation.yml +++ b/tests/.skale/config/schain_allocation.yml @@ -8,23 +8,29 @@ devnet: small: 593749504 test: 2374998016 test4: 2374998016 - leveldb: + leveldb_limits: large: - evm_storage_part: 13679988571 - logs_storage_part: 9119992381 + contract_storage: 13679988571 + db_storage: 9119992381 medium: - evm_storage_part: 427499642 - logs_storage_part: 284999761 + contract_storage: 427499642 + db_storage: 284999761 small: - evm_storage_part: 106874910 - logs_storage_part: 71249940 + contract_storage: 106874910 + db_storage: 71249940 test: - evm_storage_part: 427499642 - logs_storage_part: 284999761 + contract_storage: 427499642 + db_storage: 284999761 test4: - evm_storage_part: 427499642 - logs_storage_part: 284999761 - volume: + contract_storage: 427499642 + db_storage: 284999761 + rotate_after_block: + large: 1310721 + medium: 40960 + small: 10240 + test: 40960 + test4: 40960 + volume_limits: large: max_consensus_storage_bytes: 22799980953 max_file_storage_bytes: 22799980953 @@ -52,53 +58,59 @@ devnet: max_skaled_leveldb_storage_bytes: 712499404 mainnet: disk: - large: 1899999985664 - medium: 59374999552 - small: 14843749888 - test: 59374999552 - test4: 59374999552 - leveldb: + large: 1804999983104 + medium: 56406249472 + small: 14101562368 + test: 56406249472 + test4: 56406249472 + leveldb_limits: large: - evm_storage_part: 341999997419 - logs_storage_part: 227999998279 + contract_storage: 324899996958 + db_storage: 216599997972 medium: - evm_storage_part: 10687499919 - logs_storage_part: 7124999946 + contract_storage: 10153124904 + db_storage: 6768749936 small: - evm_storage_part: 2671874979 - logs_storage_part: 1781249986 + contract_storage: 2538281226 + db_storage: 1692187484 test: - evm_storage_part: 10687499919 - logs_storage_part: 7124999946 + contract_storage: 10153124904 + db_storage: 6768749936 test4: - evm_storage_part: 10687499919 - logs_storage_part: 7124999946 - volume: + contract_storage: 10153124904 + db_storage: 6768749936 + rotate_after_block: + large: 31129628 + medium: 972800 + small: 243200 + test: 972800 + test4: 972800 + volume_limits: large: - max_consensus_storage_bytes: 569999995699 - max_file_storage_bytes: 569999995699 - max_reserved_storage_bytes: 189999998566 - max_skaled_leveldb_storage_bytes: 569999995699 + max_consensus_storage_bytes: 541499994931 + max_file_storage_bytes: 541499994931 + max_reserved_storage_bytes: 180499998310 + max_skaled_leveldb_storage_bytes: 541499994931 medium: - max_consensus_storage_bytes: 17812499865 - max_file_storage_bytes: 17812499865 - max_reserved_storage_bytes: 5937499955 - max_skaled_leveldb_storage_bytes: 17812499865 + max_consensus_storage_bytes: 16921874841 + max_file_storage_bytes: 16921874841 + max_reserved_storage_bytes: 5640624947 + max_skaled_leveldb_storage_bytes: 16921874841 small: - max_consensus_storage_bytes: 4453124966 - max_file_storage_bytes: 4453124966 - max_reserved_storage_bytes: 1484374988 - max_skaled_leveldb_storage_bytes: 4453124966 + max_consensus_storage_bytes: 4230468710 + max_file_storage_bytes: 4230468710 + max_reserved_storage_bytes: 1410156236 + max_skaled_leveldb_storage_bytes: 4230468710 test: - max_consensus_storage_bytes: 17812499865 - max_file_storage_bytes: 17812499865 - max_reserved_storage_bytes: 5937499955 - max_skaled_leveldb_storage_bytes: 17812499865 + max_consensus_storage_bytes: 16921874841 + max_file_storage_bytes: 16921874841 + max_reserved_storage_bytes: 5640624947 + max_skaled_leveldb_storage_bytes: 16921874841 test4: - max_consensus_storage_bytes: 17812499865 - max_file_storage_bytes: 17812499865 - max_reserved_storage_bytes: 5937499955 - max_skaled_leveldb_storage_bytes: 17812499865 + max_consensus_storage_bytes: 16921874841 + max_file_storage_bytes: 16921874841 + max_reserved_storage_bytes: 5640624947 + max_skaled_leveldb_storage_bytes: 16921874841 qanet: disk: large: 189999939584 @@ -106,23 +118,29 @@ qanet: small: 1484374528 test: 5937498112 test4: 5937498112 - leveldb: + leveldb_limits: large: - evm_storage_part: 34199989125 - logs_storage_part: 22799992750 + contract_storage: 34199989125 + db_storage: 22799992750 medium: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 + contract_storage: 1068749659 + db_storage: 712499773 small: - evm_storage_part: 267187414 - logs_storage_part: 178124943 + contract_storage: 267187414 + db_storage: 178124943 test: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 + contract_storage: 1068749659 + db_storage: 712499773 test4: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 - volume: + contract_storage: 1068749659 + db_storage: 712499773 + rotate_after_block: + large: 3276803 + medium: 102400 + small: 25600 + test: 102400 + test4: 102400 + volume_limits: large: max_consensus_storage_bytes: 56999981875 max_file_storage_bytes: 56999981875 @@ -155,23 +173,29 @@ testnet: small: 1484374528 test: 5937498112 test4: 5937498112 - leveldb: + leveldb_limits: large: - evm_storage_part: 34199989125 - logs_storage_part: 22799992750 + contract_storage: 34199989125 + db_storage: 22799992750 medium: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 + contract_storage: 1068749659 + db_storage: 712499773 small: - evm_storage_part: 267187414 - logs_storage_part: 178124943 + contract_storage: 267187414 + db_storage: 178124943 test: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 + contract_storage: 1068749659 + db_storage: 712499773 test4: - evm_storage_part: 1068749659 - logs_storage_part: 712499773 - volume: + contract_storage: 1068749659 + db_storage: 712499773 + rotate_after_block: + large: 3276803 + medium: 102400 + small: 25600 + test: 102400 + test4: 102400 + volume_limits: large: max_consensus_storage_bytes: 56999981875 max_file_storage_bytes: 56999981875 diff --git a/tests/resources_test.py b/tests/resources_test.py index 7f0c6946..701689e9 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -9,8 +9,7 @@ from node_cli.core.resources import ( compose_resource_allocation_config, update_resource_allocation, - get_cpu_alloc, get_memory_alloc, - compose_storage_limit, verify_disk_size + get_cpu_alloc, get_memory_alloc, verify_disk_size ) from node_cli.utils.helper import write_json, safe_load_yml @@ -70,15 +69,7 @@ def test_generate_resource_allocation_config(): assert resource_allocation_config['ima']['cpu_shares'] == {'test4': 9, 'test': 9, 'small': 2, 'medium': 9, 'large': 307} # noqa assert isinstance(resource_allocation_config['ima']['mem'], dict) - print(resource_allocation_config['schain']['volume_limits']) assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS - assert resource_allocation_config['schain']['storage_limit'] == { - 'test4': 427499642, - 'test': 427499642, - 'small': 106874910, - 'medium': 427499642, - 'large': 13679988571 - } def test_update_allocation_config(resource_alloc_config): @@ -158,13 +149,13 @@ def test_get_memory_alloc(): assert ima_mem_alloc_dict['large'] == 3000000 -def test_compose_storage_limit(): - schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) - storage_limit = compose_storage_limit(schain_allocation_data['mainnet']['leveldb']) - assert storage_limit == { - 'large': 341999997419, - 'medium': 10687499919, - 'small': 2671874979, - 'test': 10687499919, - 'test4': 10687499919 +def test_leveldb_limits(): + with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): + resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) + assert resource_allocation_config['schain']['leveldb_limits'] == { + 'large': {'contract_storage': 13679988571, 'db_storage': 9119992381}, + 'medium': {'contract_storage': 427499642, 'db_storage': 284999761}, + 'small': {'contract_storage': 106874910, 'db_storage': 71249940}, + 'test': {'contract_storage': 427499642, 'db_storage': 284999761}, + 'test4': {'contract_storage': 427499642, 'db_storage': 284999761} } From 7e4dd5c41a6c01dfa9c3d940d6b5dee351014087 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 27 May 2021 19:12:46 +0300 Subject: [PATCH 106/140] SKALE-4212 Generate nginx config in init & update --- node_cli/configs/__init__.py | 2 +- node_cli/operations/base.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 6e7ead9f..7d99da31 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -50,7 +50,7 @@ FILESTORAGE_ARTIFACTS_FILE = os.path.join(NODE_DATA_PATH, 'filestorage_artifacts.json') ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') -NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') +NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 2644524d..95b82771 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -23,6 +23,7 @@ from node_cli.core.host import ( # noqa - tmp! prepare_host, link_env_file, run_preinstall_checks ) +from node_cli.core.nginx import generate_nginx_config from node_cli.core.resources import update_resource_allocation from node_cli.operations.common import ( @@ -50,6 +51,7 @@ def update(env_filepath: str, env: str) -> None: download_filestorage_artifacts() docker_lvmpy_update(env) sync_skale_node(env) + generate_nginx_config() prepare_host( env_filepath, @@ -86,6 +88,7 @@ def init(env_filepath: str, env: str) -> bool: configure_filebeat() configure_flask() configure_iptables() + generate_nginx_config() docker_lvmpy_install(env) From 27d0754c0c56ea968b6b39fb96b94eb1e705d805 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 28 May 2021 11:42:42 +0300 Subject: [PATCH 107/140] SKALE-4212 Mock rules in update test --- tests/cli/node_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index dc087a4b..91734290 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -144,6 +144,7 @@ def test_update_node(mocked_g_config): mock.patch('node_cli.core.node.update_op'), \ mock.patch('node_cli.core.node.get_flask_secret_key'), \ mock.patch('node_cli.core.node.save_env_params'), \ + mock.patch('node_cli.core.node.configure_firewall_rules'), \ mock.patch('node_cli.core.host.prepare_host'), \ mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ From e6b872bc62b5893590b2006044e3cc051cd3d14e Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Mon, 31 May 2021 19:56:33 +0300 Subject: [PATCH 108/140] SKALE-4212 Add watchdog https port --- node_cli/core/iptables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 76ba81d0..796f9492 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -39,10 +39,11 @@ ALLOWED_INCOMING_TCP_PORTS = [ '80', # filestorage '22', # ssh + '311', # watchdog https '8080', # http '443', # https '53', # dns - '3009', # watchdog + '3009', # watchdog http '9100' # node exporter ] From 912f0d218a0c3135fb49c9c18597482592ba9e72 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Thu, 3 Jun 2021 19:37:24 +0300 Subject: [PATCH 109/140] SKALE-4197 Create snapshots temp dir --- node_cli/configs/__init__.py | 1 + node_cli/core/host.py | 4 ++-- node_cli/operations/base.py | 11 ++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index b1a8c804..4f29b3cf 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -33,6 +33,7 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SCHAINS_MNT_DIR = '/mnt' +SNAPSHOTS_SCRATCH_DIR = os.path.join(SCHAINS_MNT_DIR, '.snapshots-temp') SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 86633855..1baeb81b 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,7 +36,7 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SCRATCH_DIR ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -107,7 +107,7 @@ def make_dirs(): CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SCRATCH_DIR ): safe_mk_dirs(dir_path) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 95b82771..fa9b8967 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -47,11 +47,6 @@ def update(env_filepath: str, env: str) -> None: remove_dynamic_containers() backup_old_contracts() - download_contracts(env) - download_filestorage_artifacts() - docker_lvmpy_update(env) - sync_skale_node(env) - generate_nginx_config() prepare_host( env_filepath, @@ -60,6 +55,12 @@ def update(env_filepath: str, env: str) -> None: env['ENV_TYPE'], allocation=True ) + download_contracts(env) + download_filestorage_artifacts() + docker_lvmpy_update(env) + sync_skale_node(env) + generate_nginx_config() + update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], From d306c6f163cbc792958e230db7f83962a236412f Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 4 Jun 2021 13:27:43 +0300 Subject: [PATCH 110/140] SKALE-4197 Add volume commands --- node_cli/configs/__init__.py | 2 +- node_cli/core/host.py | 4 ++-- node_cli/utils/docker_utils.py | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 4f29b3cf..6ba80671 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -33,7 +33,7 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SCHAINS_MNT_DIR = '/mnt' -SNAPSHOTS_SCRATCH_DIR = os.path.join(SCHAINS_MNT_DIR, '.snapshots-temp') +SNAPSHOTS_SHARED_DIR = os.path.join(SCHAINS_MNT_DIR, 'shared') SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 1baeb81b..a9c4b0b2 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,7 +36,7 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SCRATCH_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_DIR ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -107,7 +107,7 @@ def make_dirs(): CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SCRATCH_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_DIR ): safe_mk_dirs(dir_path) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index fc317731..5de0f1e2 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -127,6 +127,27 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) +def init_shared_volume(name: str): + if is_volume_exists(name): + return + size = 0 + logging.info(f'Creating volume - size: {size}, name: {name}') + driver_opts = {'size': str(size)} + volume = docker_client().volumes.create( + name=name, + driver_opts=driver_opts + ) + return volume + + +def is_volume_exists(name: str): + try: + docker_client().volumes.get(name) + except docker.errors.NotFound: + return False + return True + + def compose_rm(env={}): logger.info(f'Removing {MAIN_COMPOSE_CONTAINERS} containers') run_cmd( From fa67733629ea2e9cfd6208b852a0da7d15a82082 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Fri, 4 Jun 2021 17:52:20 +0300 Subject: [PATCH 111/140] SKALE-4197 Add init_shared_volume function --- node_cli/configs/__init__.py | 2 +- node_cli/utils/docker_utils.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 6ba80671..38e0a42d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -33,7 +33,7 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SCHAINS_MNT_DIR = '/mnt' -SNAPSHOTS_SHARED_DIR = os.path.join(SCHAINS_MNT_DIR, 'shared') +SNAPSHOTS_SHARED_DIR = os.path.join(SCHAINS_MNT_DIR, 'snapshots-shared-space') SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 5de0f1e2..7633a848 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -127,14 +127,18 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) -def init_shared_volume(name: str): +def init_shared_volume(): + init_volume('snapshots-shared-space', 0) + + +def init_volume(name: str, size: int): if is_volume_exists(name): return - size = 0 logging.info(f'Creating volume - size: {size}, name: {name}') driver_opts = {'size': str(size)} volume = docker_client().volumes.create( name=name, + driver='lvmpy', driver_opts=driver_opts ) return volume From d2cecc77d695e5cc5d9d1d673e5349ccd1b5839b Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 7 Jun 2021 12:51:47 +0300 Subject: [PATCH 112/140] SKALE-3350 Add redis to base containers --- node_cli/utils/docker_utils.py | 53 ++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index fc317731..57460d5e 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -27,7 +27,10 @@ from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.configs import ( - COMPOSE_PATH, SKALE_DIR, SGX_CERTIFICATES_DIR_NAME, REMOVED_CONTAINERS_FOLDER_PATH + COMPOSE_PATH, + REMOVED_CONTAINERS_FOLDER_PATH, + SGX_CERTIFICATES_DIR_NAME, + SKALE_DIR ) @@ -36,10 +39,19 @@ SCHAIN_REMOVE_TIMEOUT = 60 IMA_REMOVE_TIMEOUT = 20 -MAIN_COMPOSE_CONTAINERS = 'skale-api bounty skale-admin' -BASE_COMPOSE_SERVICES = 'transaction-manager skale-admin skale-api bounty nginx watchdog filebeat' # noqa -MONITORING_COMPOSE_SERVICES = 'node-exporter advisor' -NOTIFICATION_COMPOSE_SERVICES = 'celery redis' +MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin') +BASE_COMPOSE_SERVICES = ( + 'transaction-manager', + 'skale-admin', + 'skale-api', + 'bounty', + 'nginx', + 'redis', + 'watchdog', + 'filebeat' +) +MONITORING_COMPOSE_SERVICES = ('node-exporter', 'advisor') +NOTIFICATION_COMPOSE_SERVICES = ('celery',) COMPOSE_TIMEOUT = 10 DOCKER_DEFAULT_STOP_TIMEOUT = 20 @@ -130,14 +142,18 @@ def get_logs_backup_filepath(container: Container) -> str: def compose_rm(env={}): logger.info(f'Removing {MAIN_COMPOSE_CONTAINERS} containers') run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} rm -s -f {MAIN_COMPOSE_CONTAINERS}'.split(), + cmd=( + 'docker-compose', + '-f', COMPOSE_PATH, + 'rm', '-s', '-f', *MAIN_COMPOSE_CONTAINERS + ), env=env ) logger.info(f'Sleeping for {COMPOSE_TIMEOUT} seconds') sleep(COMPOSE_TIMEOUT) logger.info('Removing all compose containers') run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} rm -s -f'.split(), + cmd=('docker-compose', '-f', COMPOSE_PATH, 'rm', '-s', '-f'), env=env ) logger.info('Compose containers removed') @@ -146,7 +162,7 @@ def compose_rm(env={}): def compose_pull(): logger.info('Pulling compose containers') run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} pull'.split(), + cmd=('docker-compose', '-f', COMPOSE_PATH, 'pull'), env={ 'SKALE_DIR': SKALE_DIR } @@ -156,32 +172,27 @@ def compose_pull(): def compose_build(): logger.info('Building compose containers') run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} build'.split(), + cmd=('docker-compose', '-f', COMPOSE_PATH, 'build'), env={ 'SKALE_DIR': SKALE_DIR } ) +def get_up_compose_cmd(services): + return ('docker-compose', '-f', COMPOSE_PATH, 'up', '-d', *services) + + def compose_up(env): logger.info('Running base set of containers') if 'SGX_CERTIFICATES_DIR_NAME' not in env: env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME - run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} up -d {BASE_COMPOSE_SERVICES}'.split(), - env=env - ) + run_cmd(cmd=get_up_compose_cmd(BASE_COMPOSE_SERVICES), env=env) if str_to_bool(env.get('MONITORING_CONTAINERS', '')): logger.info('Running monitoring containers') - run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} up -d {MONITORING_COMPOSE_SERVICES}'.split(), - env=env - ) + run_cmd(cmd=get_up_compose_cmd(MONITORING_COMPOSE_SERVICES), env=env) if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') - run_cmd( - cmd=f'docker-compose -f {COMPOSE_PATH} up -d {NOTIFICATION_COMPOSE_SERVICES}'.split(), - env=env - ) + run_cmd(cmd=get_up_compose_cmd(NOTIFICATION_COMPOSE_SERVICES), env=env) From f03e32535ab8e92f8725f708c40c3284c0f96e0e Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 7 Jun 2021 17:37:38 +0300 Subject: [PATCH 113/140] SKALE-3350 Bump minor version --- node_cli/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 4f404329..4534a389 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.0.0' +__version__ = '2.0.1' if __name__ == "__main__": print(__version__) From bae3bf0e8ba28d1f5b4ea2e9f84051b21f7a56ea Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 8 Jun 2021 15:22:21 +0300 Subject: [PATCH 114/140] SKALE-4197 Get volume size from schain_allocation --- node_cli/configs/__init__.py | 4 +++- node_cli/core/host.py | 4 ++-- node_cli/core/resources.py | 25 ++++++++++++++++++++++++- node_cli/operations/base.py | 6 +++++- node_cli/utils/docker_utils.py | 4 ---- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 38e0a42d..95bb7633 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -33,7 +33,8 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SCHAINS_MNT_DIR = '/mnt' -SNAPSHOTS_SHARED_DIR = os.path.join(SCHAINS_MNT_DIR, 'snapshots-shared-space') +SNAPSHOTS_SHARED_FOLDER_NAME = 'snapshots-shared-space' +SNAPSHOTS_SHARED_FOLDER_PATH = os.path.join(SCHAINS_MNT_DIR, SNAPSHOTS_SHARED_FOLDER_NAME) SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') @@ -52,6 +53,7 @@ ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf') +SCHAIN_ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'schain_allocation.yml') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/core/host.py b/node_cli/core/host.py index a9c4b0b2..8f29387d 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,7 +36,7 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_FOLDER_PATH ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -107,7 +107,7 @@ def make_dirs(): CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_DIR + SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_FOLDER_PATH ): safe_mk_dirs(dir_path) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index df5f93d2..6a3378ab 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -23,11 +23,14 @@ import psutil +from node_cli.utils.docker_utils import init_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import ( write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) -from node_cli.configs import ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH +from node_cli.configs import ( + ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, SCHAIN_ALLOCATION_FILEPATH +) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, @@ -61,6 +64,21 @@ def get_resource_allocation_info(): return None +def get_schain_allocation_info(): + logger.info(SCHAIN_ALLOCATION_FILEPATH) + if os.path.isfile(SCHAIN_ALLOCATION_FILEPATH): + return safe_load_yml(SCHAIN_ALLOCATION_FILEPATH) + else: + return None + + +def get_shared_space_size(env_type): + schain_allocation = get_schain_allocation_info() + if not schain_allocation: + return None + return schain_allocation[env_type]['shared_space'] + + def compose_resource_allocation_config(env_type): env_configs = safe_load_yml(ENVIRONMENT_PARAMS_FILEPATH) schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) @@ -186,3 +204,8 @@ def get_allocation_option_name(schain): def get_disk_path(): with open(DISK_MOUNTPOINT_FILEPATH) as f: return f.read().strip() + + +def init_shared_space_volume(env_type): + size = get_shared_space_size(env_type) + init_volume('snapshots-shared-space', size) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index fa9b8967..7966d20f 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -24,7 +24,7 @@ prepare_host, link_env_file, run_preinstall_checks ) from node_cli.core.nginx import generate_nginx_config -from node_cli.core.resources import update_resource_allocation +from node_cli.core.resources import update_resource_allocation, init_shared_space_volume from node_cli.operations.common import ( backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, @@ -57,7 +57,10 @@ def update(env_filepath: str, env: str) -> None: ) download_contracts(env) download_filestorage_artifacts() + docker_lvmpy_update(env) + init_shared_space_volume(env['ENV_TYPE']) + sync_skale_node(env) generate_nginx_config() @@ -92,6 +95,7 @@ def init(env_filepath: str, env: str) -> bool: generate_nginx_config() docker_lvmpy_install(env) + init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 7633a848..e989b056 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -127,10 +127,6 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) -def init_shared_volume(): - init_volume('snapshots-shared-space', 0) - - def init_volume(name: str, size: int): if is_volume_exists(name): return From e4c3912399e8ede064f4e45fcc7981988f9fd2e5 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 8 Jun 2021 15:38:30 +0300 Subject: [PATCH 115/140] SKALE-4197 Rename volume --- node_cli/configs/__init__.py | 2 +- node_cli/core/resources.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 95bb7633..e17d087d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -33,7 +33,7 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SCHAINS_MNT_DIR = '/mnt' -SNAPSHOTS_SHARED_FOLDER_NAME = 'snapshots-shared-space' +SNAPSHOTS_SHARED_FOLDER_NAME = 'shared-space' SNAPSHOTS_SHARED_FOLDER_PATH = os.path.join(SCHAINS_MNT_DIR, SNAPSHOTS_SHARED_FOLDER_NAME) SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 6a3378ab..c94b6fc2 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -29,7 +29,7 @@ write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) from node_cli.configs import ( - ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, SCHAIN_ALLOCATION_FILEPATH + ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, SCHAIN_ALLOCATION_FILEPATH, SNAPSHOTS_SHARED_FOLDER_NAME ) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, @@ -208,4 +208,4 @@ def get_disk_path(): def init_shared_space_volume(env_type): size = get_shared_space_size(env_type) - init_volume('snapshots-shared-space', size) + init_volume(SNAPSHOTS_SHARED_FOLDER_NAME, size) From f8b538c87b3c2cd06446cd1f28e1c6bae636fe78 Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 8 Jun 2021 15:39:06 +0300 Subject: [PATCH 116/140] SKALE-4197 Fix flake8 --- node_cli/core/resources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index c94b6fc2..a9dbe338 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -29,7 +29,8 @@ write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) from node_cli.configs import ( - ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, SCHAIN_ALLOCATION_FILEPATH, SNAPSHOTS_SHARED_FOLDER_NAME + ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, + SCHAIN_ALLOCATION_FILEPATH, SNAPSHOTS_SHARED_FOLDER_NAME ) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, From 56db1e9b9dbb73bef9d36bbe7ac6593ee2f6e16d Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 8 Jun 2021 16:53:56 +0300 Subject: [PATCH 117/140] SKALE-4197 Fix base.py, remove redundant docker_clients --- node_cli/operations/base.py | 12 +++++------- node_cli/utils/docker_utils.py | 12 +++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 7966d20f..c9c80c85 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -47,6 +47,11 @@ def update(env_filepath: str, env: str) -> None: remove_dynamic_containers() backup_old_contracts() + download_contracts(env) + download_filestorage_artifacts() + docker_lvmpy_update(env) + sync_skale_node(env) + generate_nginx_config() prepare_host( env_filepath, @@ -55,15 +60,8 @@ def update(env_filepath: str, env: str) -> None: env['ENV_TYPE'], allocation=True ) - download_contracts(env) - download_filestorage_artifacts() - - docker_lvmpy_update(env) init_shared_space_volume(env['ENV_TYPE']) - sync_skale_node(env) - generate_nginx_config() - update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index e989b056..5cbb84c3 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -127,12 +127,13 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) -def init_volume(name: str, size: int): - if is_volume_exists(name): +def init_volume(name: str, size: int, dutils=None): + dutils = dutils or docker_client() + if is_volume_exists(name, dutils=dutils): return logging.info(f'Creating volume - size: {size}, name: {name}') driver_opts = {'size': str(size)} - volume = docker_client().volumes.create( + volume = dutils.volumes.create( name=name, driver='lvmpy', driver_opts=driver_opts @@ -140,9 +141,10 @@ def init_volume(name: str, size: int): return volume -def is_volume_exists(name: str): +def is_volume_exists(name: str, dutils=None): + dutils = dutils or docker_client() try: - docker_client().volumes.get(name) + dutils.volumes.get(name) except docker.errors.NotFound: return False return True From 04d542e31fd55ad803cea7ab1b8ff93ec9a75ded Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 8 Jun 2021 17:04:29 +0300 Subject: [PATCH 118/140] SKALE-4197 Remove redundant methods from resources.py --- node_cli/configs/__init__.py | 1 - node_cli/core/resources.py | 20 +++----------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index e17d087d..c9dc0d60 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -53,7 +53,6 @@ ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf') -SCHAIN_ALLOCATION_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'schain_allocation.yml') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index a9dbe338..bff50885 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -30,7 +30,7 @@ ) from node_cli.configs import ( ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, - SCHAIN_ALLOCATION_FILEPATH, SNAPSHOTS_SHARED_FOLDER_NAME + SNAPSHOTS_SHARED_FOLDER_NAME ) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, @@ -65,21 +65,6 @@ def get_resource_allocation_info(): return None -def get_schain_allocation_info(): - logger.info(SCHAIN_ALLOCATION_FILEPATH) - if os.path.isfile(SCHAIN_ALLOCATION_FILEPATH): - return safe_load_yml(SCHAIN_ALLOCATION_FILEPATH) - else: - return None - - -def get_shared_space_size(env_type): - schain_allocation = get_schain_allocation_info() - if not schain_allocation: - return None - return schain_allocation[env_type]['shared_space'] - - def compose_resource_allocation_config(env_type): env_configs = safe_load_yml(ENVIRONMENT_PARAMS_FILEPATH) schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) @@ -208,5 +193,6 @@ def get_disk_path(): def init_shared_space_volume(env_type): - size = get_shared_space_size(env_type) + schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) + size = schain_allocation_data[env_type]['shared_space'] init_volume(SNAPSHOTS_SHARED_FOLDER_NAME, size) From 8e29b72d591cbb85d34272cd03279fb43c9f79ba Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Jun 2021 12:52:20 +0300 Subject: [PATCH 119/140] Hofix Do not create redundant /mnt shared dir --- node_cli/configs/__init__.py | 3 +-- node_cli/core/resources.py | 18 ++++++++++++------ node_cli/operations/base.py | 6 +++--- node_cli/utils/docker_utils.py | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index c9dc0d60..4575b381 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -32,9 +32,8 @@ SKALE_STATE_DIR = '/var/lib/skale' FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') +SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/mnt' -SNAPSHOTS_SHARED_FOLDER_NAME = 'shared-space' -SNAPSHOTS_SHARED_FOLDER_PATH = os.path.join(SCHAINS_MNT_DIR, SNAPSHOTS_SHARED_FOLDER_NAME) SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index bff50885..ebd1796f 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -23,14 +23,14 @@ import psutil -from node_cli.utils.docker_utils import init_volume +from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import ( write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) from node_cli.configs import ( ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, - SNAPSHOTS_SHARED_FOLDER_NAME + SNAPSHOTS_SHARED_VOLUME ) from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, @@ -41,6 +41,10 @@ logger = logging.getLogger(__name__) +class NotEnoughDiskSpaceError(Exception): + pass + + class ResourceAlloc: def __init__(self, value, fractional=False): self.values = { @@ -79,8 +83,8 @@ def compose_resource_allocation_config(env_type): 'cpu_shares': schain_cpu_alloc.dict(), 'mem': schain_mem_alloc.dict(), 'disk': schain_allocation_data[env_type]['disk'], - 'volume_limits': schain_allocation_data[env_type]['volume_limits'], - 'leveldb_limits': schain_allocation_data[env_type]['leveldb_limits'] + 'volume_limits': schain_allocation_data[env_type]['volume_limits'], # noqa + 'leveldb_limits': schain_allocation_data[env_type]['leveldb_limits'] # noqa }, 'ima': { 'cpu_shares': ima_cpu_alloc.dict(), @@ -160,7 +164,9 @@ def verify_disk_size(env_configs: dict, env_type: str) -> dict: def check_disk_size(disk_size: int, env_disk_size: int): if env_disk_size > disk_size: - raise Exception(f'Disk size: {disk_size}, required disk size: {env_disk_size}') + raise NotEnoughDiskSpaceError( + f'Disk size: {disk_size}, required disk size: {env_disk_size}' + ) def get_disk_size() -> int: @@ -195,4 +201,4 @@ def get_disk_path(): def init_shared_space_volume(env_type): schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) size = schain_allocation_data[env_type]['shared_space'] - init_volume(SNAPSHOTS_SHARED_FOLDER_NAME, size) + ensure_volume(SNAPSHOTS_SHARED_VOLUME, size) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c9c80c85..794d6a86 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -20,7 +20,7 @@ import logging from node_cli.cli.info import VERSION -from node_cli.core.host import ( # noqa - tmp! +from node_cli.core.host import ( # noqa prepare_host, link_env_file, run_preinstall_checks ) from node_cli.core.nginx import generate_nginx_config @@ -36,13 +36,13 @@ from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta -from node_cli.utils.print_formatters import print_failed_requirements_checks # noqa - tmp! +from node_cli.utils.print_formatters import print_failed_requirements_checks # noqa logger = logging.getLogger(__name__) -def update(env_filepath: str, env: str) -> None: +def update(env_filepath: str, env: Dict) -> None: compose_rm(env) remove_dynamic_containers() diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 78660b04..b9590a13 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -139,7 +139,7 @@ def get_logs_backup_filepath(container: Container) -> str: return os.path.join(REMOVED_CONTAINERS_FOLDER_PATH, log_file_name) -def init_volume(name: str, size: int, dutils=None): +def ensure_volume(name: str, size: int, dutils=None): dutils = dutils or docker_client() if is_volume_exists(name, dutils=dutils): return From 0b187a486857124e8109a2388cd0a6c8247f8d75 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Jun 2021 12:54:05 +0300 Subject: [PATCH 120/140] Hofix Fix filestorage mapping creation --- node_cli/core/host.py | 4 ++-- node_cli/core/node.py | 12 ++++++++++-- node_cli/operations/docker_lvmpy.py | 12 +++++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 8f29387d..95e9e417 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -36,7 +36,7 @@ REDIS_DATA_PATH, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_FOLDER_PATH + SKALE_TMP_DIR ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH @@ -107,7 +107,7 @@ def make_dirs(): CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH, REDIS_DATA_PATH, - SKALE_TMP_DIR, FILESTORAGE_MAPPING, SNAPSHOTS_SHARED_FOLDER_PATH + SKALE_TMP_DIR ): safe_mk_dirs(dir_path) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index a78503af..564751b3 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,8 +28,14 @@ import docker from node_cli.configs import ( - SKALE_DIR, INIT_ENV_FILEPATH, - BACKUP_ARCHIVE_NAME, G_CONF_HOME, RESTORE_SLEEP_TIMEOUT, TM_INIT_TIMEOUT + FILESTORAGE_MAPPING, + SKALE_DIR, + INIT_ENV_FILEPATH, + BACKUP_ARCHIVE_NAME, + G_CONF_HOME, + RESTORE_SLEEP_TIMEOUT, + SCHAINS_MNT_DIR, + TM_INIT_TIMEOUT ) from node_cli.configs.cli_logger import LOG_DIRNAME @@ -158,6 +164,8 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None): env_params = extract_env_params(INIT_ENV_FILEPATH) env = { 'SKALE_DIR': SKALE_DIR, + 'SCHAINS_MNT_DIR': SCHAINS_MNT_DIR, + 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, **env_params } if inited_node: diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 64168257..ea3cc5be 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -37,8 +37,17 @@ def update_docker_lvmpy_env(env): return env +def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): + if not os.path.isdir(FILESTORAGE_MAPPING): + os.makedirs(FILESTORAGE_MAPPING) + + def sync_docker_lvmpy_repo(env): - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env["DOCKER_LVMPY_STREAM"]) + sync_repo( + DOCKER_LVMPY_REPO_URL, + DOCKER_LVMPY_PATH, + env["DOCKER_LVMPY_STREAM"] + ) def docker_lvmpy_update(env): @@ -54,6 +63,7 @@ def docker_lvmpy_update(env): def docker_lvmpy_install(env): sync_docker_lvmpy_repo(env) + ensure_filestorage_mapping() update_docker_lvmpy_env(env) run_cmd( cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), From 53b0731eb7f5bac75fcba94b7d11802b30945b8b Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Jun 2021 12:56:52 +0300 Subject: [PATCH 121/140] Hofix Fix flake8 --- node_cli/operations/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 794d6a86..bf1c3df8 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import logging +from typing import Dict from node_cli.cli.info import VERSION from node_cli.core.host import ( # noqa From 9f372378badd93e3936990e35238bc50ff408b37 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Jun 2021 13:05:17 +0300 Subject: [PATCH 122/140] Hotfix Fix docker-lvmpy update --- node_cli/operations/docker_lvmpy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index ea3cc5be..c6a6e05b 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -52,6 +52,7 @@ def sync_docker_lvmpy_repo(env): def docker_lvmpy_update(env): sync_docker_lvmpy_repo(env) + ensure_filestorage_mapping() logger.info('Running docker-lvmpy update script') update_docker_lvmpy_env(env) run_cmd( From 8cd3b47e514b057344dbed43756c3211191ad8ad Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Jun 2021 13:49:56 +0300 Subject: [PATCH 123/140] Hofix Add more logs --- node_cli/core/checks.py | 4 ++-- node_cli/core/resources.py | 1 + node_cli/utils/docker_utils.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 7b7f6271..1d3dfb1c 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -26,7 +26,7 @@ import shutil import socket from collections import namedtuple -from typing import List +from typing import Dict, List import docker import yaml @@ -74,7 +74,7 @@ def check(self) -> ListChecks: class MachineChecker(BaseChecker): - def __init__(self, requirements: dict) -> None: + def __init__(self, requirements: Dict) -> None: self.requirements = requirements def cpu_total(self) -> CheckResult: diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index ebd1796f..b2fb517c 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -199,6 +199,7 @@ def get_disk_path(): def init_shared_space_volume(env_type): + logger.info('Configuring shared space volume') schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) size = schain_allocation_data[env_type]['shared_space'] ensure_volume(SNAPSHOTS_SHARED_VOLUME, size) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index b9590a13..5ea27aba 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -142,8 +142,9 @@ def get_logs_backup_filepath(container: Container) -> str: def ensure_volume(name: str, size: int, dutils=None): dutils = dutils or docker_client() if is_volume_exists(name, dutils=dutils): + logger.info(f'Volume with name {name} already exits') return - logging.info(f'Creating volume - size: {size}, name: {name}') + logger.info(f'Creating volume - size: {size}, name: {name}') driver_opts = {'size': str(size)} volume = dutils.volumes.create( name=name, From 967988f2347383fb28524a867b2d4a6281b868e2 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 11 Jun 2021 08:19:03 +0300 Subject: [PATCH 124/140] Hotfix Exclude logs from backup --- node_cli/core/node.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 564751b3..4e9a176a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -20,7 +20,6 @@ import datetime import logging import os -import shlex import subprocess import time from enum import Enum @@ -32,12 +31,12 @@ SKALE_DIR, INIT_ENV_FILEPATH, BACKUP_ARCHIVE_NAME, - G_CONF_HOME, RESTORE_SLEEP_TIMEOUT, SCHAINS_MNT_DIR, - TM_INIT_TIMEOUT + TM_INIT_TIMEOUT, + LOG_PATH ) -from node_cli.configs.cli_logger import LOG_DIRNAME +from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH from node_cli.core.iptables import configure_iptables from node_cli.core.host import ( @@ -56,7 +55,11 @@ from node_cli.utils.helper import run_cmd, extract_env_params from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.decorators import check_not_inited, check_inited, check_user +from node_cli.utils.decorators import ( + check_not_inited, + check_inited, + check_user +) logger = logging.getLogger(__name__) @@ -220,11 +223,14 @@ def get_backup_filepath(base_path): def create_backup_archive(backup_filepath): print('Creating backup archive...') - log_skale_path = os.path.join('.skale', LOG_DIRNAME) - cmd = shlex.split( - f'tar -zcvf {backup_filepath} -C {G_CONF_HOME} ' - f'--exclude {log_skale_path} .skale' - ) + cli_log_path = CLI_LOG_DATA_PATH + container_log_path = LOG_PATH + cmd = [ + 'tar', '-zcvf', backup_filepath, + '--exclude', container_log_path, + '--exclude', cli_log_path, + SKALE_DIR, + ] try: run_cmd(cmd) print(f'Backup archive successfully created: {backup_filepath}') @@ -309,7 +315,10 @@ def get_node_info(format): elif node_info['status'] == NodeStatuses.NOT_CREATED.value: print(TEXTS['service']['node_not_registered']) else: - print_node_info(node_info, get_node_status(int(node_info['status']))) + print_node_info( + node_info, + get_node_status(int(node_info['status'])) + ) else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) From aa507241dd263251294daa0a0546eb4871b556a9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 11 Jun 2021 19:10:23 +0300 Subject: [PATCH 125/140] Fix FS update procedure --- node_cli/operations/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index bf1c3df8..9e8abd53 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -49,9 +49,9 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) + sync_skale_node(env) download_filestorage_artifacts() docker_lvmpy_update(env) - sync_skale_node(env) generate_nginx_config() prepare_host( From a18547af561279673fbb68f1cf1319ba203f8ab0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 14 Jun 2021 07:29:23 +0300 Subject: [PATCH 126/140] Hotfix Exclude logs from backup archive --- node_cli/core/node.py | 60 +++++++++++++++++++++++++++--------- tests/core_node_test.py | 67 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 106 insertions(+), 21 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 4e9a176a..920f2585 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -20,9 +20,12 @@ import datetime import logging import os -import subprocess +import tarfile import time +from contextlib import contextmanager from enum import Enum +from pathlib import Path +from typing import Tuple import docker @@ -52,7 +55,7 @@ print_failed_requirements_checks, print_node_cmd_error, print_node_info ) from node_cli.utils.helper import error_exit, get_request, post_request -from node_cli.utils.helper import run_cmd, extract_env_params +from node_cli.utils.helper import extract_env_params from node_cli.utils.texts import Texts from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.decorators import ( @@ -218,25 +221,54 @@ def get_backup_filename(): def get_backup_filepath(base_path): - return os.path.join(base_path, get_backup_filename()) + return os.path.abspath(os.path.join(base_path, get_backup_filename())) + + +@contextmanager +def chdir(dest): + old = os.getcwd() + try: + os.chdir(dest) + yield + finally: + os.chdir(old) + + +def pack_dir(source: str, dest: str, exclude: Tuple[str] = ()): + logger.info('Packing dir %s to %s excluding %s', source, dest, exclude) + + source, dest = Path(source), Path(dest) + exclude = [Path(e).relative_to(source.parent) for e in exclude] + + def logfilter(tarinfo): + path = Path(tarinfo.name) + for e in exclude: + logger.debug('Cheking if %s is parent of %s', e, tarinfo.name) + try: + path.relative_to(e) + except ValueError: + pass + else: + logger.debug('Excluding %s', tarinfo.name) + return None + return tarinfo + + with chdir(source.parent): + with tarfile.open(dest, 'w:gz') as tar: + tar.add(source.name, filter=logfilter) + logger.info('Packing finished %s', source) def create_backup_archive(backup_filepath): print('Creating backup archive...') cli_log_path = CLI_LOG_DATA_PATH container_log_path = LOG_PATH - cmd = [ - 'tar', '-zcvf', backup_filepath, - '--exclude', container_log_path, - '--exclude', cli_log_path, + pack_dir( SKALE_DIR, - ] - try: - run_cmd(cmd) - print(f'Backup archive successfully created: {backup_filepath}') - except subprocess.CalledProcessError: - logger.exception('Backup archive creation failed') - print_node_cmd_error() + backup_filepath, + exclude=(cli_log_path, container_log_path) + ) + print('Backup archive succesfully created') def set_maintenance_mode_on(): diff --git a/tests/core_node_test.py b/tests/core_node_test.py index 7ac8698d..62c8b3aa 100644 --- a/tests/core_node_test.py +++ b/tests/core_node_test.py @@ -1,9 +1,14 @@ +import os +import shutil +import tarfile import time +from pathlib import Path import docker import pytest from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive +from node_cli.core.node import pack_dir dclient = docker.from_env() @@ -13,7 +18,7 @@ @pytest.fixture -def skale_base_contianers(): +def skale_base_containers(): containers = [ dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) @@ -25,7 +30,7 @@ def skale_base_contianers(): @pytest.fixture -def skale_base_contianers_without_one(): +def skale_base_containers_without_one(): containers = [ dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) @@ -37,7 +42,7 @@ def skale_base_contianers_without_one(): @pytest.fixture -def skale_base_contianers_exited(): +def skale_base_containers_exited(): containers = [ dclient.containers.run(HELLO_WORLD_IMAGE_NAME, detach=True, name=f'skale_test{i}') @@ -49,17 +54,65 @@ def skale_base_contianers_exited(): c.remove(force=True) -def test_is_base_containers_alive(skale_base_contianers): - cont = skale_base_contianers +@pytest.fixture +def tmp_dir(): + tmp_dir = 'tmp' + yield os.path.abspath(tmp_dir) + shutil.rmtree(tmp_dir) + + +def test_pack_dir(tmp_dir): + backup_dir = os.path.join(tmp_dir, 'backup') + data_dir = os.path.join(backup_dir, 'data') + trash_dir = os.path.join(backup_dir, 'trash') + a_data = os.path.join(data_dir, 'a-data') + b_data = os.path.join(data_dir, 'b-data') + trash_data = os.path.join(trash_dir, 'trash-data') + os.makedirs(tmp_dir) + os.makedirs(data_dir) + os.makedirs(trash_dir) + + for filepath in (a_data, b_data, trash_data): + with open(filepath, 'w') as f: + f.write(f.name) + + archive_path = os.path.abspath(os.path.join(tmp_dir, 'archive.tar.gz')) + pack_dir(backup_dir, archive_path) + with tarfile.open(archive_path) as tar: + print(tar.getnames()) + assert Path(a_data).relative_to(tmp_dir).as_posix() in tar.getnames() + assert Path(b_data).relative_to(tmp_dir).as_posix() in tar.getnames() + assert Path(trash_data).relative_to(tmp_dir).as_posix() in \ + tar.getnames() + + cleaned_archive_path = os.path.abspath( + os.path.join(tmp_dir, 'cleaned-archive.tar.gz') + ) + pack_dir(backup_dir, cleaned_archive_path, exclude=(trash_dir,)) + with tarfile.open(cleaned_archive_path) as tar: + assert Path(a_data).relative_to(tmp_dir).as_posix() in tar.getnames() + assert Path(b_data).relative_to(tmp_dir).as_posix() in tar.getnames() + assert Path(trash_data).relative_to(tmp_dir).as_posix() not in \ + tar.getnames() + + # Not absolute or unrelated path in exclude raises ValueError + with pytest.raises(ValueError): + pack_dir(backup_dir, cleaned_archive_path, exclude=('trash_data',)) + + +def test_is_base_containers_alive(skale_base_containers): + cont = skale_base_containers print([c.name for c in cont]) assert is_base_containers_alive() -def test_is_base_containers_alive_one_failed(skale_base_contianers_without_one): +def test_is_base_containers_alive_one_failed( + skale_base_containers_without_one +): assert not is_base_containers_alive() -def test_is_base_containers_alive_exited(skale_base_contianers_exited): +def test_is_base_containers_alive_exited(skale_base_containers_exited): assert not is_base_containers_alive() From b286d5bdac76c0953483b9a1084e70d9a658fd58 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 14 Jun 2021 14:28:44 +0300 Subject: [PATCH 127/140] Hofix Fix cli tests --- node_cli/core/node.py | 2 +- tests/cli/node_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 920f2585..91dd83e7 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -268,7 +268,7 @@ def create_backup_archive(backup_filepath): backup_filepath, exclude=(cli_log_path, container_log_path) ) - print('Backup archive succesfully created') + print(f'Backup archive succesfully created {backup_filepath}') def set_maintenance_mode_on(): diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 1af4176a..69491e7b 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -345,7 +345,8 @@ def test_backup(): ['/tmp'] ) assert result.exit_code == 0 - assert 'Backup archive successfully created: /tmp/skale-node-backup-' in result.output + print(result.output) + assert 'Backup archive succesfully created ' in result.output def test_restore(mocked_g_config): From a10885e93ee1f4bad06b8adc567a7f0e94b642b3 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 16 Jun 2021 11:21:01 +0300 Subject: [PATCH 128/140] Hotfix Add init shared space volume to restore --- node_cli/operations/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 9e8abd53..97132831 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -124,6 +124,8 @@ def restore(env, backup_path): link_env_file() configure_iptables() docker_lvmpy_install(env) + init_shared_space_volume(env['ENV_TYPE']) + update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], From cac11390dc846f1f3c8267022c40b354ae6a79bd Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 16 Jun 2021 13:05:11 +0300 Subject: [PATCH 129/140] Hotfix Enable checks --- node_cli/core/checks.py | 126 +++++++++++++++---------- node_cli/core/host.py | 6 +- tests/core_checks_test.py | 194 +++++++++++++++++++++----------------- 3 files changed, 188 insertions(+), 138 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 1d3dfb1c..0fb949c3 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -26,6 +26,7 @@ import shutil import socket from collections import namedtuple +from functools import wraps from typing import Dict, List import docker @@ -56,12 +57,23 @@ def get_env_params(network: str = 'mainnet'): return ydata['envs'][network] +def node_check(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as err: + logger.error('%s check errored with %s', func.__name__, err) + return CheckResult(name=func.__name__, status='error', info=err) + return wrapper + + class BaseChecker: def _ok(self, name: str, info=None) -> CheckResult: return CheckResult(name=name, status='ok', info=info) - def _error(self, name: str, info=None) -> CheckResult: - return CheckResult(name=name, status='error', info=info) + def _failed(self, name: str, info=None) -> CheckResult: + return CheckResult(name=name, status='failed', info=info) def check(self) -> ListChecks: myself = inspect.stack()[0][3] @@ -77,51 +89,56 @@ class MachineChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: self.requirements = requirements + @node_check def cpu_total(self) -> CheckResult: name = 'cpu-total' actual = psutil.cpu_count(logical=True) - expected = self.requirements['server']['cpu_total'] + expected = self.requirements['cpu_total'] info = f'Expected {expected} logical cores, actual {actual} cores' if actual < expected: - return self._error(name=name, info=info) + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) + @node_check def cpu_physical(self) -> CheckResult: name = 'cpu-physical' actual = psutil.cpu_count(logical=False) - expected = self.requirements['server']['cpu_physical'] + expected = self.requirements['cpu_physical'] info = f'Expected {expected} physical cores, actual {actual} cores' if actual < expected: - return self._error(name=name, info=info) + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) + @node_check def memory(self) -> CheckResult: name = 'memory' actual = psutil.virtual_memory().total, actual = actual[0] - expected = self.requirements['server']['memory'] + expected = self.requirements['memory'] actual_gb = round(actual / 1024 ** 3, 2) expected_gb = round(expected / 1024 ** 3, 2) info = f'Expected RAM {expected_gb} GB, actual {actual_gb} GB' if actual < expected: - return self._error(name=name, info=info) + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) + @node_check def swap(self) -> CheckResult: name = 'swap' actual = psutil.swap_memory().total - expected = self.requirements['server']['swap'] + expected = self.requirements['swap'] actual_gb = round(actual / 1024 ** 3, 2) expected_gb = round(expected / 1024 ** 3, 2) info = f'Expected swap memory {expected_gb} GB, actual {actual_gb} GB' if actual < expected: - return self._error(name=name, info=info) + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) + @node_check def network(self) -> CheckResult: name = 'network' try: @@ -131,52 +148,30 @@ def network(self) -> CheckResult: return self._ok(name=name) except socket.error as err: info = f'Network checking returned error: {err}' - return self._error(name=name, info=info) + return self._failed(name=name, info=info) class PackagesChecker(BaseChecker): def __init__(self, requirements: dict) -> None: self.requirements = requirements - def docker_compose(self) -> CheckResult: - name = 'docker-compose' - cmd = shutil.which('docker-compose') - if cmd is None: - info = 'No such command: "docker-compose"' - return self._error(name=name, info=info) - - v_cmd_result = run_cmd(['docker-compose', '-v'], check_code=False) - output = v_cmd_result.stdout.decode('utf-8').rstrip() - if v_cmd_result.returncode != 0: - output = v_cmd_result.stdout.decode('utf-8') - info = f'Checking docker-compose version failed with: {output}' - return self._error(name=name, info=output) - - actual_version = output.split(',')[0].split()[-1].strip() - expected_version = self.requirements['packages']['docker-compose'] - - info = { - 'expected_version': expected_version, - 'actual_version': actual_version - } - info = f'Expected docker-compose version {expected_version}, actual {actual_version}' # noqa - if version_parse(actual_version) < version_parse(expected_version): - return self._error(name=name, info=info) - else: - return self._ok(name=name, info=info) - + @node_check def iptables_persistent(self) -> CheckResult: return self._check_apt_package('iptables-persistent') + @node_check def lvm2(self) -> CheckResult: return self._check_apt_package('lvm2') + @node_check def btrfs_progs(self) -> CheckResult: return self._check_apt_package('btrfs-progs') + @node_check def lsof(self) -> CheckResult: return self._check_apt_package('lsof') + @node_check def psmisc(self) -> CheckResult: return self._check_apt_package('psmisc') @@ -195,10 +190,10 @@ def _check_apt_package(self, package_name: str, ['dpkg', '-s', package_name], check_code=False) output = dpkg_cmd_result.stdout.decode('utf-8').strip() if dpkg_cmd_result.returncode != 0: - return self._error(name=package_name, info=output) + return self._failed(name=package_name, info=output) actual_version = self._version_from_dpkg_output(output) - expected_version = self.requirements['packages'][package_name] + expected_version = self.requirements[package_name] info = { 'expected_version': expected_version, 'actual_version': actual_version @@ -207,7 +202,7 @@ def _check_apt_package(self, package_name: str, actual_version, expected_version ) if compare_result == -1: - return self._error(name=package_name, info=info) + return self._failed(name=package_name, info=info) else: return self._ok(name=package_name, info=info) @@ -226,47 +221,77 @@ def _get_docker_version_info(self) -> dict: except Exception as err: logger.error(f'Request to docker api failed {err}') + @node_check def docker_engine(self) -> CheckResult: name = 'docker-engine' if self._check_docker_command() is None: - return self._error(name=name, info='No such command: "docker"') + return self._failed(name=name, info='No such command: "docker"') version_info = self._get_docker_version_info() if not version_info: - return self._error( + return self._failed( name=name, info='Docker api request failed. Is docker installed?' ) actual_version = self.docker_client.version()['Version'] - expected_version = self.requirements['docker']['docker-engine'] + expected_version = self.requirements['docker-engine'] info = { 'expected_version': expected_version, 'actual_version': actual_version } if version_parse(actual_version) < version_parse(expected_version): - return self._error(name=name, info=info) + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) + @node_check def docker_api(self) -> CheckResult: name = 'docker-api' if self._check_docker_command() is None: - return self._error(name=name, info='No such command: "docker"') + return self._failed(name=name, info='No such command: "docker"') version_info = self._get_docker_version_info() if not version_info: - return self._error( + return self._failed( name=name, info='Docker api request failed. Is docker installed?' ) actual_version = version_info['ApiVersion'] - expected_version = self.requirements['docker']['docker-api'] + expected_version = self.requirements['docker-api'] info = { 'expected_version': expected_version, 'actual_version': actual_version } if version_parse(actual_version) < version_parse(expected_version): - return self._error(name=name, info=info) + return self._failed(name=name, info=info) + else: + return self._ok(name=name, info=info) + + @node_check + def docker_compose(self) -> CheckResult: + name = 'docker-compose' + cmd = shutil.which('docker-compose') + if cmd is None: + info = 'No such command: "docker-compose"' + return self._failed(name=name, info=info) + + v_cmd_result = run_cmd(['docker-compose', '-v'], check_code=False) + output = v_cmd_result.stdout.decode('utf-8').rstrip() + if v_cmd_result.returncode != 0: + output = v_cmd_result.stdout.decode('utf-8') + info = f'Checking docker-compose version failed with: {output}' + return self._failed(name=name, info=output) + + actual_version = output.split(',')[0].split()[-1].strip() + expected_version = self.requirements['docker-compose'] + + info = { + 'expected_version': expected_version, + 'actual_version': actual_version + } + info = f'Expected docker-compose version {expected_version}, actual {actual_version}' # noqa + if version_parse(actual_version) < version_parse(expected_version): + return self._failed(name=name, info=info) else: return self._ok(name=name, info=info) @@ -295,6 +320,7 @@ def _check_docker_alive_option(self, config: dict) -> tuple: info = 'Docker daemon live-restore option is set as "true"' return True, info + @node_check def keeping_containers_alive(self) -> CheckResult: name = 'live-restore' config = self._get_docker_config() @@ -302,4 +328,4 @@ def keeping_containers_alive(self) -> CheckResult: if is_ok: return self._ok(name=name, info=info) else: - return self._error(name=name, info=info) + return self._failed(name=name, info=info) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 95e9e417..4cf19e14 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -85,9 +85,9 @@ def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: logger.info('Checking that host meets requirements ...') requirements = get_env_params(env_type) checkers = [ - MachineChecker(requirements), - PackagesChecker(requirements), - DockerChecker(requirements) + MachineChecker(requirements['server']), + PackagesChecker(requirements['package']), + DockerChecker(requirements['docker']) ] result = [] for checker in checkers: diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index 0950486f..fd4ba353 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -16,8 +16,7 @@ def requirements_data(): 'memory': 100, 'swap': 100 }, - 'packages': { - 'docker-compose': '1.27.4', + 'package': { 'iptables_persistant': '0.0.0', 'lvm2': '0.0.0', 'test-package': '2.2.2' @@ -25,76 +24,96 @@ def requirements_data(): 'docker': { 'docker-engine': '0.0.0', 'docker-api': '0.0.0', + 'docker-compose': '1.27.4' } } -def test_checks_cpu_total(requirements_data): - checker = MachineChecker(requirements_data) +@pytest.fixture +def server_req(requirements_data): + return requirements_data['server'] + + +def test_checks_errored(): + checker = MachineChecker({}) + r = checker.check() + for c in r: + if c.name != 'network': + assert c.status == 'error', c.name + assert isinstance(c.info, KeyError), c.name + + +def test_checks_cpu_total(server_req): + checker = MachineChecker(server_req) r = checker.cpu_total() assert r.name == 'cpu-total' assert r.status == 'ok' - requirements_data['server']['cpu_total'] = 10000 # too big - checker = MachineChecker(requirements_data) + server_req['cpu_total'] = 10000 # too big + checker = MachineChecker(server_req) r = checker.cpu_total() assert r.name == 'cpu-total' - assert r.status == 'error' - assert checker.cpu_total().status == 'error' + assert r.status == 'failed' + assert checker.cpu_total().status == 'failed' -def test_checks_cpu_physical(requirements_data): - checker = MachineChecker(requirements_data) +def test_checks_cpu_physical(server_req): + checker = MachineChecker(server_req) r = checker.cpu_physical() assert r.name == 'cpu-physical' assert r.status == 'ok' - requirements_data['server']['cpu_physical'] = 10000 # too big - checker = MachineChecker(requirements_data) + server_req['cpu_physical'] = 10000 # too big + checker = MachineChecker(server_req) r = checker.cpu_physical() assert r.name == 'cpu-physical' - assert r.status == 'error' + assert r.status == 'failed' -def test_checks_memory(requirements_data): - checker = MachineChecker(requirements_data) +def test_checks_memory(server_req): + checker = MachineChecker(server_req) r = checker.memory() assert r.name == 'memory' assert r.status == 'ok' # too big - requirements_data['server']['memory'] = 10000000000000 - checker = MachineChecker(requirements_data) + server_req['memory'] = 10000000000000 + checker = MachineChecker(server_req) r = checker.memory() assert r.name == 'memory' - assert r.status == 'error' + assert r.status == 'failed' -def test_checks_machine_check(requirements_data): - checker = MachineChecker(requirements_data) - result = checker.check() - assert any([r.status == 'ok' for r in result]) - - -def test_checks_swap(requirements_data): - checker = MachineChecker(requirements_data) +def test_checks_swap(server_req): + checker = MachineChecker(server_req) r = checker.swap() assert r.name == 'swap' assert r.status == 'ok' # too big - requirements_data['server']['swap'] = 10000000000000 - checker = MachineChecker(requirements_data) + server_req['swap'] = 10000000000000 + checker = MachineChecker(server_req) r = checker.swap() assert r.name == 'swap' - assert r.status == 'error' + assert r.status == 'failed' -def test_checks_network(requirements_data): - checker = MachineChecker(requirements_data) +def test_checks_network(server_req): + checker = MachineChecker(server_req) r = checker.network() assert r.status == 'ok' assert r.name == 'network' -def test_checks_docker_engine(requirements_data): - checker = DockerChecker(requirements_data) +def test_checks_machine_check(server_req): + checker = MachineChecker(server_req) + result = checker.check() + assert any([r.status == 'ok' for r in result]) + + +@pytest.fixture +def docker_req(requirements_data): + return requirements_data['docker'] + + +def test_checks_docker_engine(docker_req): + checker = DockerChecker(docker_req) r = checker.docker_engine() assert r.name == 'docker-engine' @@ -103,18 +122,18 @@ def test_checks_docker_engine(requirements_data): with mock.patch('shutil.which', return_value=None): r = checker.docker_engine() assert r.name == 'docker-engine' - assert r.status == 'error' + assert r.status == 'failed' assert r.info == 'No such command: "docker"' - requirements_data['docker']['docker-engine'] = '111.111.111' + docker_req['docker-engine'] = '111.111.111' r = checker.docker_engine() assert r.name == 'docker-engine' - assert r.status == 'error' + assert r.status == 'failed' assert r.info['expected_version'] == '111.111.111' -def test_checks_docker_api(requirements_data): - checker = DockerChecker(requirements_data) +def test_checks_docker_api(docker_req): + checker = DockerChecker(docker_req) r = checker.docker_api() assert r.name == 'docker-api' @@ -123,50 +142,16 @@ def test_checks_docker_api(requirements_data): with mock.patch('shutil.which', return_value=None): r = checker.docker_api() assert r.name == 'docker-api' - assert r.status == 'error' + assert r.status == 'failed' assert r.info == 'No such command: "docker"' - requirements_data['docker']['docker-api'] = '111.111.111' + docker_req['docker-api'] = '111.111.111' r = checker.docker_api() assert r.name == 'docker-api' - assert r.status == 'error' + assert r.status == 'failed' assert r.info['expected_version'] == '111.111.111' -def test_checks_apt_package(requirements_data): - checker = PackagesChecker(requirements_data) - res_mock = mock.Mock() - res_mock.stdout = b"""Package: test-package - Version: 5.2.1-2 - """ - - def run_cmd_mock(*args, **kwargs): - return res_mock - - res_mock.returncode = 0 - apt_package_name = 'test-package' - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker._check_apt_package(apt_package_name) - assert r.name == apt_package_name - assert r.status == 'ok' - - res_mock.stdout = b"""Package: test-package - Version: 1.1.1 - """ - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker._check_apt_package(apt_package_name) - assert r.name == 'test-package' - assert r.status == 'error' - - res_mock.stdout = b"""Package: test-package - Version: 2.2.2 - """ - with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): - r = checker._check_apt_package(apt_package_name) - assert r.name == 'test-package' - assert r.status == 'ok' - - @pytest.fixture def docker_compose_pkg_1_27_4(): pipmain(['install', 'docker-compose==1.27.4']) @@ -183,9 +168,8 @@ def docker_compose_pkg_1_24_1(): pipmain(['uninstall', 'docker-compose', '-y']) -def test_checks_docker_compose_valid_pkg( - requirements_data, docker_compose_pkg_1_27_4): - checker = PackagesChecker(requirements_data) +def test_checks_docker_compose_good_pkg(docker_req, docker_compose_pkg_1_27_4): + checker = DockerChecker(package_req) print('Debug: ', checker.docker_compose()) r = checker.docker_compose() @@ -193,24 +177,25 @@ def test_checks_docker_compose_valid_pkg( r.status == 'ok' -def test_checks_docker_compose_no_pkg( - requirements_data): - checker = PackagesChecker(requirements_data) +def test_checks_docker_compose_no_pkg(docker_req): + checker = DockerChecker(package_req) r = checker.docker_compose() r.name == 'docker-compose' r.status == 'ok' def test_checks_docker_compose_invalid_version( - requirements_data, docker_compose_pkg_1_24_1): - checker = PackagesChecker(requirements_data) + docker_req, + docker_compose_pkg_1_24_1 +): + checker = DockerChecker(docker_req) r = checker.docker_compose() r.name == 'docker-compose' r.status == 'ok' -def test_checks_docker_config(): - checker = DockerChecker(requirements_data) +def test_checks_docker_config(docker_req): + checker = DockerChecker(docker_req) valid_config = { 'live-restore': True } @@ -228,3 +213,42 @@ def test_checks_docker_config(): r = checker._check_docker_alive_option({}) assert r[0] is False assert r[1] == 'Docker daemon live-restore option should be set as "true"' + + +@pytest.fixture +def package_req(requirements_data): + return requirements_data['package'] + + +def test_checks_apt_package(package_req): + checker = PackagesChecker(package_req) + res_mock = mock.Mock() + res_mock.stdout = b"""Package: test-package + Version: 5.2.1-2 + """ + + def run_cmd_mock(*args, **kwargs): + return res_mock + + res_mock.returncode = 0 + apt_package_name = 'test-package' + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): + r = checker._check_apt_package(apt_package_name) + assert r.name == apt_package_name + assert r.status == 'ok' + + res_mock.stdout = b"""Package: test-package + Version: 1.1.1 + """ + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): + r = checker._check_apt_package(apt_package_name) + assert r.name == 'test-package' + assert r.status == 'failed' + + res_mock.stdout = b"""Package: test-package + Version: 2.2.2 + """ + with mock.patch('node_cli.core.checks.run_cmd', run_cmd_mock): + r = checker._check_apt_package(apt_package_name) + assert r.name == 'test-package' + assert r.status == 'ok' From 93148661b584540fcaa71e605d13a691311e0833 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 16 Jun 2021 14:50:16 +0300 Subject: [PATCH 130/140] Hotfix Uncomment check in base operations --- node_cli/operations/base.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 97132831..304bafcb 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -50,6 +50,12 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) sync_skale_node(env) + + failed_checks = run_preinstall_checks(env['ENV_TYPE']) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + download_filestorage_artifacts() docker_lvmpy_update(env) generate_nginx_config() @@ -74,10 +80,10 @@ def update(env_filepath: str, env: Dict) -> None: def init(env_filepath: str, env: str) -> bool: sync_skale_node(env) - # failed_checks = run_preinstall_checks(env['ENV_TYPE']) - # if failed_checks: - # print_failed_requirements_checks(failed_checks) - # return False + failed_checks = run_preinstall_checks(env['ENV_TYPE']) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False prepare_host( env_filepath, env['DISK_MOUNTPOINT'], @@ -121,6 +127,12 @@ def turn_on(env): def restore(env, backup_path): unpack_backup_archive(backup_path) + + failed_checks = run_preinstall_checks(env['ENV_TYPE']) + if failed_checks: + print_failed_requirements_checks(failed_checks) + return False + link_env_file() configure_iptables() docker_lvmpy_install(env) From ca8aa4eb0fc40930e2c4bc072965a0bc1c620fc8 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 17 Jun 2021 08:20:53 +0300 Subject: [PATCH 131/140] Hotfix fix checking procedure --- node_cli/core/checks.py | 13 +++++++++---- node_cli/core/host.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 0fb949c3..56d760db 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -63,8 +63,12 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as err: - logger.error('%s check errored with %s', func.__name__, err) - return CheckResult(name=func.__name__, status='error', info=err) + logger.exception('%s check errored') + return CheckResult( + name=func.__name__, + status='error', + info=repr(err) + ) return wrapper @@ -233,6 +237,7 @@ def docker_engine(self) -> CheckResult: name=name, info='Docker api request failed. Is docker installed?' ) + logger.info('Docker version info %s', version_info) actual_version = self.docker_client.version()['Version'] expected_version = self.requirements['docker-engine'] info = { @@ -256,6 +261,7 @@ def docker_api(self) -> CheckResult: name=name, info='Docker api request failed. Is docker installed?' ) + logger.info('Docker version info %s', version_info) actual_version = version_info['ApiVersion'] expected_version = self.requirements['docker-api'] info = { @@ -309,8 +315,7 @@ def _get_docker_config(self) -> dict: def _check_docker_alive_option(self, config: dict) -> tuple: actual_value = config.get('live-restore', None) - expected_value = True - if actual_value != expected_value: + if actual_value is not True: info = ( 'Docker daemon live-restore option ' 'should be set as "true"' diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 4cf19e14..4c748599 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -91,7 +91,7 @@ def run_preinstall_checks(env_type: str = 'mainnet') -> ListChecks: ] result = [] for checker in checkers: - result.extend(filter(lambda r: r.status == 'error', checker.check())) + result.extend(filter(lambda r: r.status != 'ok', checker.check())) if result: logger.info('Host is not fully meet the requirements') return result From a317709a418da00d188119c27634b0a632605203 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 17 Jun 2021 12:36:18 +0300 Subject: [PATCH 132/140] Hotfix Disable check in update --- node_cli/cli/node.py | 2 +- node_cli/core/node.py | 6 +++--- node_cli/operations/base.py | 12 +++++++----- tests/cli/node_test.py | 2 +- tests/core_checks_test.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index a3d94f20..d4708a56 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -244,7 +244,7 @@ def _set_domain_name(domain): default='mainnet', help='Network to check' ) -def check_requirements(network): +def check(network): run_checks(network) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 91dd83e7..d474858d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -132,7 +132,7 @@ def init(env_filepath): exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR ) return - logger.info('Waiting for transaction manager initialization') + logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): error_exit( @@ -189,7 +189,7 @@ def update(env_filepath): configure_firewall_rules() env = get_node_env(env_filepath, inited_node=True, sync_schains=False) update_op(env_filepath, env) - logger.info('Waiting for transaction manager initialization') + logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): print_node_cmd_error() @@ -316,7 +316,7 @@ def turn_off(maintenance_on): def turn_on(maintenance_off, sync_schains, env_file): env = get_node_env(env_file, inited_node=True, sync_schains=sync_schains) turn_on_op(env) - logger.info('Waiting for transaction manager initialization') + logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(): print_node_cmd_error() diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 304bafcb..5de3cbdc 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -37,7 +37,7 @@ from node_cli.core.iptables import configure_iptables from node_cli.utils.docker_utils import compose_rm, compose_up, remove_dynamic_containers from node_cli.utils.meta import update_meta -from node_cli.utils.print_formatters import print_failed_requirements_checks # noqa +from node_cli.utils.print_formatters import print_failed_requirements_checks logger = logging.getLogger(__name__) @@ -51,10 +51,10 @@ def update(env_filepath: str, env: Dict) -> None: download_contracts(env) sync_skale_node(env) - failed_checks = run_preinstall_checks(env['ENV_TYPE']) - if failed_checks: - print_failed_requirements_checks(failed_checks) - return False + # failed_checks = run_preinstall_checks(env['ENV_TYPE']) + # if failed_checks: + # print_failed_requirements_checks(failed_checks) + # return False download_filestorage_artifacts() docker_lvmpy_update(env) @@ -80,10 +80,12 @@ def update(env_filepath: str, env: Dict) -> None: def init(env_filepath: str, env: str) -> bool: sync_skale_node(env) + failed_checks = run_preinstall_checks(env['ENV_TYPE']) if failed_checks: print_failed_requirements_checks(failed_checks) return False + prepare_host( env_filepath, env['DISK_MOUNTPOINT'], diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 69491e7b..8cf08f70 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -157,7 +157,7 @@ def test_update_node(mocked_g_config): params, input='/dev/sdp') assert result.exit_code == 0 - # assert result.output == 'Updating the node...\nWaiting for transaction manager initialization ...\nUpdate procedure finished\n' # noqa + # assert result.output == 'Updating the node...\nWaiting for containers initialization ...\nUpdate procedure finished\n' # noqa def test_update_node_without_init(): diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index fd4ba353..440ba13b 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -40,7 +40,7 @@ def test_checks_errored(): for c in r: if c.name != 'network': assert c.status == 'error', c.name - assert isinstance(c.info, KeyError), c.name + assert c.info.startswith('KeyError'), c.name def test_checks_cpu_total(server_req): From 87400b179ea6b126cd50cdfdf9fd3623ac969ec8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 18 Jun 2021 15:52:18 +0300 Subject: [PATCH 133/140] Fix SSL tests --- tests/core_ssl_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index 6b28c577..0a769244 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -4,8 +4,8 @@ import mock import pytest -from core.ssl import check_cert_openssl, SSLHealthcheckError, upload_cert -from tools.helper import run_cmd +from node_cli.core.ssl import check_cert_openssl, SSLHealthcheckError, upload_cert +from node_cli.utils.helper import run_cmd HOST = '127.0.0.1' @@ -72,7 +72,7 @@ def test_verify_cert_bad_key(bad_key): check_cert_openssl(cert, key, host=HOST, no_client=True) -@mock.patch('core.ssl.post_request') +@mock.patch('node_cli.core.ssl.post_request') def test_upload_cert(pr_mock, cert_key_pair): cert, key = cert_key_pair upload_cert(cert, key, force=False, no_client=True) From e60234f2ccababbecddc448a8533df239f658fe2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 18 Jun 2021 18:50:31 +0300 Subject: [PATCH 134/140] Fix nginx template processing --- node_cli/core/ssl.py | 11 ++++++++--- node_cli/utils/helper.py | 8 +++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/node_cli/core/ssl.py b/node_cli/core/ssl.py index 55464342..9895ac2f 100644 --- a/node_cli/core/ssl.py +++ b/node_cli/core/ssl.py @@ -13,7 +13,7 @@ SSL_CERT_FILEPATH, SSL_KEY_FILEPATH ) -from node_cli.utils.helper import post_request, read_file +from node_cli.utils.helper import post_request from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) @@ -21,12 +21,17 @@ COMMUNICATION_TIMEOUT = 3 +def read_file_bytes(path, mode='rb'): + with open(path, mode) as f: + return f + + def load_ssl_files(key_path, cert_path): return { 'ssl_key': (os.path.basename(key_path), - read_file(key_path), 'application/octet-stream'), + read_file_bytes(key_path), 'application/octet-stream'), 'ssl_cert': (os.path.basename(cert_path), - read_file(cert_path), 'application/octet-stream') + read_file_bytes(cert_path), 'application/octet-stream') } diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 5f8afd1c..f163b832 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -258,9 +258,11 @@ def get_file_handler(log_filepath, log_level): return f_handler -def read_file(path, mode='rb'): - with open(path, mode) as f: - return f +def read_file(path): + file = open(path, 'r') + text = file.read() + file.close() + return text def to_camel_case(snake_str): From 6b7f8afd16981e1d0d0c352fd44f8c33b7b1edba Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 18 Jun 2021 18:55:33 +0300 Subject: [PATCH 135/140] Hotfix: Use context manager to read file --- node_cli/utils/helper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index f163b832..0eae5198 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -259,10 +259,8 @@ def get_file_handler(log_filepath, log_level): def read_file(path): - file = open(path, 'r') - text = file.read() - file.close() - return text + with open(path, 'r') as file: + return file.read() def to_camel_case(snake_str): From 4e9706d9cab5a57b3789ea5a86c522f98bdfd2ff Mon Sep 17 00:00:00 2001 From: Dmytro Nazarenko Date: Tue, 22 Jun 2021 13:44:32 +0300 Subject: [PATCH 136/140] SKALE-4330 Change nginx config dir --- node_cli/configs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 7b49ac34..0161c84d 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -51,7 +51,7 @@ FILESTORAGE_ARTIFACTS_FILE = os.path.join(NODE_DATA_PATH, 'filestorage_artifacts.json') ENVIRONMENT_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'environment_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') -NGINX_CONFIG_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf') +NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' From 34e0354a2dc539350c5bee24d3cb9604f91570a6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 7 Jul 2021 16:57:37 +0300 Subject: [PATCH 137/140] SKALE-4369 Change medium sChain size to 1/8 --- node_cli/configs/resource_allocation.py | 4 +- tests/.skale/config/environment_params.yaml | 77 ++--- tests/.skale/config/schain_allocation.yml | 308 ++++++++++---------- tests/resources_test.py | 65 +++-- 4 files changed, 234 insertions(+), 220 deletions(-) diff --git a/node_cli/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py index c38ca271..e51d6ec9 100644 --- a/node_cli/configs/resource_allocation.py +++ b/node_cli/configs/resource_allocation.py @@ -21,8 +21,8 @@ from node_cli.configs import NODE_DATA_PATH LARGE_DIVIDER = 1 -MEDIUM_DIVIDER = 32 -TEST_DIVIDER = 32 +MEDIUM_DIVIDER = 8 +TEST_DIVIDER = 8 SMALL_DIVIDER = 128 TIMES = 1 diff --git a/tests/.skale/config/environment_params.yaml b/tests/.skale/config/environment_params.yaml index d36a1c4d..1711bb6a 100644 --- a/tests/.skale/config/environment_params.yaml +++ b/tests/.skale/config/environment_params.yaml @@ -15,57 +15,67 @@ common: contract_storage: 0.6 db_storage: 0.4 base_rotate_after_block_divider: 61035.1 + shared_space_coefficient: 1 envs: mainnet: server: cpu_total: 8 - cpu_physical: 8 + cpu_physical: 1 memory: 32000000000 swap: 16000000000 disk: 1900000000000 - packages: - docker: 20.10 - docker-compose: 1.27.4 - iptables-persistent: 1.1.3 - lvm2: 2.02.175 + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 btrfs-progs: 4.15.1 - lsof: 4.89 - psmisc: 23.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 testnet: server: cpu_total: 8 - cpu_physical: 8 + cpu_physical: 1 memory: 32000000000 swap: 16000000000 disk: 200000000000 - packages: - docker: 20.10 - docker-compose: 1.27.4 - iptables-persistent: 1.1.3 - lvm2: 2.02.175 + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 btrfs-progs: 4.15.1 - lsof: 4.89 - psmisc: 23.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 qanet: server: - cpu_total: 4 - cpu_physical: 4 + cpu_total: 8 + cpu_physical: 1 memory: 32000000000 swap: 16000000000 disk: 200000000000 - packages: - docker: 20.10 - docker-compose: 1.27.4 - iptables-persistent: 1.1.3 - lvm2: 2.02.175 + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 btrfs-progs: 4.15.1 - lsof: 4.89 - psmisc: 23.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 devnet: server: @@ -75,15 +85,14 @@ envs: swap: 2000000000 disk: 80000000000 - packages: - docker: 0.0.0 - docker-compose: 1.27.4 - iptables-persistent: 0.0.0 - lvm2: 0.0.0 + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 btrfs-progs: 4.15.1 - lsof: 0.0.0 - psmisc: 0.0.0 + lsof: "4.89" + psmisc: 23.1-1 docker: - docker-api: 0.0.0 - docker-engine: 0.0.0 \ No newline at end of file + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 diff --git a/tests/.skale/config/schain_allocation.yml b/tests/.skale/config/schain_allocation.yml index 716eef0f..0aebc880 100644 --- a/tests/.skale/config/schain_allocation.yml +++ b/tests/.skale/config/schain_allocation.yml @@ -3,221 +3,225 @@ devnet: disk: - large: 75999936512 - medium: 2374998016 - small: 593749504 - test: 2374998016 - test4: 2374998016 + large: 71039975424 + medium: 8879996928 + small: 554999808 + test: 8879996928 + test4: 8879996928 leveldb_limits: large: - contract_storage: 13679988571 - db_storage: 9119992381 + contract_storage: 12787195576 + db_storage: 8524797050 medium: - contract_storage: 427499642 - db_storage: 284999761 + contract_storage: 1598399446 + db_storage: 1065599631 small: - contract_storage: 106874910 - db_storage: 71249940 + contract_storage: 99899965 + db_storage: 66599976 test: - contract_storage: 427499642 - db_storage: 284999761 + contract_storage: 1598399446 + db_storage: 1065599631 test4: - contract_storage: 427499642 - db_storage: 284999761 + contract_storage: 1598399446 + db_storage: 1065599631 rotate_after_block: large: 1310721 - medium: 40960 + medium: 163840 small: 10240 - test: 40960 - test4: 40960 + test: 163840 + test4: 163840 + shared_space: 8959950848 volume_limits: large: - max_consensus_storage_bytes: 22799980953 - max_file_storage_bytes: 22799980953 - max_reserved_storage_bytes: 7599993651 - max_skaled_leveldb_storage_bytes: 22799980953 + max_consensus_storage_bytes: 21311992627 + max_file_storage_bytes: 21311992627 + max_reserved_storage_bytes: 7103997542 + max_skaled_leveldb_storage_bytes: 21311992627 medium: - max_consensus_storage_bytes: 712499404 - max_file_storage_bytes: 712499404 - max_reserved_storage_bytes: 237499801 - max_skaled_leveldb_storage_bytes: 712499404 + max_consensus_storage_bytes: 2663999078 + max_file_storage_bytes: 2663999078 + max_reserved_storage_bytes: 887999692 + max_skaled_leveldb_storage_bytes: 2663999078 small: - max_consensus_storage_bytes: 178124851 - max_file_storage_bytes: 178124851 - max_reserved_storage_bytes: 59374950 - max_skaled_leveldb_storage_bytes: 178124851 + max_consensus_storage_bytes: 166499942 + max_file_storage_bytes: 166499942 + max_reserved_storage_bytes: 55499980 + max_skaled_leveldb_storage_bytes: 166499942 test: - max_consensus_storage_bytes: 712499404 - max_file_storage_bytes: 712499404 - max_reserved_storage_bytes: 237499801 - max_skaled_leveldb_storage_bytes: 712499404 + max_consensus_storage_bytes: 2663999078 + max_file_storage_bytes: 2663999078 + max_reserved_storage_bytes: 887999692 + max_skaled_leveldb_storage_bytes: 2663999078 test4: - max_consensus_storage_bytes: 712499404 - max_file_storage_bytes: 712499404 - max_reserved_storage_bytes: 237499801 - max_skaled_leveldb_storage_bytes: 712499404 + max_consensus_storage_bytes: 2663999078 + max_file_storage_bytes: 2663999078 + max_reserved_storage_bytes: 887999692 + max_skaled_leveldb_storage_bytes: 2663999078 mainnet: disk: - large: 1804999983104 - medium: 56406249472 - small: 14101562368 - test: 56406249472 - test4: 56406249472 + large: 1687199940608 + medium: 210899992576 + small: 13181249536 + test: 210899992576 + test4: 210899992576 leveldb_limits: large: - contract_storage: 324899996958 - db_storage: 216599997972 + contract_storage: 303695989309 + db_storage: 202463992872 medium: - contract_storage: 10153124904 - db_storage: 6768749936 + contract_storage: 37961998663 + db_storage: 25307999108 small: - contract_storage: 2538281226 - db_storage: 1692187484 + contract_storage: 2372624916 + db_storage: 1581749944 test: - contract_storage: 10153124904 - db_storage: 6768749936 + contract_storage: 37961998663 + db_storage: 25307999108 test4: - contract_storage: 10153124904 - db_storage: 6768749936 + contract_storage: 37961998663 + db_storage: 25307999108 rotate_after_block: large: 31129628 - medium: 972800 + medium: 3891203 small: 243200 - test: 972800 - test4: 972800 + test: 3891203 + test4: 3891203 + shared_space: 212799979520 volume_limits: large: - max_consensus_storage_bytes: 541499994931 - max_file_storage_bytes: 541499994931 - max_reserved_storage_bytes: 180499998310 - max_skaled_leveldb_storage_bytes: 541499994931 + max_consensus_storage_bytes: 506159982182 + max_file_storage_bytes: 506159982182 + max_reserved_storage_bytes: 168719994060 + max_skaled_leveldb_storage_bytes: 506159982182 medium: - max_consensus_storage_bytes: 16921874841 - max_file_storage_bytes: 16921874841 - max_reserved_storage_bytes: 5640624947 - max_skaled_leveldb_storage_bytes: 16921874841 + max_consensus_storage_bytes: 63269997772 + max_file_storage_bytes: 63269997772 + max_reserved_storage_bytes: 21089999257 + max_skaled_leveldb_storage_bytes: 63269997772 small: - max_consensus_storage_bytes: 4230468710 - max_file_storage_bytes: 4230468710 - max_reserved_storage_bytes: 1410156236 - max_skaled_leveldb_storage_bytes: 4230468710 + max_consensus_storage_bytes: 3954374860 + max_file_storage_bytes: 3954374860 + max_reserved_storage_bytes: 1318124953 + max_skaled_leveldb_storage_bytes: 3954374860 test: - max_consensus_storage_bytes: 16921874841 - max_file_storage_bytes: 16921874841 - max_reserved_storage_bytes: 5640624947 - max_skaled_leveldb_storage_bytes: 16921874841 + max_consensus_storage_bytes: 63269997772 + max_file_storage_bytes: 63269997772 + max_reserved_storage_bytes: 21089999257 + max_skaled_leveldb_storage_bytes: 63269997772 test4: - max_consensus_storage_bytes: 16921874841 - max_file_storage_bytes: 16921874841 - max_reserved_storage_bytes: 5640624947 - max_skaled_leveldb_storage_bytes: 16921874841 + max_consensus_storage_bytes: 63269997772 + max_file_storage_bytes: 63269997772 + max_reserved_storage_bytes: 21089999257 + max_skaled_leveldb_storage_bytes: 63269997772 qanet: disk: - large: 189999939584 - medium: 5937498112 - small: 1484374528 - test: 5937498112 - test4: 5937498112 + large: 177599938560 + medium: 22199992320 + small: 1387499520 + test: 22199992320 + test4: 22199992320 leveldb_limits: large: - contract_storage: 34199989125 - db_storage: 22799992750 + contract_storage: 31967988940 + db_storage: 21311992627 medium: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 small: - contract_storage: 267187414 - db_storage: 178124943 + contract_storage: 249749913 + db_storage: 166499942 test: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 test4: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 rotate_after_block: large: 3276803 - medium: 102400 + medium: 409600 small: 25600 - test: 102400 - test4: 102400 + test: 409600 + test4: 409600 + shared_space: 22399942656 volume_limits: large: - max_consensus_storage_bytes: 56999981875 - max_file_storage_bytes: 56999981875 - max_reserved_storage_bytes: 18999993958 - max_skaled_leveldb_storage_bytes: 56999981875 + max_consensus_storage_bytes: 53279981568 + max_file_storage_bytes: 53279981568 + max_reserved_storage_bytes: 17759993856 + max_skaled_leveldb_storage_bytes: 53279981568 medium: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 small: - max_consensus_storage_bytes: 445312358 - max_file_storage_bytes: 445312358 - max_reserved_storage_bytes: 148437452 - max_skaled_leveldb_storage_bytes: 445312358 + max_consensus_storage_bytes: 416249856 + max_file_storage_bytes: 416249856 + max_reserved_storage_bytes: 138749952 + max_skaled_leveldb_storage_bytes: 416249856 test: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 test4: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 testnet: disk: - large: 189999939584 - medium: 5937498112 - small: 1484374528 - test: 5937498112 - test4: 5937498112 + large: 177599938560 + medium: 22199992320 + small: 1387499520 + test: 22199992320 + test4: 22199992320 leveldb_limits: large: - contract_storage: 34199989125 - db_storage: 22799992750 + contract_storage: 31967988940 + db_storage: 21311992627 medium: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 small: - contract_storage: 267187414 - db_storage: 178124943 + contract_storage: 249749913 + db_storage: 166499942 test: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 test4: - contract_storage: 1068749659 - db_storage: 712499773 + contract_storage: 3995998617 + db_storage: 2663999078 rotate_after_block: large: 3276803 - medium: 102400 + medium: 409600 small: 25600 - test: 102400 - test4: 102400 + test: 409600 + test4: 409600 + shared_space: 22399942656 volume_limits: large: - max_consensus_storage_bytes: 56999981875 - max_file_storage_bytes: 56999981875 - max_reserved_storage_bytes: 18999993958 - max_skaled_leveldb_storage_bytes: 56999981875 + max_consensus_storage_bytes: 53279981568 + max_file_storage_bytes: 53279981568 + max_reserved_storage_bytes: 17759993856 + max_skaled_leveldb_storage_bytes: 53279981568 medium: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 small: - max_consensus_storage_bytes: 445312358 - max_file_storage_bytes: 445312358 - max_reserved_storage_bytes: 148437452 - max_skaled_leveldb_storage_bytes: 445312358 + max_consensus_storage_bytes: 416249856 + max_file_storage_bytes: 416249856 + max_reserved_storage_bytes: 138749952 + max_skaled_leveldb_storage_bytes: 416249856 test: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 test4: - max_consensus_storage_bytes: 1781249433 - max_file_storage_bytes: 1781249433 - max_reserved_storage_bytes: 593749811 - max_skaled_leveldb_storage_bytes: 1781249433 + max_consensus_storage_bytes: 6659997696 + max_file_storage_bytes: 6659997696 + max_reserved_storage_bytes: 2219999232 + max_skaled_leveldb_storage_bytes: 6659997696 diff --git a/tests/resources_test.py b/tests/resources_test.py index 701689e9..d7b11cc4 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -14,7 +14,7 @@ from node_cli.utils.helper import write_json, safe_load_yml -SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 22799980953, 'max_file_storage_bytes': 22799980953, 'max_reserved_storage_bytes': 7599993651, 'max_skaled_leveldb_storage_bytes': 22799980953}, 'medium': {'max_consensus_storage_bytes': 712499404, 'max_file_storage_bytes': 712499404, 'max_reserved_storage_bytes': 237499801, 'max_skaled_leveldb_storage_bytes': 712499404}, 'small': {'max_consensus_storage_bytes': 178124851, 'max_file_storage_bytes': 178124851, 'max_reserved_storage_bytes': 59374950, 'max_skaled_leveldb_storage_bytes': 178124851}, 'test': {'max_consensus_storage_bytes': 712499404, 'max_file_storage_bytes': 712499404, 'max_reserved_storage_bytes': 237499801, 'max_skaled_leveldb_storage_bytes': 712499404}, 'test4': {'max_consensus_storage_bytes': 712499404, 'max_file_storage_bytes': 712499404, 'max_reserved_storage_bytes': 237499801, 'max_skaled_leveldb_storage_bytes': 712499404}} # noqa +SCHAIN_VOLUME_PARTS = {'large': {'max_consensus_storage_bytes': 21311992627, 'max_file_storage_bytes': 21311992627, 'max_reserved_storage_bytes': 7103997542, 'max_skaled_leveldb_storage_bytes': 21311992627}, 'medium': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'small': {'max_consensus_storage_bytes': 166499942, 'max_file_storage_bytes': 166499942, 'max_reserved_storage_bytes': 55499980, 'max_skaled_leveldb_storage_bytes': 166499942}, 'test': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}, 'test4': {'max_consensus_storage_bytes': 2663999078, 'max_file_storage_bytes': 2663999078, 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078}} # noqa DEFAULT_ENV_TYPE = 'devnet' @@ -48,10 +48,10 @@ def test_generate_resource_allocation_config(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['cpu_shares']['test4'] == 22 - assert resource_allocation_config['schain']['cpu_shares']['test'] == 22 + assert resource_allocation_config['schain']['cpu_shares']['test4'] == 89 + assert resource_allocation_config['schain']['cpu_shares']['test'] == 89 assert resource_allocation_config['schain']['cpu_shares']['small'] == 5 - assert resource_allocation_config['schain']['cpu_shares']['medium'] == 22 + assert resource_allocation_config['schain']['cpu_shares']['medium'] == 89 assert resource_allocation_config['schain']['cpu_shares']['large'] == 716 assert isinstance(resource_allocation_config['schain']['mem']['test4'], int) @@ -60,13 +60,13 @@ def test_generate_resource_allocation_config(): assert isinstance(resource_allocation_config['schain']['mem']['medium'], int) assert isinstance(resource_allocation_config['schain']['mem']['large'], int) - assert resource_allocation_config['schain']['disk']['test4'] == 2374998016 - assert resource_allocation_config['schain']['disk']['test'] == 2374998016 - assert resource_allocation_config['schain']['disk']['small'] == 593749504 - assert resource_allocation_config['schain']['disk']['medium'] == 2374998016 - assert resource_allocation_config['schain']['disk']['large'] == 75999936512 + assert resource_allocation_config['schain']['disk']['test4'] == 8879996928 + assert resource_allocation_config['schain']['disk']['test'] == 8879996928 + assert resource_allocation_config['schain']['disk']['small'] == 554999808 + assert resource_allocation_config['schain']['disk']['medium'] == 8879996928 + assert resource_allocation_config['schain']['disk']['large'] == 71039975424 - assert resource_allocation_config['ima']['cpu_shares'] == {'test4': 9, 'test': 9, 'small': 2, 'medium': 9, 'large': 307} # noqa + assert resource_allocation_config['ima']['cpu_shares'] == {'large': 307, 'medium': 38, 'small': 2, 'test': 38, 'test4': 38} # noqa assert isinstance(resource_allocation_config['ima']['mem'], dict) assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS @@ -91,11 +91,11 @@ def test_get_static_disk_alloc_devnet(env_configs, schain_allocation_data): verify_disk_size(env_configs, DEFAULT_ENV_TYPE) assert schain_allocation_data[DEFAULT_ENV_TYPE]['disk'] == { - 'test4': 2374998016, - 'test': 2374998016, - 'small': 593749504, - 'medium': 2374998016, - 'large': 75999936512 + 'large': 71039975424, + 'medium': 8879996928, + 'small': 554999808, + 'test': 8879996928, + 'test4': 8879996928 } @@ -116,16 +116,16 @@ def test_get_cpu_alloc(): schain_cpu_alloc_dict = schain_cpu_alloc.dict() ima_cpu_alloc_dict = ima_cpu_alloc.dict() - assert schain_cpu_alloc_dict['test4'] == 22 - assert schain_cpu_alloc_dict['test'] == 22 + assert schain_cpu_alloc_dict['test4'] == 89 + assert schain_cpu_alloc_dict['test'] == 89 assert schain_cpu_alloc_dict['small'] == 5 - assert schain_cpu_alloc_dict['medium'] == 22 + assert schain_cpu_alloc_dict['medium'] == 89 assert schain_cpu_alloc_dict['large'] == 716 - assert ima_cpu_alloc_dict['test4'] == 9 - assert ima_cpu_alloc_dict['test'] == 9 + assert ima_cpu_alloc_dict['test4'] == 38 + assert ima_cpu_alloc_dict['test'] == 38 assert ima_cpu_alloc_dict['small'] == 2 - assert ima_cpu_alloc_dict['medium'] == 9 + assert ima_cpu_alloc_dict['medium'] == 38 assert ima_cpu_alloc_dict['large'] == 307 @@ -136,26 +136,27 @@ def test_get_memory_alloc(): schain_mem_alloc_dict = schain_mem_alloc.dict() ima_mem_alloc_dict = ima_mem_alloc.dict() - assert schain_mem_alloc_dict['test4'] == 218750 - assert schain_mem_alloc_dict['test'] == 218750 + assert schain_mem_alloc_dict['test4'] == 875000 + assert schain_mem_alloc_dict['test'] == 875000 assert schain_mem_alloc_dict['small'] == 54687 - assert schain_mem_alloc_dict['medium'] == 218750 + assert schain_mem_alloc_dict['medium'] == 875000 assert schain_mem_alloc_dict['large'] == 7000000 - assert ima_mem_alloc_dict['test4'] == 93750 - assert ima_mem_alloc_dict['test'] == 93750 + assert ima_mem_alloc_dict['test4'] == 375000 + assert ima_mem_alloc_dict['test'] == 375000 assert ima_mem_alloc_dict['small'] == 23437 - assert ima_mem_alloc_dict['medium'] == 93750 + assert ima_mem_alloc_dict['medium'] == 375000 assert ima_mem_alloc_dict['large'] == 3000000 def test_leveldb_limits(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) + assert resource_allocation_config['schain']['leveldb_limits'] == { - 'large': {'contract_storage': 13679988571, 'db_storage': 9119992381}, - 'medium': {'contract_storage': 427499642, 'db_storage': 284999761}, - 'small': {'contract_storage': 106874910, 'db_storage': 71249940}, - 'test': {'contract_storage': 427499642, 'db_storage': 284999761}, - 'test4': {'contract_storage': 427499642, 'db_storage': 284999761} + 'large': {'contract_storage': 12787195576, 'db_storage': 8524797050}, + 'medium': {'contract_storage': 1598399446, 'db_storage': 1065599631}, + 'small': {'contract_storage': 99899965, 'db_storage': 66599976}, + 'test': {'contract_storage': 1598399446, 'db_storage': 1065599631}, + 'test4': {'contract_storage': 1598399446, 'db_storage': 1065599631} } From c698ce24f1e4690c6a41d24024469bb5baf826a7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 8 Jul 2021 20:00:21 +0300 Subject: [PATCH 138/140] SKALE-4381 Fix SSL upload --- .flake8 | 2 +- helper-scripts | 2 +- node_cli/core/ssl.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index d0ed9fc5..12bd9cf3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 100 -exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv +exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,venv,helper-scripts diff --git a/helper-scripts b/helper-scripts index ced34747..dc21eb10 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit ced34747acb3335f1ea858b4e847f3c21b4ccf7e +Subproject commit dc21eb1005ac0d4b45d29d89a8a7783eba6ecf20 diff --git a/node_cli/core/ssl.py b/node_cli/core/ssl.py index 9895ac2f..38bfc73d 100644 --- a/node_cli/core/ssl.py +++ b/node_cli/core/ssl.py @@ -219,7 +219,11 @@ def send_saving_cert_request(key_path, cert_path, force): None, json.dumps({'force': force}), 'application/json' ) - return post_request('ssl_upload', files=files_data) + return post_request( + blueprint='ssl', + method='upload', + files=files_data + ) def upload_cert(cert_path, key_path, force, no_client=False): From 2533a7636a9a955c5e129345a1e1ae62a40bcf0f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 8 Jul 2021 20:09:04 +0300 Subject: [PATCH 139/140] SKALE-4381 Fix SSL upload --- tests/core_ssl_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py index 0a769244..318109b2 100644 --- a/tests/core_ssl_test.py +++ b/tests/core_ssl_test.py @@ -76,16 +76,16 @@ def test_verify_cert_bad_key(bad_key): def test_upload_cert(pr_mock, cert_key_pair): cert, key = cert_key_pair upload_cert(cert, key, force=False, no_client=True) - args = pr_mock.call_args.args - assert args[0] == 'ssl_upload' + # args = pr_mock.call_args.args + # assert args[0] == 'ssl_upload' kwargs = pr_mock.call_args.kwargs assert kwargs['files']['ssl_cert'][1].name == cert assert kwargs['files']['ssl_key'][1].name == key assert kwargs['files']['json'][1] == '{"force": false}' upload_cert(cert, key, force=True, no_client=True) - args = pr_mock.call_args.args - assert args[0] == 'ssl_upload' + # args = pr_mock.call_args.args + # assert args[0] == 'ssl_upload' kwargs = pr_mock.call_args.kwargs assert kwargs['files']['ssl_cert'][1].name == cert assert kwargs['files']['ssl_key'][1].name == key From 210c0a2ebaa142a0f6ba43cdec9b3187b990d941 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 9 Jul 2021 12:30:11 +0300 Subject: [PATCH 140/140] Change sChain alloc to 80/20 --- tests/.skale/config/environment_params.yaml | 8 +-- tests/resources_test.py | 54 ++++++++++----------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/.skale/config/environment_params.yaml b/tests/.skale/config/environment_params.yaml index 1711bb6a..2f6fdef1 100644 --- a/tests/.skale/config/environment_params.yaml +++ b/tests/.skale/config/environment_params.yaml @@ -1,11 +1,11 @@ common: schain: cpu: - skaled: 0.7 - ima: 0.3 + skaled: 0.8 + ima: 0.2 mem: - skaled: 0.7 - ima: 0.3 + skaled: 0.8 + ima: 0.2 volume_limits: max_consensus_storage_bytes: 0.3 max_skaled_leveldb_storage_bytes: 0.3 diff --git a/tests/resources_test.py b/tests/resources_test.py index d7b11cc4..1e772b2b 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -48,11 +48,11 @@ def test_generate_resource_allocation_config(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['cpu_shares']['test4'] == 89 - assert resource_allocation_config['schain']['cpu_shares']['test'] == 89 - assert resource_allocation_config['schain']['cpu_shares']['small'] == 5 - assert resource_allocation_config['schain']['cpu_shares']['medium'] == 89 - assert resource_allocation_config['schain']['cpu_shares']['large'] == 716 + assert resource_allocation_config['schain']['cpu_shares']['test4'] == 102 + assert resource_allocation_config['schain']['cpu_shares']['test'] == 102 + assert resource_allocation_config['schain']['cpu_shares']['small'] == 6 + assert resource_allocation_config['schain']['cpu_shares']['medium'] == 102 + assert resource_allocation_config['schain']['cpu_shares']['large'] == 819 assert isinstance(resource_allocation_config['schain']['mem']['test4'], int) assert isinstance(resource_allocation_config['schain']['mem']['test'], int) @@ -66,7 +66,7 @@ def test_generate_resource_allocation_config(): assert resource_allocation_config['schain']['disk']['medium'] == 8879996928 assert resource_allocation_config['schain']['disk']['large'] == 71039975424 - assert resource_allocation_config['ima']['cpu_shares'] == {'large': 307, 'medium': 38, 'small': 2, 'test': 38, 'test4': 38} # noqa + assert resource_allocation_config['ima']['cpu_shares'] == {'large': 204, 'medium': 25, 'small': 1, 'test': 25, 'test4': 25} # noqa assert isinstance(resource_allocation_config['ima']['mem'], dict) assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS @@ -116,17 +116,17 @@ def test_get_cpu_alloc(): schain_cpu_alloc_dict = schain_cpu_alloc.dict() ima_cpu_alloc_dict = ima_cpu_alloc.dict() - assert schain_cpu_alloc_dict['test4'] == 89 - assert schain_cpu_alloc_dict['test'] == 89 - assert schain_cpu_alloc_dict['small'] == 5 - assert schain_cpu_alloc_dict['medium'] == 89 - assert schain_cpu_alloc_dict['large'] == 716 + assert schain_cpu_alloc_dict['test4'] == 102 + assert schain_cpu_alloc_dict['test'] == 102 + assert schain_cpu_alloc_dict['small'] == 6 + assert schain_cpu_alloc_dict['medium'] == 102 + assert schain_cpu_alloc_dict['large'] == 819 - assert ima_cpu_alloc_dict['test4'] == 38 - assert ima_cpu_alloc_dict['test'] == 38 - assert ima_cpu_alloc_dict['small'] == 2 - assert ima_cpu_alloc_dict['medium'] == 38 - assert ima_cpu_alloc_dict['large'] == 307 + assert ima_cpu_alloc_dict['test4'] == 25 + assert ima_cpu_alloc_dict['test'] == 25 + assert ima_cpu_alloc_dict['small'] == 1 + assert ima_cpu_alloc_dict['medium'] == 25 + assert ima_cpu_alloc_dict['large'] == 204 def test_get_memory_alloc(): @@ -136,17 +136,17 @@ def test_get_memory_alloc(): schain_mem_alloc_dict = schain_mem_alloc.dict() ima_mem_alloc_dict = ima_mem_alloc.dict() - assert schain_mem_alloc_dict['test4'] == 875000 - assert schain_mem_alloc_dict['test'] == 875000 - assert schain_mem_alloc_dict['small'] == 54687 - assert schain_mem_alloc_dict['medium'] == 875000 - assert schain_mem_alloc_dict['large'] == 7000000 - - assert ima_mem_alloc_dict['test4'] == 375000 - assert ima_mem_alloc_dict['test'] == 375000 - assert ima_mem_alloc_dict['small'] == 23437 - assert ima_mem_alloc_dict['medium'] == 375000 - assert ima_mem_alloc_dict['large'] == 3000000 + assert schain_mem_alloc_dict['test4'] == 1000000 + assert schain_mem_alloc_dict['test'] == 1000000 + assert schain_mem_alloc_dict['small'] == 62500 + assert schain_mem_alloc_dict['medium'] == 1000000 + assert schain_mem_alloc_dict['large'] == 8000000 + + assert ima_mem_alloc_dict['test4'] == 250000 + assert ima_mem_alloc_dict['test'] == 250000 + assert ima_mem_alloc_dict['small'] == 15625 + assert ima_mem_alloc_dict['medium'] == 250000 + assert ima_mem_alloc_dict['large'] == 2000000 def test_leveldb_limits():