diff --git a/.drone.jsonnet b/.drone.jsonnet index 83e11d74d..f0dd7ad73 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -170,23 +170,22 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') publish(step_prefix='pkg', eventp=event + '/${DRONE_BUILD_NUMBER}'):: { name: 'publish ' + step_prefix, depends_on: [std.strReplace(step_prefix, ' latest', '')], - image: 'plugins/s3-sync', + image: 'amazon/aws-cli', when: { status: ['success', 'failure'], }, - settings: { - bucket: 'cspkg', - access_key: { + environment: { + AWS_ACCESS_KEY_ID: { from_secret: 'aws_access_key_id', }, - secret_key: { + AWS_SECRET_ACCESS_KEY: { from_secret: 'aws_secret_access_key', }, - source: result, - // branchp has slash if not empty - target: branchp + eventp + '/' + server + '/' + arch + '/' + result, - delete: 'true', }, + commands: [ + 'aws s3 sync ' + result + ' s3://cspkg/' + branchp + eventp + '/' + server + '/' + arch + '/' + result + ' --delete', + 'echo "Data uploaded to: ' + publish_pkg_url + '"' + ], }, local regression_tests = if (event == 'cron') then [ diff --git a/cmapi/README.md b/cmapi/README.md index 0afef4ab1..9b2f4f1c6 100644 --- a/cmapi/README.md +++ b/cmapi/README.md @@ -37,7 +37,10 @@ Packages have bundled python interpreter and python dependencies. ## Get dependencies # get portable python -wget -qO- https://cspkg.s3.amazonaws.com/python-dist-no-nis.tar.gz | tar xzf - -C ./ +wget -qO- https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst | tar --use-compress-program=unzstd -xf - -C ./ && \ +mv python pp && mv pp/install python && rm -rf pp + +There is a script dev_tools/activate that works like virtualenv activate (you can use it to work with portable Python like with virtualenv). # install python dependencies python/bin/pip3 install -t deps --only-binary :all -r requirements.txt diff --git a/cmapi/cmapi_server/controllers/dispatcher.py b/cmapi/cmapi_server/controllers/dispatcher.py index 3d1aaa0d9..ba7c90565 100644 --- a/cmapi/cmapi_server/controllers/dispatcher.py +++ b/cmapi/cmapi_server/controllers/dispatcher.py @@ -262,7 +262,7 @@ dispatcher.connect( def jsonify_error(status, message, traceback, version): \ - # pylint: disable=unused-argument + # pylint: disable=unused-argument """JSONify all CherryPy error responses (created by raising the cherrypy.HTTPError exception) """ diff --git a/cmapi/cmapi_server/controllers/endpoints.py b/cmapi/cmapi_server/controllers/endpoints.py index bf3b009c1..a1b8792cb 100644 --- a/cmapi/cmapi_server/controllers/endpoints.py +++ b/cmapi/cmapi_server/controllers/endpoints.py @@ -417,7 +417,7 @@ class ConfigController: ) if in_maintenance_state(): module_logger.info( - 'Maintaninance state is active in new config. ' + 'Maintenance state is active in new config. ' 'MCS processes should not be started.' ) cherrypy.engine.publish('failover', False) diff --git a/cmapi/cmapi_server/helpers.py b/cmapi/cmapi_server/helpers.py index ea2b61726..1cf38ef42 100644 --- a/cmapi/cmapi_server/helpers.py +++ b/cmapi/cmapi_server/helpers.py @@ -372,6 +372,11 @@ def broadcast_new_config( logging.warning( f'Timeout while pushing new config to "{node}"' ) + except requests.exceptions.RequestException as e: + logging.warning( + 'Error while pushing new config to "%s": %s"', node, str(e) + ) + logging.debug('Response: %s', r.text) except Exception as e: logging.warning( f'Got an unexpected error pushing new config to "{node}"', @@ -461,7 +466,7 @@ def get_config_parser( except PermissionError as e: # TODO: looks like it's useless here, because of creating config # from default on cmapi server startup - # Anyway looks like it have to raise error and then + # Anyway looks like it has to raise error and then # return 500 error logging.error( 'CMAPI cannot create configuration file. ' @@ -826,7 +831,7 @@ def cmapi_config_check(cmapi_conf_path: str = CMAPI_CONF_PATH): """ if not os.path.exists(cmapi_conf_path): logging.info( - f'There are no config file at "{cmapi_conf_path}". ' + f'There is no config file at "{cmapi_conf_path}". ' f'So copy default config from {CMAPI_DEFAULT_CONF_PATH} there.' ) copyfile(CMAPI_DEFAULT_CONF_PATH, cmapi_conf_path) diff --git a/cmapi/cmapi_server/logging_management.py b/cmapi/cmapi_server/logging_management.py index fc1e6b583..cffcae122 100644 --- a/cmapi/cmapi_server/logging_management.py +++ b/cmapi/cmapi_server/logging_management.py @@ -95,6 +95,12 @@ def add_logging_level(level_name, level_num, method_name=None): setattr(logging, method_name, partial(logging.log, level_num)) +def enable_console_logging(logger: logging.Logger) -> None: + """Enable logging to console for passed logger by adding a StreamHandler to it""" + console_handler = logging.StreamHandler() + logger.addHandler(console_handler) + + def config_cmapi_server_logging(): # add custom level TRACE only for develop purposes # could be activated using API endpoints or cli tool without relaunching diff --git a/cmapi/cmapi_server/node_manipulation.py b/cmapi/cmapi_server/node_manipulation.py index d4e9240ed..7c8644902 100644 --- a/cmapi/cmapi_server/node_manipulation.py +++ b/cmapi/cmapi_server/node_manipulation.py @@ -55,7 +55,7 @@ def switch_node_maintenance( maintenance_element = etree.SubElement(config_root, 'Maintenance') maintenance_element.text = str(maintenance_state).lower() node_config.write_config(config_root, filename=output_config_filename) - # TODO: probably move publishing to cherrypy.emgine failover channel here? + # TODO: probably move publishing to cherrypy.engine failover channel here? def add_node( diff --git a/cmapi/cmapi_server/process_dispatchers/base.py b/cmapi/cmapi_server/process_dispatchers/base.py index 45d10c6e8..84d252d51 100644 --- a/cmapi/cmapi_server/process_dispatchers/base.py +++ b/cmapi/cmapi_server/process_dispatchers/base.py @@ -79,7 +79,7 @@ class BaseDispatcher: del proc result = (True, output) else: - logging.debug('Waiting command to finish.') + logging.debug('Waiting for command to finish.') stdout_str, _ = proc.communicate() returncode = proc.wait() if stdout_str is not None: diff --git a/cmapi/cmapi_server/process_dispatchers/systemd.py b/cmapi/cmapi_server/process_dispatchers/systemd.py index 8b7b2714d..52433349c 100644 --- a/cmapi/cmapi_server/process_dispatchers/systemd.py +++ b/cmapi/cmapi_server/process_dispatchers/systemd.py @@ -64,7 +64,7 @@ class SystemdDispatcher(BaseDispatcher): ..Note: Not working with multiple services at a time. """ - logging.debug(f'Checking "{service}" is running.') + logging.debug(f'Checking if "{service}" is running.') # TODO: remove conditions below when we'll drop CentOS 7 support cmd = 'show -p ActiveState --value' if cls.systemctl_version < 230: # not supported --value in old version diff --git a/cmapi/dev_tools/activate b/cmapi/dev_tools/activate new file mode 100644 index 000000000..cdac4fab2 --- /dev/null +++ b/cmapi/dev_tools/activate @@ -0,0 +1,87 @@ +# Modified venv activation script used with portable Python +# Adds cmapi/ and cmapi/deps/ dirs to PYTHONPATH +# Expects to be located in cmapi/dev_tools and deps to be located in cmapi/deps (as written in README.md) + + +# This file must be used with "source bin/activate" *from bash* +# You cannot run it directly + +cmapi_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../) +venv_dir=$cmapi_dir/python +deps_dir=$cmapi_dir/deps + + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + if [ -n "${_OLD_VIRTUAL_PYTHONPATH:-}" ] ; then + PYTHONPATH="${_OLD_VIRTUAL_PYTHONPATH:-}" + export PYTHONPATH + unset _OLD_VIRTUAL_PYTHONPATH + fi + + # Call hash to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + hash -r 2> /dev/null + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +export VIRTUAL_ENV=$venv_dir + +_OLD_VIRTUAL_PATH="$PATH" +echo "Adding $VIRTUAL_ENV/bin to PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +# Save the old PYTHONPATH if it exists +if [ -n "${PYTHONPATH:-}" ]; then + _OLD_VIRTUAL_PYTHONPATH="${PYTHONPATH:-}" +fi + +# Add cmapi/deps directory to PYTHONPATH +echo "Adding $cmapi_dir and $deps_dir to PYTHONPATH" +export PYTHONPATH="${cmapi_dir}:${deps_dir}:${PYTHONPATH:-}" + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(portpy) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(portpy) " + export VIRTUAL_ENV_PROMPT +fi + +# Call hash to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +hash -r 2> /dev/null diff --git a/cmapi/mcs_cluster_tool/__main__.py b/cmapi/mcs_cluster_tool/__main__.py index 35bcaa82d..98ff3bb34 100644 --- a/cmapi/mcs_cluster_tool/__main__.py +++ b/cmapi/mcs_cluster_tool/__main__.py @@ -4,7 +4,7 @@ import sys import typer -from cmapi_server.logging_management import dict_config, add_logging_level +from cmapi_server.logging_management import dict_config, add_logging_level, enable_console_logging from mcs_cluster_tool import ( cluster_app, cmapi_app, backup_commands, restore_commands ) @@ -38,10 +38,20 @@ def help_all(): # Open the man page in interactive mode subprocess.run(['man', 'mcs']) +@app.callback() +def main(verbose: bool = typer.Option(False, '--verbose', '-v', help='Enable verbose logging to console')): + '''Add a -v option and setup logging in every subcommand''' + setup_logging(verbose) + + +def setup_logging(verbose: bool = False) -> None: + add_logging_level('TRACE', 5) + dict_config(MCS_CLI_LOG_CONF_PATH) + if verbose: + enable_console_logging(logging.getLogger()) + if __name__ == '__main__': - add_logging_level('TRACE', 5) #TODO: remove when stadalone mode added. - dict_config(MCS_CLI_LOG_CONF_PATH) logger = logging.getLogger('mcs_cli') # add separator between cli commands logging logger.debug(f'{"-":-^80}') diff --git a/cmapi/mcs_cluster_tool/decorators.py b/cmapi/mcs_cluster_tool/decorators.py index e5ab0bb40..c8ac9499a 100644 --- a/cmapi/mcs_cluster_tool/decorators.py +++ b/cmapi/mcs_cluster_tool/decorators.py @@ -21,13 +21,13 @@ def handle_output(func): return_code = 0 except CMAPIBasicError as err: typer.echo(err.message, err=True) - logger.error('Error while command execution', exc_info=True) + logger.error('Error during command execution', exc_info=True) except typer.BadParameter as err: logger.error('Bad command line parameter.') raise err except Exception: logger.error( - 'Undefined error while command execution', + 'Undefined error during command execution', exc_info=True ) typer.echo('Unknown error, check the log file.', err=True) diff --git a/cmapi/mcs_node_control/models/node_config.py b/cmapi/mcs_node_control/models/node_config.py index 91930b0bf..7dac18bce 100644 --- a/cmapi/mcs_node_control/models/node_config.py +++ b/cmapi/mcs_node_control/models/node_config.py @@ -37,7 +37,7 @@ class NodeConfig: def get_current_config_root( self, config_filename: str = DEFAULT_MCS_CONF_PATH, upgrade=True ): - """Retrievs current configuration. + """Retrieves current configuration. Read the config and returns Element. TODO: pretty the same function in misc.py - review