From 592e0a14020f0980079209807526e9005c0b6027 Mon Sep 17 00:00:00 2001 From: Arthur Lutz Date: Wed, 4 Mar 2015 18:27:37 +0100 Subject: [PATCH 1/3] [modules/postgresql] modules for cluster management in postgresql (debian specific) related to #21293 --- salt/modules/deb_postgres.py | 151 ++++++++++++++++++++++++ tests/unit/modules/deb_postgres_test.py | 122 +++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 salt/modules/deb_postgres.py create mode 100644 tests/unit/modules/deb_postgres_test.py diff --git a/salt/modules/deb_postgres.py b/salt/modules/deb_postgres.py new file mode 100644 index 000000000000..35c2000755e2 --- /dev/null +++ b/salt/modules/deb_postgres.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +''' +Module to provide Postgres compatibility to salt for debian family specific tools. + +''' + +# Import python libs +from __future__ import absolute_import +import logging +import pipes + +# Import salt libs +import salt.utils + +# Import 3rd-party libs +import salt.ext.six as six + + +log = logging.getLogger(__name__) + +__virtualname__ = 'postgres' + +def __virtual__(): + ''' + Only load this module if the pg_createcluster bin exists + ''' + if salt.utils.which('pg_createcluster'): + return __virtualname__ + return False + +def cluster_create(version, + name='main', + port=None, + locale=None, + encoding=None, + datadir=None): + ''' + Adds a cluster to the Postgres server. + + .. warning: + + Only works for debian family distros so far. + + CLI Example: + + .. code-block:: bash + + salt '*' postgres.cluster_create '9.3' + + salt '*' postgres.cluster_create '9.3' 'main' + + salt '*' postgres.cluster_create '9.3' locale='fr_FR' + + ''' + # XXX version should be a string, but it requires to use "'9.3'" in cmd line + # assert isinstance(version, six.string_types) + cmd = [salt.utils.which('pg_createcluster')] + if port: + cmd += ['--port', str(port)] + if locale: + cmd += ['--locale', locale] + if encoding: + cmd += ['--encoding', encoding] + if datadir: + cmd += ['--datadir', datadir] + cmd += [version, name] + cmdstr = ' '.join([pipes.quote(c) for c in cmd]) + ret = __salt__['cmd.run_all'](cmdstr, python_shell=False) + if ret.get('retcode', 0) != 0: + log.error('Error creating a Postgresql cluster {0}/{1}'.format(version, name)) + return False + return ret + +def cluster_list(verbose=False): + ''' + Return a list of cluster of Postgres server (tuples of version and name). + + CLI Example: + + .. code-block:: bash + + salt '*' postgres.cluster_list + + salt '*' postgres.cluster_list verbose=True + ''' + cmd = [salt.utils.which('pg_lsclusters'), '--no-header'] + ret = __salt__['cmd.run_all'](' '.join([pipes.quote(c) for c in cmd])) + if ret.get('retcode', 0) != 0: + log.error('Error listing clusters') + cluster_dict = _parse_pg_lscluster(ret['stdout']) + if verbose: + return cluster_dict + return cluster_dict.keys() + + +def cluster_exists(version, + name='main'): + ''' + Checks if a given version and name of a cluster exists. + + CLI Example: + + .. code-block:: bash + + salt '*' postgres.cluster_exists '9.3' + + salt '*' postgres.cluster_exists '9.3' 'main' + ''' + return '{0}/{1}'.format(version, name) in cluster_list() + +def cluster_remove(version, + name='main', + stop=False): + ''' + Remove a cluster on a Postgres server. By default it doesn't try + to stop the cluster. + + CLI Example: + + .. code-block:: bash + + salt '*' postgres.cluster_remove '9.3' + + salt '*' postgres.cluster_remove '9.3' 'main' + + salt '*' postgres.cluster_remove '9.3' 'main' stop=True + + ''' + cmd = [salt.utils.which('pg_dropcluster')] + if stop: + cmd += ['--stop'] + cmd += [version, name] + cmdstr = ' '.join([pipes.quote(c) for c in cmd]) + ret = __salt__['cmd.run_all'](cmdstr, python_shell=False) + # FIXME - return Boolean ? + if ret.get('retcode', 0) != 0: + log.error('Error removing a Postgresql cluster {0}/{1}'.format(version, name)) + else: + ret['changes'] = 'Successfully removed cluster {0}/{1}'.format(version, name) + return ret + +def _parse_pg_lscluster(output): + ''' + Helper function to parse the output of pg_lscluster + ''' + cluster_dict = {} + for line in output.splitlines(): + v,n,p,s,u,d,l = line.split() + cluster_dict['{0}/{1}'.format(v,n)] = {'port':int(p), 'status':s, + 'user':u, 'datadir':d, 'log':l} + return cluster_dict diff --git a/tests/unit/modules/deb_postgres_test.py b/tests/unit/modules/deb_postgres_test.py new file mode 100644 index 000000000000..4f9a1040d6ff --- /dev/null +++ b/tests/unit/modules/deb_postgres_test.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import, print_function +import re +import os + +# Import Salt Testing libs +from salttesting import skipIf, TestCase +from salttesting.helpers import ensure_in_syspath +from salttesting.mock import NO_MOCK, NO_MOCK_REASON, Mock, patch +ensure_in_syspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../../')) +ensure_in_syspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../')) + +# Import salt libs +from salt.modules import deb_postgres + +deb_postgres.__grains__ = None # in order to stub it w/patch below +deb_postgres.__salt__ = None # in order to stub it w/patch below + + +LSCLUSTER = '''8.4 main 5432 online postgres /srv/8.4/main /var/log/postgresql/postgresql-8.4-main.log +9.1 main 5433 online postgres /srv/9.1/main /var/log/postgresql/postgresql-9.1-main.log +''' +if NO_MOCK is False: + SALT_STUB = { + 'config.option': Mock(), + 'cmd.run_all': Mock(return_value={'stdout':LSCLUSTER}), + 'file.chown': Mock(), + 'file.remove': Mock(), + } +else: + SALT_STUB = {} + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch.multiple(deb_postgres, + __salt__=SALT_STUB) +@patch('salt.utils.which', Mock(return_value='/usr/bin/pg_createcluster')) +class PostgresClusterTestCase(TestCase): + + def test_cluster_create(self): + deb_postgres.cluster_create( + '9.3', + 'main', + port='5432', + locale='fr_FR', + encoding='UTF-8', + datadir='/opt/postgresql' + ) + cmd = SALT_STUB['cmd.run_all'] + + cmdstr = '/usr/bin/pg_createcluster ' \ + '--port 5432 --locale fr_FR --encoding UTF-8 ' \ + '--datadir /opt/postgresql ' \ + '9.3 main' + self.assertEqual(cmdstr, cmd.call_args[0][0]) + + # XXX version should be a string but from cmdline you get a float + # def test_cluster_create_with_float(self): + # self.assertRaises(AssertionError, deb_postgres.cluster_create, + # (9.3,'main',), + # dict(port='5432', + # locale='fr_FR', + # encoding='UTF-8', + # datadir='/opt/postgresql')) + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch.multiple(deb_postgres, + __salt__=SALT_STUB) +@patch('salt.utils.which', Mock(return_value='/usr/bin/pg_lsclusters')) +class PostgresLsClusterTestCase(TestCase): + + def test_parse_pg_lsclusters(self): + stdout = LSCLUSTER + self.maxDiff = None + self.assertDictEqual({('8.4', 'main'):{'port':5432, + 'status': 'online', + 'user': 'postgres', + 'datadir':'/srv/8.4/main', + 'log':'/var/log/postgresql/postgresql-8.4-main.log'}, + ('9.1', 'main'):{'port':5433, + 'status': 'online', + 'user': 'postgres', + 'datadir':'/srv/9.1/main', + 'log':'/var/log/postgresql/postgresql-9.1-main.log'},}, + deb_postgres._parse_pg_lscluster(stdout)) + + def test_cluster_list(self): + return_list = deb_postgres.cluster_list() + cmd = SALT_STUB['cmd.run_all'] + self.assertEqual('/usr/bin/pg_lsclusters --no-header', + cmd.call_args[0][0]) + self.assertIsInstance(return_list, list) + return_dict = deb_postgres.cluster_list(verbose=True) + self.assertIsInstance(return_dict, dict) + + def test_cluster_exists(self): + self.assertTrue(deb_postgres.cluster_exists('8.4') is True) + self.assertTrue(deb_postgres.cluster_exists('8.4', 'main') is True) + self.assertFalse(deb_postgres.cluster_exists('3.4', 'main')) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch.multiple(deb_postgres, + __salt__=SALT_STUB) +@patch('salt.utils.which', Mock(return_value='/usr/bin/pg_dropcluster')) +class PostgresDeleteClusterTestCase(TestCase): + + def test_cluster_delete(self): + return_list = deb_postgres.cluster_remove('9.3', 'main') + cmd = SALT_STUB['cmd.run_all'] + self.assertEqual('/usr/bin/pg_dropcluster 9.3 main', + cmd.call_args[0][0]) + return_list = deb_postgres.cluster_remove('9.3', 'main', stop=True) + cmd = SALT_STUB['cmd.run_all'] + self.assertEqual('/usr/bin/pg_dropcluster --stop 9.3 main', + cmd.call_args[0][0]) + +if __name__ == '__main__': + from integration import run_tests + run_tests(PostgresClusterTestCase, PostgresLsClusterTestCase, + PostgresDeleteClusterTestCase, needs_daemon=False) From f1111bd238644585073f5bcad8fcc0e4822404be Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Fri, 6 Mar 2015 16:01:25 +0100 Subject: [PATCH 2/3] lint --- salt/modules/deb_postgres.py | 35 ++++++++++------ tests/unit/modules/deb_postgres_test.py | 55 ++++++++++++++----------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/salt/modules/deb_postgres.py b/salt/modules/deb_postgres.py index 35c2000755e2..2ace81090d25 100644 --- a/salt/modules/deb_postgres.py +++ b/salt/modules/deb_postgres.py @@ -13,13 +13,12 @@ import salt.utils # Import 3rd-party libs -import salt.ext.six as six - log = logging.getLogger(__name__) __virtualname__ = 'postgres' + def __virtual__(): ''' Only load this module if the pg_createcluster bin exists @@ -28,6 +27,7 @@ def __virtual__(): return __virtualname__ return False + def cluster_create(version, name='main', port=None, @@ -35,7 +35,7 @@ def cluster_create(version, encoding=None, datadir=None): ''' - Adds a cluster to the Postgres server. + Adds a cluster to the Postgres server. .. warning: @@ -52,8 +52,6 @@ def cluster_create(version, salt '*' postgres.cluster_create '9.3' locale='fr_FR' ''' - # XXX version should be a string, but it requires to use "'9.3'" in cmd line - # assert isinstance(version, six.string_types) cmd = [salt.utils.which('pg_createcluster')] if port: cmd += ['--port', str(port)] @@ -67,10 +65,12 @@ def cluster_create(version, cmdstr = ' '.join([pipes.quote(c) for c in cmd]) ret = __salt__['cmd.run_all'](cmdstr, python_shell=False) if ret.get('retcode', 0) != 0: - log.error('Error creating a Postgresql cluster {0}/{1}'.format(version, name)) + log.error('Error creating a Postgresql' + ' cluster {0}/{1}'.format(version, name)) return False return ret + def cluster_list(verbose=False): ''' Return a list of cluster of Postgres server (tuples of version and name). @@ -108,6 +108,7 @@ def cluster_exists(version, ''' return '{0}/{1}'.format(version, name) in cluster_list() + def cluster_remove(version, name='main', stop=False): @@ -132,20 +133,28 @@ def cluster_remove(version, cmd += [version, name] cmdstr = ' '.join([pipes.quote(c) for c in cmd]) ret = __salt__['cmd.run_all'](cmdstr, python_shell=False) - # FIXME - return Boolean ? + # FIXME - return Boolean ? if ret.get('retcode', 0) != 0: - log.error('Error removing a Postgresql cluster {0}/{1}'.format(version, name)) + log.error('Error removing a Postgresql' + ' cluster {0}/{1}'.format(version, name)) else: - ret['changes'] = 'Successfully removed cluster {0}/{1}'.format(version, name) + ret['changes'] = ('Successfully removed' + ' cluster {0}/{1}').format(version, name) return ret - + + def _parse_pg_lscluster(output): ''' Helper function to parse the output of pg_lscluster ''' cluster_dict = {} for line in output.splitlines(): - v,n,p,s,u,d,l = line.split() - cluster_dict['{0}/{1}'.format(v,n)] = {'port':int(p), 'status':s, - 'user':u, 'datadir':d, 'log':l} + version, name, port, status, user, datadir, log = ( + line.split()) + cluster_dict['{0}/{1}'.format(version, name)] = { + 'port': int(port), + 'status': status, + 'user': user, + 'datadir': datadir, + 'log': log} return cluster_dict diff --git a/tests/unit/modules/deb_postgres_test.py b/tests/unit/modules/deb_postgres_test.py index 4f9a1040d6ff..f054fcf3b80d 100644 --- a/tests/unit/modules/deb_postgres_test.py +++ b/tests/unit/modules/deb_postgres_test.py @@ -2,15 +2,16 @@ # Import python libs from __future__ import absolute_import, print_function -import re import os # Import Salt Testing libs from salttesting import skipIf, TestCase from salttesting.helpers import ensure_in_syspath from salttesting.mock import NO_MOCK, NO_MOCK_REASON, Mock, patch -ensure_in_syspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../../')) -ensure_in_syspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../')) +ensure_in_syspath( + os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../../')) +ensure_in_syspath( + os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../')) # Import salt libs from salt.modules import deb_postgres @@ -19,19 +20,23 @@ deb_postgres.__salt__ = None # in order to stub it w/patch below -LSCLUSTER = '''8.4 main 5432 online postgres /srv/8.4/main /var/log/postgresql/postgresql-8.4-main.log -9.1 main 5433 online postgres /srv/9.1/main /var/log/postgresql/postgresql-9.1-main.log +LSCLUSTER = '''\ +8.4 main 5432 online postgres /srv/8.4/main \ + /var/log/postgresql/postgresql-8.4-main.log +9.1 main 5433 online postgres /srv/9.1/main \ + /var/log/postgresql/postgresql-9.1-main.log ''' if NO_MOCK is False: SALT_STUB = { 'config.option': Mock(), - 'cmd.run_all': Mock(return_value={'stdout':LSCLUSTER}), + 'cmd.run_all': Mock(return_value={'stdout': LSCLUSTER}), 'file.chown': Mock(), 'file.remove': Mock(), } else: SALT_STUB = {} + @skipIf(NO_MOCK, NO_MOCK_REASON) @patch.multiple(deb_postgres, __salt__=SALT_STUB) @@ -57,13 +62,14 @@ def test_cluster_create(self): # XXX version should be a string but from cmdline you get a float # def test_cluster_create_with_float(self): - # self.assertRaises(AssertionError, deb_postgres.cluster_create, - # (9.3,'main',), + # self.assertRaises(AssertionError, deb_postgres.cluster_create, + # (9.3,'main',), # dict(port='5432', # locale='fr_FR', # encoding='UTF-8', # datadir='/opt/postgresql')) + @skipIf(NO_MOCK, NO_MOCK_REASON) @patch.multiple(deb_postgres, __salt__=SALT_STUB) @@ -73,17 +79,20 @@ class PostgresLsClusterTestCase(TestCase): def test_parse_pg_lsclusters(self): stdout = LSCLUSTER self.maxDiff = None - self.assertDictEqual({('8.4', 'main'):{'port':5432, - 'status': 'online', - 'user': 'postgres', - 'datadir':'/srv/8.4/main', - 'log':'/var/log/postgresql/postgresql-8.4-main.log'}, - ('9.1', 'main'):{'port':5433, - 'status': 'online', - 'user': 'postgres', - 'datadir':'/srv/9.1/main', - 'log':'/var/log/postgresql/postgresql-9.1-main.log'},}, - deb_postgres._parse_pg_lscluster(stdout)) + self.assertDictEqual( + {('8.4', 'main'): { + 'port': 5432, + 'status': 'online', + 'user': 'postgres', + 'datadir': '/srv/8.4/main', + 'log': '/var/log/postgresql/postgresql-8.4-main.log'}, + ('9.1', 'main'): { + 'port': 5433, + 'status': 'online', + 'user': 'postgres', + 'datadir': '/srv/9.1/main', + 'log': '/var/log/postgresql/postgresql-9.1-main.log'}}, + deb_postgres._parse_pg_lscluster(stdout)) def test_cluster_list(self): return_list = deb_postgres.cluster_list() @@ -105,18 +114,18 @@ def test_cluster_exists(self): __salt__=SALT_STUB) @patch('salt.utils.which', Mock(return_value='/usr/bin/pg_dropcluster')) class PostgresDeleteClusterTestCase(TestCase): - + def test_cluster_delete(self): - return_list = deb_postgres.cluster_remove('9.3', 'main') + deb_postgres.cluster_remove('9.3', 'main') cmd = SALT_STUB['cmd.run_all'] self.assertEqual('/usr/bin/pg_dropcluster 9.3 main', cmd.call_args[0][0]) - return_list = deb_postgres.cluster_remove('9.3', 'main', stop=True) + deb_postgres.cluster_remove('9.3', 'main', stop=True) cmd = SALT_STUB['cmd.run_all'] self.assertEqual('/usr/bin/pg_dropcluster --stop 9.3 main', cmd.call_args[0][0]) if __name__ == '__main__': from integration import run_tests - run_tests(PostgresClusterTestCase, PostgresLsClusterTestCase, + run_tests(PostgresClusterTestCase, PostgresLsClusterTestCase, PostgresDeleteClusterTestCase, needs_daemon=False) From 5f4905da62821d1d3fb39c0dc3a698d1112e34a7 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Fri, 6 Mar 2015 17:13:31 +0100 Subject: [PATCH 3/3] fix tests --- tests/unit/modules/deb_postgres_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/deb_postgres_test.py b/tests/unit/modules/deb_postgres_test.py index f054fcf3b80d..8499684b5133 100644 --- a/tests/unit/modules/deb_postgres_test.py +++ b/tests/unit/modules/deb_postgres_test.py @@ -80,13 +80,13 @@ def test_parse_pg_lsclusters(self): stdout = LSCLUSTER self.maxDiff = None self.assertDictEqual( - {('8.4', 'main'): { + {('8.4/main'): { 'port': 5432, 'status': 'online', 'user': 'postgres', 'datadir': '/srv/8.4/main', 'log': '/var/log/postgresql/postgresql-8.4-main.log'}, - ('9.1', 'main'): { + ('9.1/main'): { 'port': 5433, 'status': 'online', 'user': 'postgres',