From ab151f9e7b024b63115b69e99e05b29470022c5a Mon Sep 17 00:00:00 2001 From: Carson Gee Date: Tue, 17 Dec 2013 17:49:46 -0500 Subject: [PATCH] Several code enhancements to sysadmin dashboard Improved testing (cleaning up afterwards, catching stdout) Refactored import function to it's own file Removed monkey patch to log capturing, but still needs work Translation fixes Removal of several gettext phrases --- lms/djangoapps/dashboard/git_import.py | 207 +++++++++++++++++ .../management/commands/git_add_course.py | 211 +----------------- .../commands/tests/test_git_add_course.py | 46 ++-- lms/djangoapps/dashboard/sysadmin.py | 44 ++-- .../dashboard/tests/test_sysadmin.py | 55 ++--- 5 files changed, 299 insertions(+), 264 deletions(-) create mode 100644 lms/djangoapps/dashboard/git_import.py diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py new file mode 100644 index 000000000000..67c2942ee195 --- /dev/null +++ b/lms/djangoapps/dashboard/git_import.py @@ -0,0 +1,207 @@ +""" +Provides a function for importing a git repository into the lms +instance when using a mongo modulestore +""" + +import os +import re +import StringIO +import subprocess +import logging + +from django.conf import settings +from django.core import management +from django.core.management.base import CommandError +from django.utils import timezone +from django.utils.translation import ugettext as _ +import mongoengine + +from dashboard.models import CourseImportLog + +log = logging.getLogger(__name__) + +GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos') +GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True) + + +class GitImportError(Exception): + """ + Exception class for handling the typical errors in a git import. + """ + + NO_DIR = _("Path {0} doesn't exist, please create it, " + "or configure a different path with " + "GIT_REPO_DIR").format(GIT_REPO_DIR) + URL_BAD = _('Non usable git url provided. Expecting something like:' + ' git@github.com:mitocw/edx4edx_lite.git') + BAD_REPO = _('Unable to get git log') + CANNOT_PULL = _('git clone or pull failed!') + XML_IMPORT_FAILED = _('Unable to run import command.') + UNSUPPORTED_STORE = _('The underlying module store does not support import.') + + +def cmd_log(cmd, cwd): + """ + Helper function to redirect stderr to stdout and log the command + used along with the output. Will raise subprocess.CalledProcessError if + command doesn't return 0, and returns the command's output. + """ + output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT) + log.debug('Command was: {0!r}. ' + 'Working directory was: {1!r}'.format(' '.join(cmd), cwd)) + log.debug('Command output was: {0!r}'.format(output)) + return output + + +def add_repo(repo, rdir_in): + """This will add a git repo into the mongo modulestore""" + # pylint: disable=R0915 + + # Set defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', ]: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + if not os.path.isdir(GIT_REPO_DIR): + raise GitImportError(GitImportError.NO_DIR) + # pull from git + if not (repo.endswith('.git') or + repo.startswith(('http:', 'https:', 'git:', 'file:'))): + raise GitImportError(GitImportError.URL_BAD) + + if rdir_in: + rdir = os.path.basename(rdir_in) + else: + rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] + log.debug('rdir = {0}'.format(rdir)) + + rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir) + if os.path.exists(rdirp): + log.info('directory already exists, doing a git pull instead ' + 'of git clone') + cmd = ['git', 'pull', ] + cwd = rdirp + else: + cmd = ['git', 'clone', repo, ] + cwd = GIT_REPO_DIR + + cwd = os.path.abspath(cwd) + try: + ret_git = cmd_log(cmd, cwd=cwd) + except subprocess.CalledProcessError: + raise GitImportError(GitImportError.CANNOT_PULL) + + # get commit id + cmd = ['git', 'log', '-1', '--format=%H', ] + try: + commit_id = cmd_log(cmd, cwd=rdirp) + except subprocess.CalledProcessError: + raise GitImportError(GitImportError.BAD_REPO) + + ret_git += '\nCommit ID: {0}'.format(commit_id) + + # get branch + cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ] + try: + branch = cmd_log(cmd, cwd=rdirp) + except subprocess.CalledProcessError: + raise GitImportError(GitImportError.BAD_REPO) + + ret_git += '{0}Branch: {1}'.format(' \n', branch) + + # Get XML logging logger and capture debug to parse results + output = StringIO.StringIO() + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course', + 'xmodule.modulestore.xml', 'xmodule.seq_module', ] + loggers = [] + + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + loggers.append(logger) + + try: + management.call_command('import', GIT_REPO_DIR, rdir, + nostatic=not GIT_IMPORT_STATIC) + except CommandError: + raise GitImportError(GitImportError.XML_IMPORT_FAILED) + except NotImplementedError: + raise GitImportError(GitImportError.UNSUPPORTED_STORE) + + ret_import = output.getvalue() + + # Remove handler hijacks + for logger in loggers: + logger.setLevel(logging.NOTSET) + logger.removeHandler(import_log_handler) + + course_id = 'unknown' + location = 'unknown' + + # extract course ID from output of import-command-run and make symlink + # this is needed in order for custom course scripts to work + match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)', + ret_import) + if match: + location = match.group(1).strip() + log.debug('location = {0}'.format(location)) + course_id = location.replace('i4x://', '').replace( + '/course/', '/').split('\n')[0].strip() + + cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1]) + log.debug('Studio course dir = {0}'.format(cdir)) + + if os.path.exists(cdir) and not os.path.islink(cdir): + log.debug(' -> exists, but is not symlink') + log.debug(subprocess.check_output(['ls', '-l', ], + cwd=os.path.abspath(cdir))) + try: + os.rmdir(os.path.abspath(cdir)) + except OSError: + log.exception('Failed to remove course directory') + + if not os.path.exists(cdir): + log.debug(' -> creating symlink between {0} and {1}'.format(rdirp, cdir)) + try: + os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir)) + except OSError: + log.exception('Unable to create course symlink') + log.debug(subprocess.check_output(['ls', '-l', ], + cwd=os.path.abspath(cdir))) + + # store import-command-run output in mongo + mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError: + log.exception('Unable to connect to mongodb to save log, please ' + 'check MONGODB_LOG settings') + cil = CourseImportLog( + course_id=course_id, + location=location, + repo_dir=rdir, + created=timezone.now(), + import_log=ret_import, + git_log=ret_git, + ) + cil.save() + + log.debug('saved CourseImportLog for {0}'.format(cil.course_id)) + mdb.disconnect() diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py index 1d095b11a62c..4a184e4cd24d 100644 --- a/lms/djangoapps/dashboard/management/commands/git_add_course.py +++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py @@ -4,212 +4,22 @@ import os import re -import datetime import StringIO import subprocess import logging -from django.conf import settings from django.core import management from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ -import mongoengine +import dashboard.git_import +from dashboard.git_import import GitImportError from dashboard.models import CourseImportLog from xmodule.modulestore.django import modulestore from xmodule.modulestore.xml import XMLModuleStore log = logging.getLogger(__name__) -GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos') -GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True) - -GIT_IMPORT_NO_DIR = -1 -GIT_IMPORT_URL_BAD = -2 -GIT_IMPORT_CANNOT_PULL = -3 -GIT_IMPORT_XML_IMPORT_FAILED = -4 -GIT_IMPORT_UNSUPPORTED_STORE = -5 -GIT_IMPORT_MONGODB_FAIL = -6 -GIT_IMPORT_BAD_REPO = -7 - - -def add_repo(repo, rdir_in): - """This will add a git repo into the mongo modulestore""" - # pylint: disable=R0915 - - # Set defaults even if it isn't defined in settings - mongo_db = { - 'host': 'localhost', - 'user': '', - 'password': '', - 'db': 'xlog', - } - - # Allow overrides - if hasattr(settings, 'MONGODB_LOG'): - for config_item in ['host', 'user', 'password', 'db', ]: - mongo_db[config_item] = settings.MONGODB_LOG.get( - config_item, mongo_db[config_item]) - - if not os.path.isdir(GIT_REPO_DIR): - log.critical(_("Path {0} doesn't exist, please create it, " - "or configure a different path with " - "GIT_REPO_DIR").format(GIT_REPO_DIR)) - return GIT_IMPORT_NO_DIR - - # pull from git - if not repo.endswith('.git') or not ( - repo.startswith('http:') or - repo.startswith('https:') or - repo.startswith('git:') or - repo.startswith('file:')): - - log.error(_('Oops, not a git ssh url?')) - log.error(_('Expecting something like ' - 'git@github.com:mitocw/edx4edx_lite.git')) - return GIT_IMPORT_URL_BAD - - if rdir_in: - rdir = rdir_in - rdir = os.path.basename(rdir) - else: - rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] - - log.debug('rdir = {0}'.format(rdir)) - - rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir) - if os.path.exists(rdirp): - log.info(_('directory already exists, doing a git pull instead ' - 'of git clone')) - cmd = ['git', 'pull', ] - cwd = '{0}/{1}'.format(GIT_REPO_DIR, rdir) - else: - cmd = ['git', 'clone', repo, ] - cwd = GIT_REPO_DIR - - log.debug(cmd) - cwd = os.path.abspath(cwd) - try: - ret_git = subprocess.check_output(cmd, cwd=cwd) - except subprocess.CalledProcessError: - log.exception(_('git clone or pull failed!')) - return GIT_IMPORT_CANNOT_PULL - log.debug(ret_git) - - # get commit id - cmd = ['git', 'log', '-1', '--format=%H', ] - try: - commit_id = subprocess.check_output(cmd, cwd=rdirp) - except subprocess.CalledProcessError: - log.exception(_('Unable to get git log')) - return GIT_IMPORT_BAD_REPO - - ret_git += _('\nCommit ID: {0}').format(commit_id) - - # get branch - cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ] - try: - branch = subprocess.check_output(cmd, cwd=rdirp) - except subprocess.CalledProcessError: - log.exception(_('Unable to get branch info')) - return GIT_IMPORT_BAD_REPO - - ret_git += ' \nBranch: {0}'.format(branch) - - # Get XML logging logger and capture debug to parse results - output = StringIO.StringIO() - import_log_handler = logging.StreamHandler(output) - import_log_handler.setLevel(logging.DEBUG) - - logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course', - 'xmodule.modulestore.xml', 'xmodule.seq_module', ] - loggers = [] - - for logger_name in logger_names: - logger = logging.getLogger(logger_name) - logger.old_level = logger.level - logger.setLevel(logging.DEBUG) - logger.addHandler(import_log_handler) - loggers.append(logger) - - try: - management.call_command('import', GIT_REPO_DIR, rdir, - nostatic=not GIT_IMPORT_STATIC) - except CommandError: - log.exception(_('Unable to run import command.')) - return GIT_IMPORT_XML_IMPORT_FAILED - except NotImplementedError: - log.exception(_('The underlying module store does not support import.')) - return GIT_IMPORT_UNSUPPORTED_STORE - - ret_import = output.getvalue() - - # Remove handler hijacks - for logger in loggers: - logger.setLevel(logger.old_level) - logger.removeHandler(import_log_handler) - - course_id = 'unknown' - location = 'unknown' - - # extract course ID from output of import-command-run and make symlink - # this is needed in order for custom course scripts to work - match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)', - ret_import) - if match: - location = match.group(1).strip() - log.debug('location = {0}'.format(location)) - course_id = location.replace('i4x://', '').replace( - '/course/', '/').split('\n')[0].strip() - - cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1]) - log.debug(_('Studio course dir = {0}').format(cdir)) - - if os.path.exists(cdir) and not os.path.islink(cdir): - log.debug(_(' -> exists, but is not symlink')) - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - try: - os.rmdir(os.path.abspath(cdir)) - except OSError: - log.exception(_('Failed to remove course directory')) - - if not os.path.exists(cdir): - log.debug(_(' -> creating symlink between {0} and {1}').format(rdirp, cdir)) - try: - os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir)) - except OSError: - log.exception(_('Unable to create course symlink')) - log.debug(subprocess.check_output(['ls', '-l', ], - cwd=os.path.abspath(cdir))) - - # store import-command-run output in mongo - mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) - - try: - if mongo_db['user'] and mongo_db['password']: - mdb = mongoengine.connect(mongo_db['db'], host=mongouri) - else: - mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) - except mongoengine.connection.ConnectionError: - log.exception(_('Unable to connect to mongodb to save log, please ' - 'check MONGODB_LOG settings')) - return GIT_IMPORT_MONGODB_FAIL - cil = CourseImportLog( - course_id=course_id, - location=location, - repo_dir=rdir, - created=datetime.datetime.now(), - import_log=ret_import, - git_log=ret_git, - ) - cil.save() - - log.debug(_('saved CourseImportLog for {0}').format(cil.course_id)) - mdb.disconnect() - return 0 - - class Command(BaseCommand): """ Pull a git repo and import into the mongo based content database. @@ -222,21 +32,22 @@ def handle(self, *args, **options): """Check inputs and run the command""" if isinstance(modulestore, XMLModuleStore): - raise CommandError(_('This script requires a mongo module store')) + raise CommandError('This script requires a mongo module store') if len(args) < 1: - raise CommandError(_('This script requires at least one argument, ' - 'the git URL')) + raise CommandError('This script requires at least one argument, ' + 'the git URL') if len(args) > 2: - raise CommandError(_('This script requires no more than two ' - 'arguments')) + raise CommandError('This script requires no more than two ' + 'arguments') rdir_arg = None if len(args) > 1: rdir_arg = args[1] - if add_repo(args[0], rdir_arg) != 0: - raise CommandError(_('Repo was not added, check log output ' - 'for details')) + try: + dashboard.git_import.add_repo(args[0], rdir_arg) + except GitImportError as ex: + raise CommandError(str(ex)) diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py index 0b3c685fbd21..ad4f05eebb55 100644 --- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py +++ b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py @@ -5,6 +5,7 @@ import unittest import os import shutil +import StringIO import subprocess from django.conf import settings @@ -14,7 +15,8 @@ from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -import dashboard.management.commands.git_add_course as git_add_course +import dashboard.git_import as git_import +from dashboard.git_import import GitImportError TEST_MONGODB_LOG = { 'host': 'localhost', @@ -43,8 +45,9 @@ def assertCommandFailureRegexp(self, regex, *args): Convenience function for testing command failures """ with self.assertRaises(SystemExit): - self.assertRaisesRegexp(CommandError, regex, - call_command('git_add_course', *args)) + with self.assertRaisesRegexp(CommandError, regex): + call_command('git_add_course', *args, + stderr=StringIO.StringIO()) def test_command_args(self): """ @@ -78,27 +81,24 @@ def test_add_repo(self): """ Various exit path tests for test_add_repo """ - self.assertEqual(git_add_course.GIT_IMPORT_NO_DIR, - git_add_course.add_repo(self.TEST_REPO, None)) - try: - os.mkdir(getattr(settings, 'GIT_REPO_DIR')) - except OSError: - pass - self.assertEqual(git_add_course.GIT_IMPORT_URL_BAD, - git_add_course.add_repo('foo', None)) + with self.assertRaisesRegexp(GitImportError, GitImportError.NO_DIR): + git_import.add_repo(self.TEST_REPO, None) - self.assertEqual( - git_add_course.GIT_IMPORT_CANNOT_PULL, - git_add_course.add_repo('file:///foobar.git', None) - ) + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + self.addCleanup(shutil.rmtree, getattr(settings, 'GIT_REPO_DIR')) + + with self.assertRaisesRegexp(GitImportError, GitImportError.URL_BAD): + git_import.add_repo('foo', None) + + with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL): + git_import.add_repo('file:///foobar.git', None) # Test git repo that exists, but is "broken" bare_repo = os.path.abspath('{0}/{1}'.format(settings.TEST_ROOT, 'bare.git')) - os.mkdir(os.path.abspath(bare_repo)) - subprocess.call(['git', '--bare', 'init', ], cwd=bare_repo) - - self.assertEqual( - git_add_course.GIT_IMPORT_BAD_REPO, - git_add_course.add_repo('file://{0}'.format(bare_repo), None) - ) - shutil.rmtree(bare_repo) + os.mkdir(bare_repo) + self.addCleanup(shutil.rmtree, bare_repo) + subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT, + cwd=bare_repo) + + with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO): + git_import.add_repo('file://{0}'.format(bare_repo), None) diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py index 9627e37f0b96..a35a5f1a9344 100644 --- a/lms/djangoapps/dashboard/sysadmin.py +++ b/lms/djangoapps/dashboard/sysadmin.py @@ -9,7 +9,6 @@ import subprocess import time import StringIO -from datetime import datetime from django.conf import settings from django.contrib.auth import authenticate @@ -20,6 +19,7 @@ from django.http import HttpResponse, Http404 from django.utils.decorators import method_decorator from django.utils.html import escape +from django.utils import timezone from django.utils.translation import ugettext as _ from django.views.decorators.cache import cache_control from django.views.generic.base import TemplateView @@ -30,7 +30,8 @@ from courseware.courses import get_course_by_id from courseware.roles import CourseStaffRole, CourseInstructorRole -import dashboard.management.commands.git_add_course as git_add_course +import dashboard.git_import as git_import +from dashboard.git_import import GitImportError from dashboard.models import CourseImportLog from external_auth.models import ExternalAuthMap from external_auth.views import generate_password @@ -46,6 +47,7 @@ log = logging.getLogger(__name__) + class SysadminDashboardView(TemplateView): """Base class for sysadmin dashboard views with common methods""" @@ -214,7 +216,7 @@ def create_user(self, uname, name, password=None): external_credentials=json.dumps(credentials), ) eamap.user = user - eamap.dtsignup = datetime.now() + eamap.dtsignup = timezone.now() eamap.save() msg += _('User {0} created successfully!').format(user) @@ -368,7 +370,7 @@ def import_mongo_course(self, gitloc): msg = u'' - logging.debug(_('Adding course using git repo {0}').format(gitloc)) + logging.debug('Adding course using git repo {0}'.format(gitloc)) # Grab logging output for debugging imports output = StringIO.StringIO() @@ -376,28 +378,38 @@ def import_mongo_course(self, gitloc): import_log_handler.setLevel(logging.DEBUG) logger_names = ['xmodule.modulestore.xml_importer', - 'dashboard.management.commands.git_add_course', - 'xmodule.modulestore.xml', 'xmodule.seq_module', ] + 'dashboard.git_import', + 'xmodule.modulestore.xml', + 'xmodule.seq_module', ] loggers = [] for logger_name in logger_names: logger = logging.getLogger(logger_name) - logger.old_level = logger.level logger.setLevel(logging.DEBUG) logger.addHandler(import_log_handler) loggers.append(logger) - git_add_course.add_repo(gitloc, None) + error_msg = '' + try: + git_import.add_repo(gitloc, None) + except GitImportError as ex: + error_msg = str(ex) ret = output.getvalue() # Remove handler hijacks for logger in loggers: - logger.setLevel(logger.old_level) + logger.setLevel(logging.NOTSET) logger.removeHandler(import_log_handler) - msg = u"

{0} {1}

".format( - _('Added course from'), gitloc) - msg += _("
{0}
").format(escape(ret)) + if error_msg: + msg_header = error_msg + color = 'red' + else: + msg_header = _('Added Course') + color = 'blue' + + msg = u"

{1}

".format(color, msg_header) + msg += "
{0}
".format(escape(ret)) return msg def import_xml_course(self, gitloc, datatable): @@ -422,7 +434,9 @@ def import_xml_course(self, gitloc, datatable): cwd = settings.DATA_DIR cwd = os.path.abspath(cwd) try: - cmd_output = escape(subprocess.check_output(cmd, cwd=cwd)) + cmd_output = escape( + subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd) + ) except subprocess.CalledProcessError: return _('Unable to clone or pull repository. Please check your url.') @@ -660,8 +674,8 @@ def get(self, request, *args, **kwargs): else: mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) except mongoengine.connection.ConnectionError: - logging.exception(_('Unable to connect to mongodb to save log, ' - 'please check MONGODB_LOG settings.')) + logging.exception('Unable to connect to mongodb to save log, ' + 'please check MONGODB_LOG settings.') if course_id is None: # Require staff if not going to specific course diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py index 37080723d028..17d88038e5a0 100644 --- a/lms/djangoapps/dashboard/tests/test_sysadmin.py +++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py @@ -2,9 +2,10 @@ Provide tests for sysadmin dashboard feature in sysadmin.py """ -import unittest +import glob import os import shutil +import unittest from django.conf import settings from django.contrib.auth.hashers import check_password @@ -20,6 +21,7 @@ from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from dashboard.models import CourseImportLog from dashboard.sysadmin import Users +from dashboard.git_import import GitImportError from external_auth.models import ExternalAuthMap from student.tests.factories import UserFactory from xmodule.modulestore.django import modulestore @@ -43,15 +45,6 @@ class SysadminBaseTestCase(ModuleStoreTestCase): Base class with common methods used in XML and Mongo tests """ - @classmethod - def tearDownClass(cls): - """Delete all repos imported during tests.""" - super(SysadminBaseTestCase, cls).tearDownClass() - try: - shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) - except OSError: - pass - def setUp(self): """Setup test case by adding primary user.""" super(SysadminBaseTestCase, self).setUp() @@ -74,18 +67,37 @@ def _add_edx4edx(self): def _rm_edx4edx(self): """Deletes the sample course from the XML store""" def_ms = modulestore() + course_path = '{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)) try: # using XML store - course = def_ms.courses.get('{0}/edx4edx_lite'.format( - os.path.abspath(settings.DATA_DIR)), None) + course = def_ms.courses.get(course_path, None) except AttributeError: # Using mongo store course = def_ms.get_course('MITx/edx4edx/edx4edx') # Delete git loaded course - return self.client.post(reverse('sysadmin_courses'), + response = self.client.post(reverse('sysadmin_courses'), {'course_id': course.id, 'action': 'del_course', }) + self.addCleanup(self._rm_glob, '{0}_deleted_*'.format(course_path)) + + return response + + def _rm_glob(self, path): + """ + Create a shell expansion of passed in parameter and iteratively + remove them. Must only expand to directories. + """ + for path in glob.glob(path): + shutil.rmtree(path) + + def _mkdir(self, path): + """ + Create directory and add the cleanup for it. + """ + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), @@ -401,9 +413,7 @@ def test_missing_repo_dir(self): # Create git loaded course response = self._add_edx4edx() - self.assertIn(escape(_("Path {0} doesn't exist, please create it, or " - "configure a different path with " - "GIT_REPO_DIR").format(settings.GIT_REPO_DIR)), + self.assertIn(GitImportError.NO_DIR, response.content.decode('UTF-8')) def test_mongo_course_add_delete(self): @@ -413,8 +423,7 @@ def test_mongo_course_add_delete(self): """ self._setstaff_login() - if not os.path.isdir(getattr(settings, 'GIT_REPO_DIR')): - os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + self._mkdir(getattr(settings, 'GIT_REPO_DIR')) def_ms = modulestore() self.assertFalse(isinstance(def_ms, XMLModuleStore)) @@ -433,10 +442,7 @@ def test_gitlogs(self): """ self._setstaff_login() - try: - os.mkdir(getattr(settings, 'GIT_REPO_DIR')) - except OSError: - pass + self._mkdir(getattr(settings, 'GIT_REPO_DIR')) self._add_edx4edx() response = self.client.get(reverse('gitlogs')) @@ -468,10 +474,7 @@ def test_gitlog_courseteam_access(self): Ensure course team users are allowed to access only their own course. """ - try: - os.mkdir(getattr(settings, 'GIT_REPO_DIR')) - except OSError: - pass + self._mkdir(getattr(settings, 'GIT_REPO_DIR')) self._setstaff_login() self._add_edx4edx()