From 6dce1bbcdda1d85a3ee767a38f12b7993f8707ff Mon Sep 17 00:00:00 2001 From: "Bryan A. Jones" Date: Mon, 26 Jun 2017 11:14:27 -0500 Subject: [PATCH] Fix: Use a single DBURL environment variable, instead of individual DBUSER/DBPASS/DBHOST/DBNAME. --- runestone/activecode/activecode.py | 23 ++++------------ runestone/activecode/test/pavement.py | 2 +- runestone/assess/test/pavement.py | 2 +- runestone/clickableArea/test/pavement.py | 2 +- .../common/project_template/pavement.tmpl | 2 +- runestone/datafile/__init__.py | 9 +++---- runestone/dragndrop/test/pavement.py | 2 +- runestone/external/external.py | 3 +-- runestone/fitb/test/pavement.py | 2 +- runestone/poll/test/pavement.py | 2 +- runestone/question/test/pavement.py | 2 +- runestone/reveal/test/pavement.py | 2 +- runestone/server/__init__.py | 5 ++-- runestone/server/chapternames.py | 19 ++------------ runestone/server/componentdb.py | 26 +++++++------------ runestone/shortanswer/test/pavement.py | 2 +- runestone/tabbedStuff/test/pavement.py | 2 +- runestone/usageAssignment/__init__.py | 13 +++------- 18 files changed, 38 insertions(+), 82 deletions(-) diff --git a/runestone/activecode/activecode.py b/runestone/activecode/activecode.py index 47914f07f..99f4fb8a5 100644 --- a/runestone/activecode/activecode.py +++ b/runestone/activecode/activecode.py @@ -23,7 +23,7 @@ from .textfield import * from sqlalchemy import create_engine, Table, MetaData, select, delete from runestone.server import get_dburl -from runestone.server.componentdb import addQuestionToDB, addHTMLToDB +from runestone.server.componentdb import addQuestionToDB, addHTMLToDB, engine, meta from runestone.common.runestonedirective import RunestoneDirective try: @@ -122,17 +122,6 @@ def process_activcode_nodes(app, env, docname): def purge_activecodes(app, env, docname): pass -database_connection = True -try: - engine = create_engine(get_dburl(locals())) - meta = MetaData() - Source_code = Table('source_code', meta, autoload=True, autoload_with=engine) - Div = Table('div_ids', meta, autoload=True, autoload_with=engine) -except: - print("Cannot connect") - database_connection = False - - class ActiveCode(RunestoneDirective): """ .. activecode:: uniqueid @@ -324,7 +313,8 @@ def run(self): course_name = env.config.html_context['course_id'] divid = self.options['divid'] - try: + if engine: + Source_code = Table('source_code', meta, autoload=True, autoload_with=engine) engine.execute(Source_code.delete().where(Source_code.c.acid == divid).where(Source_code.c.course_id == course_name)) engine.execute(Source_code.insert().values( acid = divid, @@ -339,6 +329,7 @@ def run(self): except: ch, sub_ch = (env.docname, 'null subchapter') + Div = Table('div_ids', meta, autoload=True, autoload_with=engine) engine.execute(Div.delete()\ .where(Div.c.course_name == course_name)\ .where(Div.c.chapter == ch)\ @@ -353,11 +344,7 @@ def run(self): )) - except Exception as e: - import traceback - print("The exception is ", e) - traceback.print_exc() - print(env.config.html_context['course_id']) + else: print("Unable to save to source_code table in activecode.py. Possible problems:") print(" 1. dburl or course_id are not set in conf.py for your book") print(" 2. unable to connect to the database using dburl") diff --git a/runestone/activecode/test/pavement.py b/runestone/activecode/test/pavement.py index e54382b54..c02600d68 100644 --- a/runestone/activecode/test/pavement.py +++ b/runestone/activecode/test/pavement.py @@ -36,7 +36,7 @@ ) ) -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/assess/test/pavement.py b/runestone/assess/test/pavement.py index 8641de746..dc12cbb0a 100644 --- a/runestone/assess/test/pavement.py +++ b/runestone/assess/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/clickableArea/test/pavement.py b/runestone/clickableArea/test/pavement.py index 35ff83333..b9e84d82e 100644 --- a/runestone/clickableArea/test/pavement.py +++ b/runestone/clickableArea/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/common/project_template/pavement.tmpl b/runestone/common/project_template/pavement.tmpl index 80022ef0e..6fe7ea161 100644 --- a/runestone/common/project_template/pavement.tmpl +++ b/runestone/common/project_template/pavement.tmpl @@ -40,7 +40,7 @@ options( version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/datafile/__init__.py b/runestone/datafile/__init__.py index 70b96a077..4ca750c5b 100644 --- a/runestone/datafile/__init__.py +++ b/runestone/datafile/__init__.py @@ -21,7 +21,7 @@ from docutils.parsers.rst import directives from docutils.parsers.rst import Directive from sqlalchemy import create_engine, Table, MetaData, select, delete -from runestone.server import get_dburl +from runestone.server.componentdb import engine, meta from runestone.common.runestonedirective import RunestoneDirective def setup(app): @@ -147,9 +147,7 @@ def run(self): else: self.options['edit'] = "false" - try: - engine = create_engine(get_dburl(locals())) - meta = MetaData() + if engine: Source_code = Table('source_code', meta, autoload=True, autoload_with=engine) course_name = env.config.html_context['course_id'] divid = self.options['divid'] @@ -160,8 +158,7 @@ def run(self): course_id = course_name, main_code= source, )) - except Exception as e: - print("the error is ", e) + else: print("Unable to save to source_code table in datafile__init__.py. Possible problems:") print(" 1. dburl or course_id are not set in conf.py for your book") print(" 2. unable to connect to the database using dburl") diff --git a/runestone/dragndrop/test/pavement.py b/runestone/dragndrop/test/pavement.py index 6f8e27023..d071fe77d 100644 --- a/runestone/dragndrop/test/pavement.py +++ b/runestone/dragndrop/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/external/external.py b/runestone/external/external.py index 8ba2ab76d..15b95dcc1 100644 --- a/runestone/external/external.py +++ b/runestone/external/external.py @@ -19,7 +19,6 @@ from runestone.common.runestonedirective import RunestoneDirective from docutils.parsers.rst import Directive from sqlalchemy import create_engine, Table, MetaData, select, delete -from runestone.server import get_dburl from runestone.server.componentdb import addQuestionToDB, addHTMLToDB from runestone.common.runestonedirective import RunestoneDirective @@ -29,7 +28,7 @@ from cgi import escape # py2 __author__ = 'jczetta' -# Code template is directly from question.py at the moment, which is (c) Bradley N. Miller. +# Code template is directly from question.py at the moment, which is (c) Bradley N. Miller. #This is intended as the basis for a potential new gradeable directive class, still potential TODO. diff --git a/runestone/fitb/test/pavement.py b/runestone/fitb/test/pavement.py index 887306b7c..c887abcda 100644 --- a/runestone/fitb/test/pavement.py +++ b/runestone/fitb/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/poll/test/pavement.py b/runestone/poll/test/pavement.py index 492b67b7a..02db28672 100644 --- a/runestone/poll/test/pavement.py +++ b/runestone/poll/test/pavement.py @@ -36,7 +36,7 @@ ) ) -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/question/test/pavement.py b/runestone/question/test/pavement.py index bc9589d88..3d4005c05 100644 --- a/runestone/question/test/pavement.py +++ b/runestone/question/test/pavement.py @@ -36,7 +36,7 @@ ) ) -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/reveal/test/pavement.py b/runestone/reveal/test/pavement.py index 9f2fda861..9e7d88637 100644 --- a/runestone/reveal/test/pavement.py +++ b/runestone/reveal/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/server/__init__.py b/runestone/server/__init__.py index 53c5e6adc..d016ef80c 100644 --- a/runestone/server/__init__.py +++ b/runestone/server/__init__.py @@ -1,4 +1,3 @@ -from .chapternames import * from os import environ import re @@ -14,8 +13,8 @@ def get_dburl(outer={}): # outer may contain the locals from the calling function # nonlocal env, settings # Python 3 only - if all([x in environ for x in ['DBUSER', 'DBHOST', 'DBNAME']]): - return 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**environ) + if 'DBURL' in environ: + return environ['DBURL'] if 'options' in outer: return outer['options'].build.template_args['dburl'] diff --git a/runestone/server/chapternames.py b/runestone/server/chapternames.py index 44036e527..34f1b52b9 100644 --- a/runestone/server/chapternames.py +++ b/runestone/server/chapternames.py @@ -3,6 +3,7 @@ import re from sqlalchemy import create_engine, Table, MetaData, select, delete +from .componentdb import engine, meta from collections import OrderedDict import sys from functools import reduce @@ -87,26 +88,10 @@ def getTOCEntries(ftext): def addChapterInfoToDB(subChapD, chapTitles, course_id): - dbname = 'runestone' - # Provide a default database URI if the ``USER`` environment variables is define (it is on Linux/Mac). - uname = os.environ.get('USER') - if uname == 'bnmnetp': - uname = 'bnmnetp_courselib' - dbname = 'bnmnetp_courselib' - - dburl = 'postgresql://{}@localhost/{}'.format(uname,dbname) - - - if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']): - dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ) - - try: - engine = create_engine(dburl) - except ImportError as imperr: + if not engine: print("You need to install a DBAPI module - psycopg2 for Postgres") return - meta = MetaData() chapters = Table('chapters', meta, autoload=True, autoload_with=engine) sub_chapters = Table('sub_chapters', meta, autoload=True, autoload_with=engine) diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py index 47010200a..df2b32ff3 100644 --- a/runestone/server/componentdb.py +++ b/runestone/server/componentdb.py @@ -21,21 +21,25 @@ import os from sqlalchemy import create_engine, Table, MetaData, select, delete, update, and_ +from . import get_dburl # create a global DB query engine to share for the rest of the file -if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']): - dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ) +try: + dburl = get_dburl() engine = create_engine(dburl) +except RuntimeError as e: + dburl = None + engine = None + meta = None + print("Skipping all DB operations because environment variables not set up") +else: + # If no exceptions are raised, then set up the database. meta = MetaData() questions = Table('questions', meta, autoload=True, autoload_with=engine) assignment_types = Table('assignment_types', meta, autoload=True, autoload_with=engine) assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine) courses = Table('courses', meta, autoload=True, autoload_with=engine) -else: - dburl = None - engine = None - print("Skipping all DB operations because environment variables not set up") def logSource(self): sourcelog = self.state.document.settings.env.config.html_context.get('dsource', None) @@ -51,11 +55,6 @@ def logSource(self): def addQuestionToDB(self): - if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']): - dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ) - else: - dburl = None - if dburl: basecourse = self.state.document.settings.env.config.html_context.get('basecourse', "unknown") if basecourse == "unknown": @@ -226,11 +225,6 @@ def addAssignmentToDB(name = None, course_id = None, assignment_type_id = None, return a_id def addHTMLToDB(divid, basecourse, htmlsrc): - if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']): - dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ) - else: - dburl = None - if dburl: last_changed = datetime.now() sel = select([questions]).where(and_(questions.c.name == divid, diff --git a/runestone/shortanswer/test/pavement.py b/runestone/shortanswer/test/pavement.py index d4786a7a4..d973f12d5 100644 --- a/runestone/shortanswer/test/pavement.py +++ b/runestone/shortanswer/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/tabbedStuff/test/pavement.py b/runestone/tabbedStuff/test/pavement.py index 3d17fb09b..64dacec54 100644 --- a/runestone/tabbedStuff/test/pavement.py +++ b/runestone/tabbedStuff/test/pavement.py @@ -40,7 +40,7 @@ version = pkg_resources.require("runestone")[0].version options.build.template_args['runestone_version'] = version -# If DBUSER etc. are in the environment override dburl +# If DBURL is in the environment override dburl options.build.template_args['dburl'] = get_dburl(outer=locals()) from runestone import build # build is called implicitly by the paver driver. diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py index 7245ac2e3..a8086b808 100644 --- a/runestone/usageAssignment/__init__.py +++ b/runestone/usageAssignment/__init__.py @@ -23,7 +23,7 @@ from sqlalchemy import create_engine, Table, MetaData, select, delete from sqlalchemy.orm import sessionmaker from runestone.common.runestonedirective import RunestoneDirective -from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID, addAssignmentQuestionToDB, getOrInsertQuestionForPage +from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID, addAssignmentQuestionToDB, getOrInsertQuestionForPage, engine, meta from datetime import datetime from collections import OrderedDict import os @@ -65,7 +65,7 @@ def visit_ua_node(self,node): if d['ch'] not in chapters_and_subchapters: chapters_and_subchapters[d['ch']] = d['sub_chs'] else: - # The order matters with respect to the list wherein they're added to the dictionary. + # The order matters with respect to the list wherein they're added to the dictionary. for subch in d['sub_chs']: chapters_and_subchapters[d['ch']].append(subch) @@ -80,7 +80,7 @@ def visit_ua_node(self,node): s += '' s += '' - # is this needed?? + # is this needed?? s = s.replace("u'","'") # hack: there must be a better way to include the list and avoid unicode strings self.body.append(s) @@ -141,14 +141,9 @@ def run(self): :points: """ - if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']): - dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ) - else: - dburl = None + if not engine: self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Environment variables not set for DB access; can't save usageassignment to DB") return [usageAssignmentNode(self.options)] - engine = create_engine(dburl) - meta = MetaData() # create a configured "Session" class Session = sessionmaker(bind=engine) session = Session()