From f9f24b65074d073080ca9e0b5ba33b156f10cfab Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Tue, 15 Sep 2015 21:51:30 -0400
Subject: [PATCH 01/46] show email address instead of username in user menu

---
 runestone/common/js/bookfuncs.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runestone/common/js/bookfuncs.js b/runestone/common/js/bookfuncs.js
index adbfced43..d51aa47f0 100644
--- a/runestone/common/js/bookfuncs.js
+++ b/runestone/common/js/bookfuncs.js
@@ -92,7 +92,8 @@ function gotUser(data, status, whatever) {
         }
     } else {
         if (!caughtErr) {
-            mess = "username: " + d.nick;
+            // mess = "username: " + d.nick;
+            mess = "user: " + d.email;
             eBookConfig.email = d.email;
             eBookConfig.isLoggedIn = true;
             eBookConfig.cohortId = d.cohortId;

From b0bed81db4fb458c858f2f33607f9e1ea539d116 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 16 Sep 2015 09:56:26 -0400
Subject: [PATCH 02/46] use document.getElementById to find div when id may
 have weird characters in it; update varnames to be clearer

---
 runestone/activecode/js/activecode.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/runestone/activecode/js/activecode.js b/runestone/activecode/js/activecode.js
index dbea47989..e45349f8c 100755
--- a/runestone/activecode/js/activecode.js
+++ b/runestone/activecode/js/activecode.js
@@ -1471,19 +1471,21 @@ ACFactory.createActiveCode = function (orig, lang) {
 }
 
 // used by web2py controller(s)
-ACFactory.addActiveCodeToDiv = function(outerdiv, acdiv, sid, initialcode, language) {
+ACFactory.addActiveCodeToDiv = function(outerdivid, acdivid, sid, initialcode, language) {
     var  thepre, newac;
-    $("#"+acdiv).empty();
+
+    acdiv = document.getElementById(acdivid);
+    $(acdiv).empty();
     thepre = document.createElement("textarea");
     thepre['data-component'] = "activecode";
     thepre.id = acdiv;
     $(thepre).data('lang', language);
-    $("#"+acdiv).append(thepre);
+    $(acdiv).append(thepre);
     var opts = {'orig' : thepre, 'useRunestoneServices': true };
 
     newac = ACFactory.createActiveCode(thepre,language);
     savediv = newac.divid;
-    newac.divid = outerdiv;
+    newac.divid = outerdivid;
     newac.sid = sid;
     if (! initialcode ) {
         newac.loadEditor();

From 7f952d82348226096fd71627b3c6e24a4836c462 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 26 Dec 2015 15:25:23 -0500
Subject: [PATCH 03/46] put width:60% back in .css so that enumerated items
 don't go wide

---
 runestone/common/css/runestone-custom-sphinx-bootstrap.css | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runestone/common/css/runestone-custom-sphinx-bootstrap.css b/runestone/common/css/runestone-custom-sphinx-bootstrap.css
index e1f219565..1548a702f 100644
--- a/runestone/common/css/runestone-custom-sphinx-bootstrap.css
+++ b/runestone/common/css/runestone-custom-sphinx-bootstrap.css
@@ -49,7 +49,8 @@ div.section {
 	padding-right:0;
 }
  .container  .section >*:not(.section) {
-	max-width: 500pt;
+	/*max-width: 500pt;*/
+	width: 60%;
 	margin-left: auto;
 	margin-right: auto;
 }

From fa8e2ab9226607a41e9587ba24ccacaf69b4199a Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Tue, 5 Jan 2016 15:45:09 -0500
Subject: [PATCH 04/46] usageassignment now generates HTML with list of links
 to subchapters

---
 runestone/usageAssignment/__init__.py | 112 ++++++++++++++++----------
 1 file changed, 70 insertions(+), 42 deletions(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 0d03672c4..2b14c8c7c 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2011  Bradley N. Miller
+# Copyright (C) 2015  Paul Resnick
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,39 +27,50 @@
 def setup(app):
     app.add_directive('usageassignment',usageAssignment)
 
-    # app.add_node(usageAssignmentNode, html=(visit_df_node, depart_df_node))
+    app.add_node(usageAssignmentNode, html=(visit_ua_node, depart_ua_node))
 
     app.connect('doctree-resolved',process_nodes)
     app.connect('env-purge-doc', purge)
 
-
-# class usageAssignmentNode(nodes.General, nodes.Element):
-#     def __init__(self,content):
-#         """
-#         Arguments:
-#         - `self`:
-#         - `content`:
-#         """
-#         super(DataFileNode,self).__init__()
-#         self.df_content = content
+class usageAssignmentNode(nodes.General, nodes.Element):
+    def __init__(self,content):
+        """
+        Arguments:
+        - `self`:
+        - `content`:
+        """
+        super(usageAssignmentNode,self).__init__()
+        self.ua_content = content
 
 # self for these functions is an instance of the writer class.  For example
 # in html, self is sphinx.writers.html.SmartyPantsHTMLTranslator
 # The node that is passed as a parameter is an instance of our node class.
-# def visit_df_node(self,node):
-#     res = TEMPLATE
-#     res = res % node.df_content
-#
-#     res = res.replace("u'","'")  # hack:  there must be a better way to include the list and avoid unicode strings
-#
-#     self.body.append(res)
-#
-# def depart_df_node(self,node):
-#     ''' This is called at the start of processing an datafile node.  If datafile had recursive nodes
-#         etc and did not want to do all of the processing in visit_ac_node any finishing touches could be
-#         added here.
-#     '''
-#     pass
+def visit_ua_node(self,node):
+    course_name = node.ua_content['course_name']
+    s = ""
+    for d in node.ua_content['chapter_data']:
+        ch_name, sub_chs = d['ch'], d['sub_chs']
+        s += '<div class="panel-heading">'
+        s += ch_name
+        s += '<ul class="list-group">'
+        for sub_ch_name in sub_chs:
+            s += '<li class="simple">'
+            s += '<a href = "/runestone/static/%s/%s/%s.html">%s</a>' % (course_name, ch_name, sub_ch_name, sub_ch_name)
+            s += '</li>'
+        s += '</ul>'
+        s += '</div>'
+
+    # is this needed??
+    s = s.replace("u'","'")  # hack:  there must be a better way to include the list and avoid unicode strings
+
+    self.body.append(s)
+
+def depart_ua_node(self,node):
+    ''' This is called at the start of processing a ua node.  If ua had recursive nodes
+        etc and did not want to do all of the processing in visit_ua_node any finishing touches could be
+        added here.
+    '''
+    pass
 
 
 def process_nodes(app,env,docname):
@@ -68,6 +79,8 @@ def process_nodes(app,env,docname):
 
 def purge(app,env,docname):
     pass
+
+
 class usageAssignment(Directive):
     required_arguments = 1  # requires an id for the directive
     optional_arguments = 0
@@ -151,34 +164,49 @@ def run(self):
         session = Session()
 
         course_name = env.config.html_context['course_id']
+        self.options['course_name'] = course_name
         course_id = str(session.query(Course).filter(Course.c.course_name == course_name).first().id)
 
         # Accumulate all the Chapters and SubChapters that are to be visited
         # For each chapter, accumulate all subchapters
+        self.options['chapter_data'] = []
         sub_chs = []
         if 'chapters' in self.options:
-            for nm in self.options.get('chapters').split(','):
-                ch = session.query(Chapter).filter(Chapter.c.course_id == course_name,
-                                                   Chapter.c.chapter_label == nm.strip()).first()
-                results = session.query(SubChapter).filter(SubChapter.c.chapter_id == str(ch.id)).all()
-                sub_chs += results
+            try:
+                for nm in self.options.get('chapters').split(','):
+                    ch = session.query(Chapter).filter(Chapter.c.course_id == course_name,
+                                                       Chapter.c.chapter_label == nm.strip()).first()
+
+                    results = session.query(SubChapter).filter(SubChapter.c.chapter_id == str(ch.id)).all()
+                    sub_chs += results
+                    chapter_data = {'ch': nm, 'sub_chs': [r.sub_chapter_label for r in results]}
+                    self.options['chapter_data'].append(chapter_data)
+            except:
+                print("Chapters requested not found: %s" % (self.options.get('chapters')))
         # Add any explicit subchapters
         if 'subchapters' in self.options:
-            for nm in self.options.get('subchapters').split(','):
-                (ch_dir, subch_name) = nm.strip().split('/')
-                ch_id = session.query(Chapter).filter(Chapter.c.course_id == course_name, Chapter.c.chapter_label == ch_dir).first().id
-                subch = session.query(SubChapter).filter(SubChapter.c.chapter_id == ch_id, SubChapter.c.sub_chapter_label == subch_name).first()
-                sub_chs.append(subch)
+            try:
+                for nm in self.options.get('subchapters').split(','):
+                    (ch_dir, subch_name) = nm.strip().split('/')
+                    ch_id = session.query(Chapter).filter(Chapter.c.course_id == course_name, Chapter.c.chapter_label == ch_dir).first().id
+                    subch = session.query(SubChapter).filter(SubChapter.c.chapter_id == ch_id, SubChapter.c.sub_chapter_label == subch_name).first()
+                    sub_chs.append(subch)
+                    self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
+            except:
+                print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
 
         # Accumulate all the ActiveCodes that are to be run and URL paths to be visited
         divs = []
         paths = []
         for subch in sub_chs:
-            ch_name = session.query(Chapter).filter(Chapter.c.id == subch.chapter_id).first().chapter_label
-            divs += session.query(Div).filter(Div.c.course_name == course_name,
-                                              Div.c.chapter == ch_name,
-                                              Div.c.subchapter == subch.sub_chapter_label).all()
-            paths.append('/runestone/static/%s/%s/%s.html' % (course_name, ch_name, subch.sub_chapter_label))
+            try:
+                ch_name = session.query(Chapter).filter(Chapter.c.id == subch.chapter_id).first().chapter_label
+                divs += session.query(Div).filter(Div.c.course_name == course_name,
+                                                  Div.c.chapter == ch_name,
+                                                  Div.c.subchapter == subch.sub_chapter_label).all()
+                paths.append('/runestone/static/%s/%s/%s.html' % (course_name, ch_name, subch.sub_chapter_label))
+            except:
+                print ("Subchapter not found: %s" % (subch))
         tracked_div_types = ['activecode', 'actex']
         active_codes = [d.div_id for d in divs if d.div_type in tracked_div_types]
 
@@ -240,4 +268,4 @@ def run(self):
 
         session.commit()
 
-        return []
+        return [usageAssignmentNode(self.options)]

From 31846119d3a495a475324139154ceeac2245e76e Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sun, 10 Jan 2016 10:31:21 -0500
Subject: [PATCH 05/46] get rid of spaces in URLs generated by usageassignment
 directive

---
 runestone/usageAssignment/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 2b14c8c7c..8e34466e8 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -174,8 +174,9 @@ def run(self):
         if 'chapters' in self.options:
             try:
                 for nm in self.options.get('chapters').split(','):
+                    nm = nm.strip()
                     ch = session.query(Chapter).filter(Chapter.c.course_id == course_name,
-                                                       Chapter.c.chapter_label == nm.strip()).first()
+                                                       Chapter.c.chapter_label == nm).first()
 
                     results = session.query(SubChapter).filter(SubChapter.c.chapter_id == str(ch.id)).all()
                     sub_chs += results

From ca8b52c5cd71d057afb3fbd311a9e85bf2f91183 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Fri, 15 Jan 2016 16:44:14 -0500
Subject: [PATCH 06/46] better error reporting for not found subchapters

---
 runestone/usageAssignment/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 8e34466e8..c57c829d7 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -192,6 +192,8 @@ def run(self):
                     ch_id = session.query(Chapter).filter(Chapter.c.course_id == course_name, Chapter.c.chapter_label == ch_dir).first().id
                     subch = session.query(SubChapter).filter(SubChapter.c.chapter_id == ch_id, SubChapter.c.sub_chapter_label == subch_name).first()
                     sub_chs.append(subch)
+                    if not subch:
+                        print("problem with: %s" % nm)
                     self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
             except:
                 print("Subchapters requested not found: %s" % (self.options.get('subchapters')))

From 1823711a4068f93975e2a90b14fdc74ba3f4e741 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Fri, 10 Jun 2016 10:04:55 -0400
Subject: [PATCH 07/46] reset to master's version of .css

---
 .../css/runestone-custom-sphinx-bootstrap.css     | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/runestone/common/css/runestone-custom-sphinx-bootstrap.css b/runestone/common/css/runestone-custom-sphinx-bootstrap.css
index 196581037..eaba59316 100644
--- a/runestone/common/css/runestone-custom-sphinx-bootstrap.css
+++ b/runestone/common/css/runestone-custom-sphinx-bootstrap.css
@@ -48,13 +48,22 @@ div.section {
 	padding-left:0;
 	padding-right:0;
 }
- .container  .section >*:not(.section) {
-	/*max-width: 500pt;*/
-	width: 60%;
+.container  .section >*:not(.section) {
+	max-width: 500pt;
 	margin-left: auto;
 	margin-right: auto;
 }
 
+/* This rule is meant to override the behavior of the
+   previous rule since it is not possible to exclude
+   more than one section in the not() part of the rule
+*/
+.container .section div.full-width.container {
+	margin-left: auto;
+	margin-right: auto;
+    max-width: 90%;
+}
+
 .container .section>img {
 	display: block;
 	margin-left: auto;

From 12b2af2eb9708107993190ba582bbb9cbe2576f4 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Thu, 11 Aug 2016 15:15:53 -0400
Subject: [PATCH 08/46] fix typo

---
 runestone/assess/multiplechoice.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runestone/assess/multiplechoice.py b/runestone/assess/multiplechoice.py
index 09f839e01..92d4e1013 100644
--- a/runestone/assess/multiplechoice.py
+++ b/runestone/assess/multiplechoice.py
@@ -161,7 +161,7 @@ def run(self):
 class MChoiceMA(MChoice):
     def run(self):
         self.options['multiple_answers'] = 'multipleAnswers'
-        print("This directive has been depreciated. Please convert to the new directive 'mchoice'")
+        print("This directive has been deprecated. Please convert to the new directive 'mchoice'")
         mchoicemaNode = super(MChoiceMA,self).run()[0]
 
         return [mchoicemaNode]
@@ -169,7 +169,7 @@ def run(self):
 class MChoiceRandomMF(MChoice):
     def run(self):
         self.options['random'] = 'random'
-        print("This directive has been depreciated. Please convert to the new directive 'mchoice'")
+        print("This directive has been deprecated. Please convert to the new directive 'mchoice'")
         mchoicerandommfNode = super(MChoiceRandomMF,self).run()[0]
 
         return[mchoicerandommfNode]

From 9ccb40adc85e6abf7831f4c9364579a259234aa9 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Fri, 26 Aug 2016 15:34:19 -0400
Subject: [PATCH 09/46] 
 https://github.com/RunestoneInteractive/RunestoneComponents/issues/248

---
 runestone/question/question.py  |  3 ++-
 runestone/server/componentdb.py | 10 ++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/runestone/question/question.py b/runestone/question/question.py
index a5f2c64b2..ca00d5c03 100644
--- a/runestone/question/question.py
+++ b/runestone/question/question.py
@@ -89,7 +89,8 @@ class QuestionDirective(RunestoneDirective):
     final_argument_whitespace = True
     has_content = True
     option_spec = RunestoneDirective.option_spec.copy()
-    option_spec.update({'number': directives.positive_int})
+    option_spec.update({'number': directives.positive_int,
+                        'gradeable_div': directives.unchanged})
 
     def run(self):
         self.assert_has_content()  # make sure question has something in it
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 831f185fa..318c72a9e 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -66,20 +66,22 @@ def addQuestionToDB(self):
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
         chapter = srcpath.split(os.path.sep)[-2]
+        gradeable_div = self.options.get('gradeable_div', None)
 
         sel = select([questions]).where(and_(questions.c.name == self.arguments[0],
                                               questions.c.base_course == basecourse))
         res = engine.execute(sel).first()
         try:
             if res:
-                if res['question'] != self.block_text:
-                    stmt = questions.update().where(questions.c.id == res['id']).values(question = self.block_text.encode('utf8'), timestamp=last_changed)
-                    engine.execute(stmt)
+                stmt = questions.update().where(questions.c.id == res['id']).values(question = self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
+question_type=self.name, subchapter=subchapter,
+                                                author=author,difficulty=difficulty,chapter=chapter, gradeable_div=gradeable_div)
+                engine.execute(stmt)
             else:
                 ins = questions.insert().values(base_course=basecourse, name=self.arguments[0],
                                                 question=self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
                                                 question_type=self.name, subchapter=subchapter,
-                                                author=author,difficulty=difficulty,chapter=chapter)
+                                                author=author,difficulty=difficulty,chapter=chapter, gradeable_div=gradeable_div)
                 engine.execute(ins)
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))

From 456cc64b50ee3338fac79af0726c7158a23879bc Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Fri, 26 Aug 2016 15:35:25 -0400
Subject: [PATCH 10/46] add gradebutton flag for activecode

---
 runestone/activecode/activecode.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runestone/activecode/activecode.py b/runestone/activecode/activecode.py
index 185f98c33..47e76e04c 100644
--- a/runestone/activecode/activecode.py
+++ b/runestone/activecode/activecode.py
@@ -153,6 +153,7 @@ class ActiveCode(RunestoneDirective):
         'tour_5': directives.unchanged,
         'nocodelens': directives.flag,
         'coach': directives.flag,
+        'gradebutton': directives.flag,
         'timelimit': directives.unchanged,
         'stdin' : directives.unchanged,
         'datafile' : directives.unchanged,
@@ -263,6 +264,8 @@ def run(self):
 
         if 'gradebutton' not in self.options:
             self.options['gradebutton'] = ''
+        else:
+            self.options['gradebutton'] = "data-gradebutton=true"
 
         if self.content:
             if '====' in self.content:

From 75d5d0f5c706ce2c12b32cd11854ae7a1a3cd850 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 27 Aug 2016 08:56:37 -0400
Subject: [PATCH 11/46] revert processing of gradeable_div for questions; not
 needed now

Conflicts:
	runestone/server/componentdb.py
---
 runestone/question/question.py  | 3 +--
 runestone/server/componentdb.py | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/runestone/question/question.py b/runestone/question/question.py
index ca00d5c03..a5f2c64b2 100644
--- a/runestone/question/question.py
+++ b/runestone/question/question.py
@@ -89,8 +89,7 @@ class QuestionDirective(RunestoneDirective):
     final_argument_whitespace = True
     has_content = True
     option_spec = RunestoneDirective.option_spec.copy()
-    option_spec.update({'number': directives.positive_int,
-                        'gradeable_div': directives.unchanged})
+    option_spec.update({'number': directives.positive_int})
 
     def run(self):
         self.assert_has_content()  # make sure question has something in it
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 853e25e4e..7f9ab7f7a 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -66,7 +66,6 @@ def addQuestionToDB(self):
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
         chapter = srcpath.split(os.path.sep)[-2]
-        gradeable_div = self.options.get('gradeable_div', None)
 
         sel = select([questions]).where(and_(questions.c.name == self.arguments[0],
                                               questions.c.base_course == basecourse))
@@ -74,13 +73,14 @@ def addQuestionToDB(self):
         try:
             if res:
                 stmt = questions.update().where(questions.c.id == res['id']).values(question = self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
-question_type=self.name, subchapter=subchapter, author=author,difficulty=difficulty,chapter=chapter)
+question_type=self.name, subchapter=subchapter,
+                                                author=author,difficulty=difficulty,chapter=chapter)
                 engine.execute(stmt)
             else:
                 ins = questions.insert().values(base_course=basecourse, name=self.arguments[0],
                                                 question=self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
                                                 question_type=self.name, subchapter=subchapter,
-                                                author=author,difficulty=difficulty,chapter=chapter, gradeable_div=gradeable_div)
+                                                author=author,difficulty=difficulty,chapter=chapter)
                 engine.execute(ins)
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))

From e0cfd3daab0855da21eb0b086c7ba7aa9d7a3647 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 27 Aug 2016 18:10:42 -0400
Subject: [PATCH 12/46] additions for autograding

---
 runestone/activecode/activecode.py |  3 ++-
 runestone/server/componentdb.py    | 10 ++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/runestone/activecode/activecode.py b/runestone/activecode/activecode.py
index 47e76e04c..eccad975d 100644
--- a/runestone/activecode/activecode.py
+++ b/runestone/activecode/activecode.py
@@ -158,7 +158,8 @@ class ActiveCode(RunestoneDirective):
         'stdin' : directives.unchanged,
         'datafile' : directives.unchanged,
         'sourcefile' : directives.unchanged,
-        'available_files' : directives.unchanged
+        'available_files' : directives.unchanged,
+        'autograde': directives.unchanged
     })
 
     def run(self):
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 7f9ab7f7a..46bdbf60e 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -66,6 +66,8 @@ def addQuestionToDB(self):
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
         chapter = srcpath.split(os.path.sep)[-2]
+        grading_type = self.options.get('autograde', None)
+        print (self.arguments[0], grading_type)
 
         sel = select([questions]).where(and_(questions.c.name == self.arguments[0],
                                               questions.c.base_course == basecourse))
@@ -73,14 +75,10 @@ def addQuestionToDB(self):
         try:
             if res:
                 stmt = questions.update().where(questions.c.id == res['id']).values(question = self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
-question_type=self.name, subchapter=subchapter,
-                                                author=author,difficulty=difficulty,chapter=chapter)
+question_type=self.name, subchapter=subchapter, grading_type = grading_type, author=author,difficulty=difficulty,chapter=chapter)
                 engine.execute(stmt)
             else:
-                ins = questions.insert().values(base_course=basecourse, name=self.arguments[0],
-                                                question=self.block_text.encode('utf8'), timestamp=last_changed, is_private='F',
-                                                question_type=self.name, subchapter=subchapter,
-                                                author=author,difficulty=difficulty,chapter=chapter)
+                ins = questions.insert().values(base_course=basecourse, name=self.arguments[0], question=self.block_text.encode('utf8'), timestamp=last_changed, is_private='F', question_type=self.name, subchapter=subchapter, grading_type = grading_type, author=author,difficulty=difficulty,chapter=chapter)
                 engine.execute(ins)
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))

From 04cebeb2257c67ba64c3a19309a848d05d0c968e Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Mon, 29 Aug 2016 10:26:00 -0400
Subject: [PATCH 13/46] Add files for external exercise directive.

---
 runestone/external/__init__.py |  1 +
 runestone/external/external.py | 87 ++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 runestone/external/__init__.py
 create mode 100644 runestone/external/external.py

diff --git a/runestone/external/__init__.py b/runestone/external/__init__.py
new file mode 100644
index 000000000..93114cdc4
--- /dev/null
+++ b/runestone/external/__init__.py
@@ -0,0 +1 @@
+from .external import *
\ No newline at end of file
diff --git a/runestone/external/external.py b/runestone/external/external.py
new file mode 100644
index 000000000..62fd180cd
--- /dev/null
+++ b/runestone/external/external.py
@@ -0,0 +1,87 @@
+# Copyright (C) 2016  Bradley N. Miller
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+__author__ = 'jczetta'
+# Most of the code from question.py and shortanswer.py, (c) Bradley N. Miller.
+import os
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+from docutils.parsers.rst import Directive
+from runestone.assess import Assessment
+from runestone.server.componentdb import addQuestionToDB, addHTMLToDB
+from runestone.common.runestonedirective import RunestoneDirective
+try:
+    from html import escape  # py3
+except ImportError:
+    from cgi import escape  # py2
+
+def setup(app):
+    app.add_directive('external', ExternalDirective)
+    app.add_node(ExternalNode, html=(visit_external_node, depart_external_node))
+
+# NOTE: This external questions directive is intended for pointing students to external exercises, yet still linking the instructions to a grade. There is no autograde capability, because the activity is entiely external to the textbook. 
+# This directive and activecode by extension may be used as tpls during a refactor of a base class -> using inheritance in gradable directives. That's still a potential TODO.
+
+TEXT = """
+<p data-component="external" id=%(divid)s %(optional)s> %(content)s</p>
+"""
+
+class ExternalNode(nodes.General, nodes.Element):
+    def __init__(self, options):
+        super(ExternalNode, self).__init__()
+        self.external_options = options
+
+def visit_external_node(self, node):
+    div_id = node.external_options['divid']
+    components = dict(node.external_options)
+    components.update({'divid': div_id})
+    res = TEXT % components
+    addHTMLToDB(div_id, components['basecourse'], res)
+
+    self.body.append(res)
+
+def depart_external_node(self,node):
+    pass
+
+
+class ExternalDirective(Assessment): # RunestoneDirective or Assessment inheritance?
+    """
+.. external:: uniqueid
+
+   text of the question goes here
+    """
+    required_arguments = 1  # the div id
+    optional_arguments = 0
+    final_argument_whitespace = True
+    has_content = True
+    option_spec = RunestoneDirective.option_spec.copy()
+    option_spec.update({'optional': directives.flag})
+
+    node_class = ExternalNode
+
+    # This run method is also meant to be the basis for a class to be inherited from. We can build up from here, and also remove all unnecessary things.
+    def run(self):
+        # Raise an error if the directive does not have contents.
+        self.assert_has_content()
+        addQuestionToDB(self)
+        # keeping the optional for now in case we want to use that designation for grading
+        self.options['optional'] = 'data-optional' if 'optional' in self.options else '' # May be useful
+        self.options['divid'] = self.arguments[0]
+        self.options['content'] = "<p>".join(self.content)
+    
+        external_node = ExternalNode(self.options)
+
+        return [external_node]

From 242676ea41595e1932f4c5fe158907d83ed8effa Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Mon, 29 Aug 2016 15:50:41 -0400
Subject: [PATCH 14/46] Minimal cleanup.

---
 runestone/external/__init__.py | 2 +-
 runestone/external/external.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runestone/external/__init__.py b/runestone/external/__init__.py
index 93114cdc4..ff94a4bc0 100644
--- a/runestone/external/__init__.py
+++ b/runestone/external/__init__.py
@@ -1 +1 @@
-from .external import *
\ No newline at end of file
+from .external import *
diff --git a/runestone/external/external.py b/runestone/external/external.py
index 62fd180cd..1f83c676d 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -57,7 +57,7 @@ def depart_external_node(self,node):
     pass
 
 
-class ExternalDirective(Assessment): # RunestoneDirective or Assessment inheritance?
+class ExternalDirective(Assessment):
     """
 .. external:: uniqueid
 
@@ -72,7 +72,7 @@ class ExternalDirective(Assessment): # RunestoneDirective or Assessment inherita
 
     node_class = ExternalNode
 
-    # This run method is also meant to be the basis for a class to be inherited from. We can build up from here, and also remove all unnecessary things.
+    # This run method is also meant to be the basis for a class to be inherited from. We can build up from here, and continue to remove all unnecessary things.
     def run(self):
         # Raise an error if the directive does not have contents.
         self.assert_has_content()

From 951f706f05202e169b287e11544497b1a4f0bccf Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Mon, 29 Aug 2016 15:58:34 -0400
Subject: [PATCH 15/46] This inheritance may be more sensible.

---
 runestone/external/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runestone/external/external.py b/runestone/external/external.py
index 1f83c676d..3f3e7bcf6 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -57,7 +57,7 @@ def depart_external_node(self,node):
     pass
 
 
-class ExternalDirective(Assessment):
+class ExternalDirective(RunestoneDirective):
     """
 .. external:: uniqueid
 

From af68d9d2a8fb3abdf8ebbeba82270f4a81fa5883 Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Tue, 30 Aug 2016 14:58:24 -0400
Subject: [PATCH 16/46] Add html to db properly.

---
 runestone/external/external.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/runestone/external/external.py b/runestone/external/external.py
index 3f3e7bcf6..0993a36bd 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -54,7 +54,19 @@ def visit_external_node(self, node):
     self.body.append(res)
 
 def depart_external_node(self,node):
-    pass
+    # From activecode:
+    ''' This is called at the start of processing an activecode node.  If activecode had recursive nodes
+        etc and did not want to do all of the processing in visit_ac_node any finishing touches could be
+        added here.
+    '''
+    res = TEXT % node.ac_components
+    self.body.append(res)
+
+    addHTMLToDB(node.ac_components['divid'],
+                node.ac_components['basecourse'],
+                "".join(self.body[self.body.index(node.delimiter) + 1:]))
+
+    self.body.remove(node.delimiter)
 
 
 class ExternalDirective(RunestoneDirective):
@@ -84,4 +96,5 @@ def run(self):
     
         external_node = ExternalNode(self.options)
 
+
         return [external_node]

From 706a94d778c5291a17845c114677eedfd2ba245f Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Tue, 30 Aug 2016 16:51:54 -0400
Subject: [PATCH 17/46] Switch to using question.py code as template for
 external directive. TODO: make it easier to refactor from this basis, CSS to
 fix formatting to be its own and static with other 'problem set' problems.

---
 runestone/external/external.py | 94 +++++++++++++++++++---------------
 1 file changed, 52 insertions(+), 42 deletions(-)

diff --git a/runestone/external/external.py b/runestone/external/external.py
index 0993a36bd..4bd8494dc 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016  Bradley N. Miller
+# Copyright (C) 2011  Bradley N. Miller
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,88 +13,98 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-__author__ = 'jczetta'
-# Most of the code from question.py and shortanswer.py, (c) Bradley N. Miller.
-import os
 
 from docutils import nodes
 from docutils.parsers.rst import directives
+from runestone.common.runestonedirective import RunestoneDirective
 from docutils.parsers.rst import Directive
-from runestone.assess import Assessment
+from sqlalchemy import create_engine, Table, MetaData, select, delete
+from runestone.server import get_dburl
 from runestone.server.componentdb import addQuestionToDB, addHTMLToDB
 from runestone.common.runestonedirective import RunestoneDirective
+
 try:
     from html import escape  # py3
 except ImportError:
     from cgi import escape  # py2
 
+__author__ = 'jczetta'
+# Code template is directly from question.py at the moment, which is (c) Bradley N. Miller. 
+#This is intended as the basis for a potential new gradeable directive class, still potential TODO.
+
+
 def setup(app):
     app.add_directive('external', ExternalDirective)
-    app.add_node(ExternalNode, html=(visit_external_node, depart_external_node))
 
-# NOTE: This external questions directive is intended for pointing students to external exercises, yet still linking the instructions to a grade. There is no autograde capability, because the activity is entiely external to the textbook. 
-# This directive and activecode by extension may be used as tpls during a refactor of a base class -> using inheritance in gradable directives. That's still a potential TODO.
+    app.add_node(ExternalNode, html=(visit_external_node, depart_external_node))
 
-TEXT = """
-<p data-component="external" id=%(divid)s %(optional)s> %(content)s</p>
-"""
 
 class ExternalNode(nodes.General, nodes.Element):
-    def __init__(self, options):
+    def __init__(self, content):
         super(ExternalNode, self).__init__()
-        self.external_options = options
+        self.external_options = content
+
 
 def visit_external_node(self, node):
-    div_id = node.external_options['divid']
-    components = dict(node.external_options)
-    components.update({'divid': div_id})
-    res = TEXT % components
-    addHTMLToDB(div_id, components['basecourse'], res)
+    # Set options and format templates accordingly
+    # env = node.document.settings.env
 
+    res = TEMPLATE_START % node.external_options
     self.body.append(res)
 
-def depart_external_node(self,node):
-    # From activecode:
-    ''' This is called at the start of processing an activecode node.  If activecode had recursive nodes
-        etc and did not want to do all of the processing in visit_ac_node any finishing touches could be
-        added here.
-    '''
-    res = TEXT % node.ac_components
+    addHTMLToDB(node.external_options['divid'],
+                node.external_options['basecourse'],
+                "".join(""))
+
+    # self.body.remove(node.delimiter)
+
+
+def depart_external_node(self, node):
+    # Set options and format templates accordingly
+    res = TEMPLATE_END % node.external_options
+    delimiter = "_start__{}_".format(node.external_options['divid'])
+
     self.body.append(res)
 
-    addHTMLToDB(node.ac_components['divid'],
-                node.ac_components['basecourse'],
-                "".join(self.body[self.body.index(node.delimiter) + 1:]))
 
-    self.body.remove(node.delimiter)
+# Templates to be formatted by node options
+TEMPLATE_START = '''
+    <div data-component="question" class="full-width container question" id="%(divid)s" >
+    <li class="alert alert-warning">
+
+    '''
+TEMPLATE_END = '''
+    </li>
+    </div>
+    '''
 
 
 class ExternalDirective(RunestoneDirective):
     """
-.. external:: uniqueid
+.. external:: identifier
 
-   text of the question goes here
+   Content  Everything here is part of the activity
+   Content  Can include links...
     """
-    required_arguments = 1  # the div id
+    required_arguments = 1
     optional_arguments = 0
     final_argument_whitespace = True
     has_content = True
     option_spec = RunestoneDirective.option_spec.copy()
-    option_spec.update({'optional': directives.flag})
-
-    node_class = ExternalNode
+    option_spec.update({'number': directives.positive_int})
 
-    # This run method is also meant to be the basis for a class to be inherited from. We can build up from here, and continue to remove all unnecessary things.
     def run(self):
-        # Raise an error if the directive does not have contents.
-        self.assert_has_content()
         addQuestionToDB(self)
-        # keeping the optional for now in case we want to use that designation for grading
-        self.options['optional'] = 'data-optional' if 'optional' in self.options else '' # May be useful
+
+        self.assert_has_content()  # make sure activity has something in it
         self.options['divid'] = self.arguments[0]
-        self.options['content'] = "<p>".join(self.content)
-    
+        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
+
+        self.options['name'] = self.arguments[0].strip()
+
         external_node = ExternalNode(self.options)
+        self.add_name(external_node)
 
+        self.state.nested_parse(self.content, self.content_offset, external_node)
 
         return [external_node]

From 62ada9ac691c904b32045a5e1c47a0080758a703 Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Tue, 30 Aug 2016 17:00:01 -0400
Subject: [PATCH 18/46] Add some extra css. This could be cleaned up/should
 change as evolved.

---
 runestone/external/css/external.css | 109 ++++++++++++++++++++++++++++
 runestone/external/external.py      |   2 +-
 2 files changed, 110 insertions(+), 1 deletion(-)
 create mode 100644 runestone/external/css/external.css

diff --git a/runestone/external/css/external.css b/runestone/external/css/external.css
new file mode 100644
index 000000000..8768f5ec3
--- /dev/null
+++ b/runestone/external/css/external.css
@@ -0,0 +1,109 @@
+.modal-profile {
+	display:none;
+    min-height: 300px;
+    overflow: hidden;
+	width: 700px;
+	padding:25px;
+	border:1px solid #fff;
+	box-shadow: 0px 2px 7px #292929;
+	-moz-box-shadow: 0px 2px 7px #292929;
+	-webkit-box-shadow: 0px 2px 7px #292929;
+	border-radius:10px;
+	-moz-border-radius:10px;
+	-webkit-border-radius:10px;
+	background: #f2f2f2;
+	z-index:50;
+}
+
+.modal-lightsout {
+	display:none;
+    position:absolute;
+	top:0;
+	left:0;
+	width:100%;
+	z-index:25;
+	background:#000 ;
+}
+
+.modal-close-profile {
+	display:none;
+	position:absolute;
+    height: 43px;
+	width: 43px;
+    background-image: url('close.png');
+	top:1px;
+	right:0.5px;
+}
+
+
+.ac_actions{
+	text-align: center;
+}
+
+.ac_sep {
+	background: #000;
+	display: inline-block;
+	margin: -15px 10px;
+	width: 1px;
+	height: 35px;
+	padding: 0;
+	border: 0;
+}
+.ac_section{
+	position: relative;
+	clear:both;
+}
+.ac_section>* {
+	max-width: 500pt;
+	margin-left: auto;
+	margin-right: auto;
+	position:relative;
+}
+.ac_section .clearfix{
+	position: initial;
+}
+.ac_output{
+	display:none;
+	max-width: 450px;
+}
+
+.ac_caption {
+    text-align: center;
+    font-weight: bold;
+}
+
+.ac_caption_text {
+    font-weight: normal;
+}
+.ac_caption:before {
+    content: "ActiveCode: " counter(activecode) " ";
+    counter-increment: activecode;
+}
+
+.active_out {
+    background-color:#dcdcdc;
+    border-radius: 6px;
+    min-width: 20em;
+    max-height: 300px;
+    overflow: auto;
+}
+
+.visible-ac-canvas {
+    border: 2px solid black;
+}
+
+.ac_section>.col-md-12 {
+	max-width: 100% !important;
+}
+
+.full_width ol {
+	max-width: 100% !important;
+}
+
+.ac-disabled {
+	pointer-events: none;
+}
+
+.ac-feedback {
+	border: 1px solid black;
+}
diff --git a/runestone/external/external.py b/runestone/external/external.py
index 4bd8494dc..2706ded9e 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -69,7 +69,7 @@ def depart_external_node(self, node):
 
 # Templates to be formatted by node options
 TEMPLATE_START = '''
-    <div data-component="question" class="full-width container question" id="%(divid)s" >
+    <div data-component="external" class="full-width container external" id="%(divid)s" >
     <li class="alert alert-warning">
 
     '''

From 9286816514384286ccd117266e0daf9ac8050e35 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Tue, 30 Aug 2016 18:10:58 -0400
Subject: [PATCH 19/46] first version of assignment component; not debuggede

---
 runestone/server/componentdb.py | 104 +++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 1 deletion(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 46bdbf60e..47b8f179a 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -67,7 +67,6 @@ def addQuestionToDB(self):
         subchapter = os.path.basename(srcpath).replace('.rst','')
         chapter = srcpath.split(os.path.sep)[-2]
         grading_type = self.options.get('autograde', None)
-        print (self.arguments[0], grading_type)
 
         sel = select([questions]).where(and_(questions.c.name == self.arguments[0],
                                               questions.c.base_course == basecourse))
@@ -83,6 +82,109 @@ def addQuestionToDB(self):
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))
 
+def getOrCreateAssignmentType(assignment_type_name):
+    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+    else:
+        dburl = None
+    engine = create_engine(dburl)
+    meta = MetaData()
+    assignment_types = Table('assignment_types', meta, autoload=True, autoload_with=engine)
+
+    res = engine.execute(sel).first()
+    if res:
+        return res['id']
+    else:
+        # create the assignment type
+        ins = assignment_types.insert().values(
+            name=assignment_type_name)
+        return engine.execute(ins).inserted_primary_key()
+
+def addAssignmentQuestionToDB(self, basecourse, assignment_id, question_name, points, timed):
+    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+    else:
+        dburl = None
+
+    engine = create_engine(dburl)
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+    assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
+
+    # first get the question_id associated with question_name
+    sel = select([questions]).where(and_(questions.c.name == question_name,
+                                          questions.c.base_course == basecourse))
+    res = engine.execute(sel).first()
+    question_id = res['id']
+
+    # now insert or update the assignment_questions row
+    sel = select([assignment_questions]).where(and_(assignment_questions.c.assignment_id == assignment_id,
+                                          assignment_questions.c.question_id == question_id))
+    res = engine.execute(sel).first()
+    if res:
+        #insert
+        pass
+    else:
+        #update
+        pass
+
+def addAssignmentToDB(self):
+    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+    else:
+        dburl = None
+
+    if dburl:
+        basecourse = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
+        if basecourse == "unknown":
+            raise self.severe("Cannot update database because basecourse is unknown")
+            return
+
+        last_changed = datetime.now()
+
+        engine = create_engine(dburl)
+        meta = MetaData()
+        assignments = Table('assignments', meta, autoload=True, autoload_with=engine)
+        assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
+
+        """
+            .. assignment:
+                :name: Problem Set 1
+                :assignment_type: formative
+                :questions: (divid_1 50), (divid_2 100), ...
+                :deadline: <some datetime>
+                :points: integer
+        """
+        assignment_type_name = self.options.get('assignment_type_name', 'Problem Set')
+        assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
+
+        name = self.options.get('name') # required; error if missing
+        if 'name' in self.options:
+            deadline = datetime.strptime(self.options['name'], '%m/%d/%Y %H:%M')
+        else:
+            deadline = None
+        points = self.options.get('points', None)
+
+        sel = select([assignments]).where(and_(assignments.c.name == assignment_name,
+                                              assignments.c.base_course == basecourse))
+        res = engine.execute(sel).first()
+        if res:
+            stmt = assignments.update().where(assignments.c.id == res['id']).values(
+                assignment_type = assignment_type,
+                deadline = deadline,
+                points = points
+            )
+            engine.execute(stmt)
+        else:
+            ins = assignments.insert().values(
+                base_course=basecourse,
+                name=name,
+                assignment_type = assignment_type,
+                deadline = deadline,
+                points = points)
+            engine.execute(ins)
+
+
 def addHTMLToDB(divid, basecourse, htmlsrc):
     if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
         dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)

From 6fe830b1f4651427d17657e42a11dfe14095ea35 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 08:16:35 -0400
Subject: [PATCH 20/46] intermediate save of componentdb.py to enable merge
 from autograde

---
 runestone/server/componentdb.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 47b8f179a..10bb6bec2 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -100,7 +100,7 @@ def getOrCreateAssignmentType(assignment_type_name):
             name=assignment_type_name)
         return engine.execute(ins).inserted_primary_key()
 
-def addAssignmentQuestionToDB(self, basecourse, assignment_id, question_name, points, timed):
+def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points, timed):
     if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
         dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
     else:
@@ -152,7 +152,7 @@ def addAssignmentToDB(self):
                 :name: Problem Set 1
                 :assignment_type: formative
                 :questions: (divid_1 50), (divid_2 100), ...
-                :deadline: <some datetime>
+                :deadline: 23-09-2016 15:30
                 :points: integer
         """
         assignment_type_name = self.options.get('assignment_type_name', 'Problem Set')
@@ -160,7 +160,7 @@ def addAssignmentToDB(self):
 
         name = self.options.get('name') # required; error if missing
         if 'name' in self.options:
-            deadline = datetime.strptime(self.options['name'], '%m/%d/%Y %H:%M')
+            deadline = datetime.strptime(self.options['name'], '%d-%m-%Y %H:%M')
         else:
             deadline = None
         points = self.options.get('points', None)

From d4d62bb64ea3d7c8cc39e7ce1caefa26077671e9 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 15:13:03 -0400
Subject: [PATCH 21/46] refactoring and additions to componentdb.py

---
 runestone/server/componentdb.py | 142 ++++++++++++++++----------------
 1 file changed, 71 insertions(+), 71 deletions(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index efb82f2d1..9a22865dc 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -22,6 +22,14 @@
 import os
 from sqlalchemy import create_engine, Table, MetaData, select, delete, update, and_
 
+# create a global DB query engine to share for the rest of the file
+if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+    dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+    engine = create_engine(dburl)
+else:
+    dburl = None
+    engine = None
+    print("Skipping all DB operations because environment variables not set up")
 
 def logSource(self):
     sourcelog = self.state.document.settings.env.config.html_context.get('dsource', None)
@@ -84,31 +92,28 @@ def addQuestionToDB(self):
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))
 
-def getOrCreateAssignmentType(assignment_type_name):
-    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
-        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
-    else:
-        dburl = None
-    engine = create_engine(dburl)
+def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_possible = None, assignments_count = None, assignments_dropped = None):
+
     meta = MetaData()
     assignment_types = Table('assignment_types', meta, autoload=True, autoload_with=engine)
 
+    # search for it in the DB
+    sel = select([assignment_types]).where(assignment_types.c.name == assignment_type_name)
     res = engine.execute(sel).first()
     if res:
         return res['id']
     else:
         # create the assignment type
         ins = assignment_types.insert().values(
-            name=assignment_type_name)
-        return engine.execute(ins).inserted_primary_key()
-
-def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points, timed):
-    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
-        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
-    else:
-        dburl = None
-
-    engine = create_engine(dburl)
+            name=assignment_type_name,
+            grade_type = grade_type,
+            points_possible = points_possible,
+            assignments_count = assignments_count,
+            assignments_dropped = assignments_dropped)
+        res = engine.execute(ins)
+        return res.inserted_primary_key[0]
+
+def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points, timed=None):
     meta = MetaData()
     questions = Table('questions', meta, autoload=True, autoload_with=engine)
     assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
@@ -124,67 +129,62 @@ def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points,
                                           assignment_questions.c.question_id == question_id))
     res = engine.execute(sel).first()
     if res:
-        #insert
-        pass
-    else:
         #update
-        pass
-
-def addAssignmentToDB(self):
-    if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
-        dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+        stmt = assignment_questions.update().where(assignment_questions.c.id == res['id']).values( \
+            assignment_id = assignment_id,
+            question_id = question_id,
+            points = points,
+            timed=timed
+            )
+        engine.execute(stmt)
     else:
-        dburl = None
+        #insert
+        ins = assignment_questions.insert().values(
+            assignment_id = assignment_id,
+            question_id = question_id,
+            points = points,
+            timed=timed
+            )
+        engine.execute(ins)
 
-    if dburl:
-        basecourse = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
-        if basecourse == "unknown":
-            raise self.severe("Cannot update database because basecourse is unknown")
-            return
+def getCourseID(coursename):
+    meta = MetaData()
+    courses = Table('courses', meta, autoload=True, autoload_with=engine)
 
-        last_changed = datetime.now()
+    sel = select([courses]).where(courses.c.course_name == coursename)
+    res = engine.execute(sel).first()
+    return res['id']
 
-        engine = create_engine(dburl)
-        meta = MetaData()
-        assignments = Table('assignments', meta, autoload=True, autoload_with=engine)
-        assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
-
-        """
-            .. assignment:
-                :name: Problem Set 1
-                :assignment_type: formative
-                :questions: (divid_1 50), (divid_2 100), ...
-                :deadline: 23-09-2016 15:30
-                :points: integer
-        """
-        assignment_type_name = self.options.get('assignment_type_name', 'Problem Set')
-        assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
-
-        name = self.options.get('name') # required; error if missing
-        if 'name' in self.options:
-            deadline = datetime.strptime(self.options['name'], '%d-%m-%Y %H:%M')
-        else:
-            deadline = None
-        points = self.options.get('points', None)
+def addAssignmentToDB(name = None, course_id = None, assignment_type_id = None, deadline = None, points = None, threshold = None):
 
-        sel = select([assignments]).where(and_(assignments.c.name == assignment_name,
-                                              assignments.c.base_course == basecourse))
-        res = engine.execute(sel).first()
-        if res:
-            stmt = assignments.update().where(assignments.c.id == res['id']).values(
-                assignment_type = assignment_type,
-                deadline = deadline,
-                points = points
-            )
-            engine.execute(stmt)
-        else:
-            ins = assignments.insert().values(
-                base_course=basecourse,
-                name=name,
-                assignment_type = assignment_type,
-                deadline = deadline,
-                points = points)
-            engine.execute(ins)
+    last_changed = datetime.now()
+
+    meta = MetaData()
+    assignments = Table('assignments', meta, autoload=True, autoload_with=engine)
+    assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
+
+    sel = select([assignments]).where(and_(assignments.c.name == name,
+                                          assignments.c.course == course_id))
+    res = engine.execute(sel).first()
+    if res:
+        stmt = assignments.update().where(assignments.c.id == res['id']).values(
+            assignment_type = assignment_type_id,
+            duedate = deadline,
+            points = points,
+            threshold = threshold
+        )
+        engine.execute(stmt)
+        return res['id']
+    else:
+        ins = assignments.insert().values(
+            course=course_id,
+            name=name,
+            assignment_type = assignment_type_id,
+            duedate = deadline,
+            points = points,
+            threshold = threshold)
+        res = engine.execute(ins)
+        return res.inserted_primary_key[0]
 
 
 def addHTMLToDB(divid, basecourse, htmlsrc):

From 8d6732c012c85eed6b25a0884f61ac924ee9d6f1 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 15:15:47 -0400
Subject: [PATCH 22/46] refactored usageAssignment to use componentDB

---
 runestone/__init__.py                 |   2 +-
 runestone/usageAssignment/__init__.py | 155 ++++++++------------------
 2 files changed, 47 insertions(+), 110 deletions(-)

diff --git a/runestone/__init__.py b/runestone/__init__.py
index eb0922d92..dbaa59fda 100644
--- a/runestone/__init__.py
+++ b/runestone/__init__.py
@@ -87,7 +87,7 @@ def build(options):
             idxfile = os.path.join(options.build.sourcedir,'index.rst')
 
         populateChapterInfo(options.build.project_name, idxfile)
-        print('Creating Chapter Information')
+        print('Creating Chapter Information for {}'.format(idxfile))
     except ImportError as e:
         print('Chapter information database population skipped, This is OK for a standalone build.',e)
     except Exception as e:
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index b4216f1f1..428036453 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -23,6 +23,9 @@
 from sqlalchemy import create_engine, Table, MetaData, select, delete
 from sqlalchemy.orm import sessionmaker
 from runestone.common.runestonedirective import RunestoneDirective
+from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID
+from datetime import datetime
+import os
 
 def setup(app):
     app.add_directive('usageassignment',usageAssignment)
@@ -100,7 +103,7 @@ class usageAssignment(Directive):
    :pct_required: <int>   :points: <int>
 
     """
-    required_arguments = 1  # requires an id for the directive
+    required_arguments = 0  # use assignment_name parameter
     optional_arguments = 0
     has_content = False
     option_spec = RunestoneDirective.option_spec.copy()
@@ -115,32 +118,6 @@ class usageAssignment(Directive):
         'points':directives.positive_int
     })
 
-    def get_or_make_assignment_type(self, engine, session, AssignmentType):
-        # There will normally be only one "usage" type of assignment; we'll take that, or create it if it doesn't exist
-        a = session.query(AssignmentType).filter(AssignmentType.c.grade_type == 'use').first()
-        if not a:
-            print("creating Lecture Prep assignment type")
-            engine.execute(AssignmentType.insert().values(
-                name = 'Lecture Prep',
-                grade_type = 'use',
-                points_possible = '50',
-                assignments_count = 23,
-                assignments_dropped = 3))
-            a = session.query(AssignmentType).filter(AssignmentType.c.grade_type == 'use').first()
-        return a.id
-
-    def get_or_make_course_section(self, course_id, engine, session, Section):
-        # Currently only works with a single default section for the whole course
-        # Could add sections to the directive to allow different sections to get different deadlines
-        a = session.query(Section).filter(Section.c.course_id == course_id).first()
-        if not a:
-            print("creating default course section")
-            engine.execute(Section.insert().values(
-                course_id = course_id,
-                name = 'Default Section'))
-            a = session.query(Section).filter(Section.c.course_id == course_id).first()
-        return a.id
-
     def run(self):
         """
           .. usageassignment:: prep_1
@@ -153,39 +130,36 @@ def run(self):
             :pct_required: <int>
             :points: <int>
         """
-        self.options['divid'] = self.arguments[0]
-        try:
-            env = self.state.document.settings.env
-            # print("DBURL = ",env.config['dburl'])
-            engine = create_engine(env.config.html_context['dburl'])
-            meta = MetaData()
-            Assignment = Table('assignments', meta, autoload=True, autoload_with=engine)
-            Chapter = Table('chapters', meta, autoload=True, autoload_with=engine)
-            SubChapter = Table('sub_chapters', meta, autoload=True, autoload_with=engine)
-            Problem = Table('problems', meta, autoload=True, autoload_with=engine)
-            Div = Table('div_ids', meta, autoload=True, autoload_with=engine)
-            Course = Table('courses', meta, autoload=True, autoload_with=engine)
-            PIPDeadline = Table('pipactex_deadline', meta, autoload=True, autoload_with=engine)
-            Deadline = Table('deadlines', meta, autoload=True, autoload_with=engine)
-            AssignmentType = Table('assignment_types', meta, autoload=True, autoload_with=engine)
-            Section = Table('sections', meta, autoload=True, autoload_with=engine)
-            # create a configured "Session" class
-            Session = sessionmaker(bind=engine)
-        except:
-            print("Unable to create and save usage assignment. Possible problems:")
-            print("  1. dburl or course_id are not set in conf.py for your book")
-            print("  2. unable to connect to the database using dburl")
-            print()
-            print("This should only affect the grading interface. Everything else should be fine.")
-            return [usageAssignmentNode(self.options)]
-
 
-        # create a Session
-        session = Session()
-
-        course_name = env.config.html_context['course_id']
+        if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+            dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+        else:
+            dburl = None
+            raise self.warn("Environment variables not set for DB access; can't save usageassignment to DB")
+            return [usageAssignmentNode(self.options)]
+        engine = create_engine(dburl)
+        meta = MetaData()
+        # create a configured "Session" class
+        Session = sessionmaker(bind=engine)
+
+        Chapter = Table('chapters', meta, autoload=True, autoload_with=engine)
+        SubChapter = Table('sub_chapters', meta, autoload=True, autoload_with=engine)
+        Problem = Table('problems', meta, autoload=True, autoload_with=engine)
+        Div = Table('div_ids', meta, autoload=True, autoload_with=engine)
+        AssignmentType = Table('assignment_types', meta, autoload=True, autoload_with=engine)
+        Section = Table('sections', meta, autoload=True, autoload_with=engine)
+
+        assignment_type_id = getOrCreateAssignmentType("Lecture Prep",
+                                  grade_type = 'use',
+                                  points_possible = '50',
+                                  assignments_count = 23,
+                                  assignments_dropped = 3)
+
+
+        course_name = self.state.document.settings.env.config.html_context['course_id']
         self.options['course_name'] = course_name
-        course_id = str(session.query(Course).filter(Course.c.course_name == course_name).first().id)
+        course_id = getCourseID(course_name)
+        basecourse_name = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
 
         # Accumulate all the Chapters and SubChapters that are to be visited
         # For each chapter, accumulate all subchapters
@@ -236,59 +210,22 @@ def run(self):
 
 
         min_activities = (len(paths) + len(active_codes)) * self.options.get('pct_required', 0) / 100
-        assignment_type = self.get_or_make_assignment_type(engine, session, AssignmentType)
-
-        # Add or update Assignment
-        a = session.query(Assignment).filter(Assignment.c.name == self.options.get('assignment_name', 'dummy_assignment'), Assignment.c.course == course_id).first()
-        if a:
-            engine.execute(Assignment.update()\
-                           .where(Assignment.c.name == self.options.get('assignment_name', 'dummy_assignment'))\
-                           .where(Assignment.c.course == course_id)\
-                           .values(
-                                name = self.options.get('assignment_name', 'dummy_assignment'),
-                                assignment_type = assignment_type,
-                                points = self.options.get('points', 0),
-                                threshold = min_activities,
-            ))
 
-        else:
-            engine.execute(Assignment.insert().values(
-                course = course_id,
-                assignment_type = assignment_type,
-                name = self.options.get('assignment_name', 'dummy_assignment'),
-                points = self.options.get('points', 0),
-                threshold = min_activities
-            ))
-            a = session.query(Assignment).filter(Assignment.c.name == self.options.get('assignment_name', 'dummy_assignment'), Assignment.c.course == course_id).first()
-
-        # Replace any existing deadlines
-        section_id = self.get_or_make_course_section(course_id, engine, session, Section)
-        engine.execute(Deadline.delete()\
-                      .where(Deadline.c.section == section_id)\
-                      .where(Deadline.c.assignment == a.id))
         if 'deadline' in self.options:
-            engine.execute(Deadline.insert()\
-                          .values(section = section_id,
-                                  assignment = a.id,
-                                  deadline = self.options.get('deadline')))
-        #I think pipactex_deadlines are deprected; let's see if anything breaks by just deleting them
-        for acid in active_codes:
-            engine.execute(PIPDeadline.delete()\
-                          .where(PIPDeadline.c.section == section_id)\
-                          .where(PIPDeadline.c.acid_prefix == acid))
-            # if 'deadline' in self.options:
-            #     engine.execute(Deadline.insert()\
-            #                   .values(section = section_id,
-            #                           acid_prefix = acid,
-            #                           deadline = self.options.get('deadline')))
-            #
-        # replace any existing problems for this assignment with the new ones
-        engine.execute(Problem.delete()\
-                       .where(Problem.c.assignment == a.id))
-        for acid in paths + active_codes:
-            engine.execute(Problem.insert()\
-                .values(acid = acid, assignment = a.id))
+            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+        else:
+            deadline = None
+        points = self.options.get('points', 0)
 
-        session.commit()
+        assignment_id = addAssignmentToDB(name = self.options.get('assignment_name', 'dummy_assignment'),
+                          course_id = course_id,
+                          assignment_type_id = assignment_type_id,
+                          deadline = deadline,
+                          points = points,
+                          threshold = min_activities)
+
+        for acid in paths + active_codes:
+            print( "adding {} to assignment {}".format(acid, assignment_id))
+            addAssignmentQuestionToDB(base_course_name, assignment_id, acid, 1)
 
         return [usageAssignmentNode(self.options)]

From 37b0212acdd4d137be529e788453c89d3448e9ad Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 15:18:04 -0400
Subject: [PATCH 23/46] Revert "eradicate hard coded forward slashes" because
 on Windows os.path.sep is \ but we forward slashes in the .rst files

This reverts commit aafe26b9f38da8f7ed37f754769594a2fa77410d.

Conflicts:
	runestone/common/runestonedirective.py
---
 runestone/common/runestonedirective.py | 3 +--
 runestone/server/chapternames.py       | 4 ++--
 runestone/server/componentdb.py        | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/runestone/common/runestonedirective.py b/runestone/common/runestonedirective.py
index 6317e0fac..9c85fd8f8 100644
--- a/runestone/common/runestonedirective.py
+++ b/runestone/common/runestonedirective.py
@@ -40,10 +40,9 @@ def __init__(self, *args, **kwargs):
         super(RunestoneDirective,self).__init__(*args, **kwargs)
         srcpath, self.line = self.state_machine.get_source_and_line()
         self.subchapter = os.path.basename(srcpath).replace('.rst','')
-        self.chapter = srcpath.split(os.path.sep)[-2]
+        self.chapter = srcpath.split('/')[-2]
         self.srcpath = srcpath
         self.basecourse = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
         self.options['basecourse'] = self.basecourse
         self.options['chapter'] = self.chapter
         self.options['subchapter'] = self.subchapter
-
diff --git a/runestone/server/chapternames.py b/runestone/server/chapternames.py
index 426ce0549..fa9eb8244 100644
--- a/runestone/server/chapternames.py
+++ b/runestone/server/chapternames.py
@@ -38,8 +38,8 @@ def findChaptersSubChapters(tocfile):
         else:
             stop = toclines[i + 1]
         for j in range(start, stop):
-            if ".rst" in ftext[j] and os.path.sep in ftext[j]:
-                chapter, subchapter = ftext[j].strip()[:-4].split(os.path.sep)
+            if ".rst" in ftext[j] and "/" in ftext[j]:
+                chapter, subchapter = ftext[j].strip()[:-4].split('/')
                 chapter = chapter.strip()
                 subchapter = subchapter.strip()
                 if chapter not in chdict:
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 9a22865dc..82a134a8c 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -73,7 +73,7 @@ def addQuestionToDB(self):
 
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
-        chapter = srcpath.split(os.path.sep)[-2]
+        chapter = srcpath.split('/')[-2]
 
         autograde = self.options.get('autograde', None)
 

From 0cf0b891928dd34770b130030a55267cc156f0ab Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 16:33:41 -0400
Subject: [PATCH 24/46] os.path where appropriate, and hardcoded / where
 appropriate

---
 runestone/common/runestonedirective.py | 2 +-
 runestone/server/chapternames.py       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runestone/common/runestonedirective.py b/runestone/common/runestonedirective.py
index 9c85fd8f8..87d260365 100644
--- a/runestone/common/runestonedirective.py
+++ b/runestone/common/runestonedirective.py
@@ -40,7 +40,7 @@ def __init__(self, *args, **kwargs):
         super(RunestoneDirective,self).__init__(*args, **kwargs)
         srcpath, self.line = self.state_machine.get_source_and_line()
         self.subchapter = os.path.basename(srcpath).replace('.rst','')
-        self.chapter = srcpath.split('/')[-2]
+        self.chapter = srcpath.split(os.path.sep)[-2]
         self.srcpath = srcpath
         self.basecourse = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
         self.options['basecourse'] = self.basecourse
diff --git a/runestone/server/chapternames.py b/runestone/server/chapternames.py
index fa9eb8244..c4445e5be 100644
--- a/runestone/server/chapternames.py
+++ b/runestone/server/chapternames.py
@@ -62,10 +62,10 @@ def findV2ChaptersSubChapters(tocfile):
     toclines = getTOCEntries(ftext)
     chdict = OrderedDict()
     chtitles = {}
-    toclines = [x for x in toclines if os.path.sep in x]
+    toclines = [x for x in toclines if '/' in x]
     basepath = os.path.dirname(tocfile)
     for subchapter in toclines:
-        chapter = subchapter.split(os.path.sep)[0]
+        chapter = subchapter.split('/')[0]
         with open(os.path.join(basepath,subchapter),'r') as scfile:
             ft = scfile.readline().strip()
             chtitles[chapter] = ft

From ade77841ad7badbef1543aa68010f2fcd6005330 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 16:40:34 -0400
Subject: [PATCH 25/46] usageAssignment creates questions for pages

---
 runestone/server/componentdb.py       | 62 +++++++++++++++++++++++----
 runestone/usageAssignment/__init__.py | 12 +++---
 2 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 82a134a8c..04d6e484b 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -73,7 +73,7 @@ def addQuestionToDB(self):
 
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
-        chapter = srcpath.split('/')[-2]
+        chapter = srcpath.split(os.path.sep)[-2]
 
         autograde = self.options.get('autograde', None)
 
@@ -92,6 +92,56 @@ def addQuestionToDB(self):
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))
 
+def getQuestionID(base_course, name):
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+
+
+    sel = select([questions]).where(and_(questions.c.name == name,
+                                          questions.c.base_course == base_course))
+    res = engine.execute(sel).first()
+    if res:
+        return res['id']
+    else:
+        return None
+
+def getOrInsertQuestionForPage(base_course=None, name=None, is_private='F', question_type="page", autograde = "visited", author=None, difficulty=1,chapter=None):
+    last_changed = datetime.now()
+
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+
+
+    sel = select([questions]).where(and_(questions.c.name == name,
+                                          questions.c.base_course == base_course))
+    res = engine.execute(sel).first()
+
+    if res:
+        id = res['id']
+        stmt = questions.update().where(questions.c.id == id).values(
+            timestamp=last_changed,
+            is_private= is_private,
+            question_type=question_type,
+            autograde = autograde,
+            author=author,
+            difficulty=difficulty,
+            chapter=chapter)
+        res = engine.execute(stmt)
+        return id
+    else:
+        ins = questions.insert().values(
+            base_course= base_course,
+            name= name,
+            timestamp=last_changed,
+            is_private= is_private,
+            question_type=question_type,
+            autograde = autograde,
+            author=author,
+            difficulty=difficulty,
+            chapter=chapter)
+        res = engine.execute(ins)
+        return res.inserted_primary_key[0]
+
 def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_possible = None, assignments_count = None, assignments_dropped = None):
 
     meta = MetaData()
@@ -113,17 +163,11 @@ def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_po
         res = engine.execute(ins)
         return res.inserted_primary_key[0]
 
-def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points, timed=None):
+def addAssignmentQuestionToDB(question_id, assignment_id, points, timed=None):
     meta = MetaData()
     questions = Table('questions', meta, autoload=True, autoload_with=engine)
     assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
 
-    # first get the question_id associated with question_name
-    sel = select([questions]).where(and_(questions.c.name == question_name,
-                                          questions.c.base_course == basecourse))
-    res = engine.execute(sel).first()
-    question_id = res['id']
-
     # now insert or update the assignment_questions row
     sel = select([assignment_questions]).where(and_(assignment_questions.c.assignment_id == assignment_id,
                                           assignment_questions.c.question_id == question_id))
@@ -134,7 +178,7 @@ def addAssignmentQuestionToDB(basecourse, assignment_id, question_name, points,
             assignment_id = assignment_id,
             question_id = question_id,
             points = points,
-            timed=timed
+            timed= timed
             )
         engine.execute(stmt)
     else:
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 428036453..a1acc7739 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -23,7 +23,7 @@
 from sqlalchemy import create_engine, Table, MetaData, select, delete
 from sqlalchemy.orm import sessionmaker
 from runestone.common.runestonedirective import RunestoneDirective
-from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID
+from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID, addAssignmentQuestionToDB, getOrInsertQuestionForPage
 from datetime import datetime
 import os
 
@@ -141,6 +141,7 @@ def run(self):
         meta = MetaData()
         # create a configured "Session" class
         Session = sessionmaker(bind=engine)
+        session = Session()
 
         Chapter = Table('chapters', meta, autoload=True, autoload_with=engine)
         SubChapter = Table('sub_chapters', meta, autoload=True, autoload_with=engine)
@@ -176,6 +177,7 @@ def run(self):
                     sub_chs += results
                     chapter_data = {'ch': nm, 'sub_chs': [r.sub_chapter_label for r in results]}
                     self.options['chapter_data'].append(chapter_data)
+
             except:
                 print("Chapters requested not found: %s" % (self.options.get('chapters')))
         # Add any explicit subchapters
@@ -189,8 +191,8 @@ def run(self):
                     if not subch:
                         print("problem with: %s" % nm)
                     self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
-            except:
-                print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
+        except:
+            print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
 
         # Accumulate all the ActiveCodes that are to be run and URL paths to be visited
         divs = []
@@ -225,7 +227,7 @@ def run(self):
                           threshold = min_activities)
 
         for acid in paths + active_codes:
-            print( "adding {} to assignment {}".format(acid, assignment_id))
-            addAssignmentQuestionToDB(base_course_name, assignment_id, acid, 1)
+            q_id = getOrInsertQuestionForPage(base_course=basecourse_name, name=acid, is_private='F', question_type="page", autograde = "visited", difficulty=1,chapter=None)
+            addAssignmentQuestionToDB(q_id, assignment_id, 1)
 
         return [usageAssignmentNode(self.options)]

From ccf608519f3df3aa880c49092cc36539d148cf56 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 31 Aug 2016 18:01:24 -0400
Subject: [PATCH 26/46] fix indentation

---
 runestone/usageAssignment/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index a1acc7739..8d10d69b6 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -191,8 +191,8 @@ def run(self):
                     if not subch:
                         print("problem with: %s" % nm)
                     self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
-        except:
-            print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
+            except:
+                print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
 
         # Accumulate all the ActiveCodes that are to be run and URL paths to be visited
         divs = []

From 45c0bb7f0002e6a8f98b14a909af5716c92f97a7 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Thu, 1 Sep 2016 11:06:27 -0400
Subject: [PATCH 27/46] add the assignment directive, for creating assignments
 in DB based on .rst file

---
 runestone/assignment/__init__.py | 108 +++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 runestone/assignment/__init__.py

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
new file mode 100644
index 000000000..2a2b04223
--- /dev/null
+++ b/runestone/assignment/__init__.py
@@ -0,0 +1,108 @@
+# Copyright (C) 2015  Paul Resnick
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import print_function
+
+__author__ = 'Paul Resnick'
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+from docutils.parsers.rst import Directive
+from runestone.server.componentdb import addAssignmentToDB, addAssignmentQuestionToDB, getCourseID, getOrCreateAssignmentType, getQuestionID
+from runestone.common.runestonedirective import RunestoneDirective
+from datetime import datetime
+
+def setup(app):
+    app.add_directive('assignment',Assignment)
+
+    app.connect('doctree-resolved',process_nodes)
+    app.connect('env-purge-doc', purge)
+
+def process_nodes(app,env,docname):
+    pass
+
+
+def purge(app,env,docname):
+    pass
+
+
+class Assignment(RunestoneDirective):
+    """
+        .. assignment:
+            :name: Problem Set 1
+            :assignment_type: formative
+            :questions: (divid_1 50), (divid_2 100), ...
+            :deadline: 23-09-2016 15:30
+            :points: integer
+    """
+    required_arguments = 0 # not a sphinx_id for these; just a name parameter
+    optional_arguments = 0
+    has_content = False
+    option_spec = RunestoneDirective.option_spec.copy()
+    option_spec.update({
+        'name': directives.unchanged,
+        'assignment_type': directives.unchanged,
+        'questions': directives.unchanged,
+        'deadline':directives.unchanged,
+        'points':directives.positive_int
+    })
+
+    def run(self):
+        """
+            .. assignment:
+                :name: Problem Set 1
+                :assignment_type: formative
+                :questions: (divid_1 50), (divid_2 100), ...
+                :deadline: 23-09-2016 15:30
+                :points: integer
+        """
+
+        course_name = self.state.document.settings.env.config.html_context['course_id']
+        self.options['course_name'] = course_name
+        course_id = getCourseID(course_name)
+        basecourse_name = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
+
+        name = self.options.get('name') # required; error if missing
+        assignment_type_name = self.options.get('assignment_type')
+        assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
+
+        if 'deadline' in self.options:
+            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+        else:
+            deadline = None
+        points = self.options.get('points', 0)
+
+        assignment_id = addAssignmentToDB(name = name,
+                          course_id = course_id,
+                          assignment_type_id = assignment_type_id,
+                          deadline = deadline,
+                          points = points)
+
+        unparsed = self.options.get('questions', None)
+        if unparsed:
+            q_strings = unparsed.split(',')
+            for q in q_strings:
+                (question_name, points) = q.strip().split()
+
+                # first get the question_id associated with question_name
+                question_id = getQuestionID(basecourse_name, question_name)
+                if question_id:
+                    addAssignmentQuestionToDB(question_id, assignment_id, points)
+                else:
+                    raise self.warn("Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
+        else:
+            raise self.warn("No questions for assignment {}".format(name))
+
+        return []

From f8b3c37af61c51bcf7c31e96cef827994ccfcdda Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Thu, 1 Sep 2016 13:10:43 -0400
Subject: [PATCH 28/46] save assessment_type as summative for assignment
 directive

---
 runestone/assignment/__init__.py | 3 ++-
 runestone/server/componentdb.py  | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index 2a2b04223..b255a052a 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -92,6 +92,7 @@ def run(self):
 
         unparsed = self.options.get('questions', None)
         if unparsed:
+            summative_type_id = getOrCreateAssignmentType("summative")
             q_strings = unparsed.split(',')
             for q in q_strings:
                 (question_name, points) = q.strip().split()
@@ -99,7 +100,7 @@ def run(self):
                 # first get the question_id associated with question_name
                 question_id = getQuestionID(basecourse_name, question_name)
                 if question_id:
-                    addAssignmentQuestionToDB(question_id, assignment_id, points)
+                    addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = summative_type_id)
                 else:
                     raise self.warn("Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
         else:
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 04d6e484b..a77c4423f 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -163,7 +163,7 @@ def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_po
         res = engine.execute(ins)
         return res.inserted_primary_key[0]
 
-def addAssignmentQuestionToDB(question_id, assignment_id, points, timed=None):
+def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = None, timed=None):
     meta = MetaData()
     questions = Table('questions', meta, autoload=True, autoload_with=engine)
     assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
@@ -178,7 +178,8 @@ def addAssignmentQuestionToDB(question_id, assignment_id, points, timed=None):
             assignment_id = assignment_id,
             question_id = question_id,
             points = points,
-            timed= timed
+            timed= timed,
+            assessment_type = assessment_type
             )
         engine.execute(stmt)
     else:
@@ -187,7 +188,8 @@ def addAssignmentQuestionToDB(question_id, assignment_id, points, timed=None):
             assignment_id = assignment_id,
             question_id = question_id,
             points = points,
-            timed=timed
+            timed=timed,
+            assessment_type = assessment_type
             )
         engine.execute(ins)
 

From d264ee8f06150d0812a8407471ccae735b416194 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 3 Sep 2016 11:01:20 -0400
Subject: [PATCH 29/46] allow seconds or not in deadlines; fix bug in emitting
 warnings

---
 runestone/assignment/__init__.py      | 17 ++++++++++++-----
 runestone/usageAssignment/__init__.py | 15 +++++++++++----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index b255a052a..ca876ecb0 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -78,10 +78,17 @@ def run(self):
         assignment_type_name = self.options.get('assignment_type')
         assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
 
+        deadline = None
         if 'deadline' in self.options:
-            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
-        else:
-            deadline = None
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+            except:
+                print("deadline not in preferred format %Y-%m-%d %H:%M")
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+            except:
+                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+
         points = self.options.get('points', 0)
 
         assignment_id = addAssignmentToDB(name = name,
@@ -102,8 +109,8 @@ def run(self):
                 if question_id:
                     addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = summative_type_id)
                 else:
-                    raise self.warn("Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
         else:
-            raise self.warn("No questions for assignment {}".format(name))
+            self.state.document.settings.env.warn(self.state.document.settings.env.docname, "No questions for assignment {}".format(name))
 
         return []
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 8d10d69b6..954625dc9 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -135,7 +135,7 @@ def run(self):
             dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
         else:
             dburl = None
-            raise self.warn("Environment variables not set for DB access; can't save usageassignment to DB")
+            self.state.document.settings.env.warn("Environment variables not set for DB access; can't save usageassignment to DB")
             return [usageAssignmentNode(self.options)]
         engine = create_engine(dburl)
         meta = MetaData()
@@ -213,10 +213,17 @@ def run(self):
 
         min_activities = (len(paths) + len(active_codes)) * self.options.get('pct_required', 0) / 100
 
+        deadline = None
         if 'deadline' in self.options:
-            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
-        else:
-            deadline = None
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+            except:
+                print("deadline not in preferred format %Y-%m-%d %H:%M")
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+            except:
+                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+
         points = self.options.get('points', 0)
 
         assignment_id = addAssignmentToDB(name = self.options.get('assignment_name', 'dummy_assignment'),

From 337ba70cd391ffb3bbd95ef8e8c65c8491d5ac67 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 3 Sep 2016 11:01:20 -0400
Subject: [PATCH 30/46] allow seconds or not in deadlines; fix bug in emitting
 warnings

---
 runestone/assignment/__init__.py      | 17 ++++++++++++-----
 runestone/usageAssignment/__init__.py | 15 +++++++++++----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index b255a052a..ca876ecb0 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -78,10 +78,17 @@ def run(self):
         assignment_type_name = self.options.get('assignment_type')
         assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
 
+        deadline = None
         if 'deadline' in self.options:
-            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
-        else:
-            deadline = None
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+            except:
+                print("deadline not in preferred format %Y-%m-%d %H:%M")
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+            except:
+                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+
         points = self.options.get('points', 0)
 
         assignment_id = addAssignmentToDB(name = name,
@@ -102,8 +109,8 @@ def run(self):
                 if question_id:
                     addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = summative_type_id)
                 else:
-                    raise self.warn("Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
         else:
-            raise self.warn("No questions for assignment {}".format(name))
+            self.state.document.settings.env.warn(self.state.document.settings.env.docname, "No questions for assignment {}".format(name))
 
         return []
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 8d10d69b6..954625dc9 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -135,7 +135,7 @@ def run(self):
             dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
         else:
             dburl = None
-            raise self.warn("Environment variables not set for DB access; can't save usageassignment to DB")
+            self.state.document.settings.env.warn("Environment variables not set for DB access; can't save usageassignment to DB")
             return [usageAssignmentNode(self.options)]
         engine = create_engine(dburl)
         meta = MetaData()
@@ -213,10 +213,17 @@ def run(self):
 
         min_activities = (len(paths) + len(active_codes)) * self.options.get('pct_required', 0) / 100
 
+        deadline = None
         if 'deadline' in self.options:
-            deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
-        else:
-            deadline = None
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
+            except:
+                print("deadline not in preferred format %Y-%m-%d %H:%M")
+            try:
+                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+            except:
+                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+
         points = self.options.get('points', 0)
 
         assignment_id = addAssignmentToDB(name = self.options.get('assignment_name', 'dummy_assignment'),

From 3852af1ff803c1b1a823ccffb0eb8a6b88ec7d53 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sat, 3 Sep 2016 11:12:05 -0400
Subject: [PATCH 31/46] switch from print to sphinx warnings

---
 runestone/assignment/__init__.py      |  4 ++--
 runestone/usageAssignment/__init__.py | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index ca876ecb0..18180d975 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -83,11 +83,11 @@ def run(self):
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
             except:
-                print("deadline not in preferred format %Y-%m-%d %H:%M")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline not in preferred format %Y-%m-%d %H:%M")
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
             except:
-                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
 
         points = self.options.get('points', 0)
 
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 954625dc9..f08472362 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -179,7 +179,7 @@ def run(self):
                     self.options['chapter_data'].append(chapter_data)
 
             except:
-                print("Chapters requested not found: %s" % (self.options.get('chapters')))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Chapters requested not found: %s" % (self.options.get('chapters')))
         # Add any explicit subchapters
         if 'subchapters' in self.options:
             try:
@@ -189,10 +189,10 @@ def run(self):
                     subch = session.query(SubChapter).filter(SubChapter.c.chapter_id == ch_id, SubChapter.c.sub_chapter_label == subch_name).first()
                     sub_chs.append(subch)
                     if not subch:
-                        print("problem with: %s" % nm)
+                        self.state.document.settings.env.warn(self.state.document.settings.env.docname("problem with: %s" % nm)
                     self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
             except:
-                print("Subchapters requested not found: %s" % (self.options.get('subchapters')))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Subchapters requested not found: %s" % (self.options.get('subchapters')))
 
         # Accumulate all the ActiveCodes that are to be run and URL paths to be visited
         divs = []
@@ -205,7 +205,7 @@ def run(self):
                                                   Div.c.subchapter == subch.sub_chapter_label).all()
                 paths.append('/runestone/static/%s/%s/%s.html' % (course_name, ch_name, subch.sub_chapter_label))
             except:
-                print ("Subchapter not found: %s" % (subch))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Subchapter not found: %s" % (subch))
         tracked_div_types = ['activecode', 'actex']
         active_codes = [d.div_id for d in divs if d.div_type in tracked_div_types]
 
@@ -218,11 +218,11 @@ def run(self):
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
             except:
-                print("deadline not in preferred format %Y-%m-%d %H:%M")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline not in preferred format %Y-%m-%d %H:%M")
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
             except:
-                print("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
 
         points = self.options.get('points', 0)
 

From b9cb81e57e06342cb21371fa91e6bedd332199d0 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Mon, 5 Sep 2016 08:34:31 -0400
Subject: [PATCH 32/46] fix calls to warn

---
 runestone/usageAssignment/__init__.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index f08472362..609514ab9 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -135,7 +135,7 @@ def run(self):
             dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
         else:
             dburl = None
-            self.state.document.settings.env.warn("Environment variables not set for DB access; can't save usageassignment to DB")
+            self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Environment variables not set for DB access; can't save usageassignment to DB")
             return [usageAssignmentNode(self.options)]
         engine = create_engine(dburl)
         meta = MetaData()
@@ -179,7 +179,7 @@ def run(self):
                     self.options['chapter_data'].append(chapter_data)
 
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Chapters requested not found: %s" % (self.options.get('chapters')))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Chapters requested not found: %s" % (self.options.get('chapters')))
         # Add any explicit subchapters
         if 'subchapters' in self.options:
             try:
@@ -189,10 +189,10 @@ def run(self):
                     subch = session.query(SubChapter).filter(SubChapter.c.chapter_id == ch_id, SubChapter.c.sub_chapter_label == subch_name).first()
                     sub_chs.append(subch)
                     if not subch:
-                        self.state.document.settings.env.warn(self.state.document.settings.env.docname("problem with: %s" % nm)
+                        self.state.document.settings.env.warn(self.state.document.settings.env.docname, "problem with: %s" % nm)
                     self.options['chapter_data'].append({'ch': ch_dir, 'sub_chs': [subch_name]})
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Subchapters requested not found: %s" % (self.options.get('subchapters')))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Subchapters requested not found: %s" % (self.options.get('subchapters')))
 
         # Accumulate all the ActiveCodes that are to be run and URL paths to be visited
         divs = []
@@ -205,7 +205,7 @@ def run(self):
                                                   Div.c.subchapter == subch.sub_chapter_label).all()
                 paths.append('/runestone/static/%s/%s/%s.html' % (course_name, ch_name, subch.sub_chapter_label))
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("Subchapter not found: %s" % (subch))
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Subchapter not found: %s" % (subch))
         tracked_div_types = ['activecode', 'actex']
         active_codes = [d.div_id for d in divs if d.div_type in tracked_div_types]
 
@@ -218,11 +218,11 @@ def run(self):
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline not in preferred format %Y-%m-%d %H:%M")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline not in preferred format %Y-%m-%d %H:%M")
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
 
         points = self.options.get('points', 0)
 

From 26657226375329b18b96c0b55ceaae843bb7cdc5 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Mon, 5 Sep 2016 11:20:00 -0400
Subject: [PATCH 33/46] fix deadline warnings when not formatted correctly

---
 runestone/assignment/__init__.py      | 11 ++++++-----
 runestone/usageAssignment/__init__.py | 10 +++++-----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index 18180d975..979db8957 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -79,15 +79,16 @@ def run(self):
         assignment_type_id = getOrCreateAssignmentType(assignment_type_name)
 
         deadline = None
+
         if 'deadline' in self.options:
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline not in preferred format %Y-%m-%d %H:%M")
-            try:
-                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
-            except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname("deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+                try:
+                    deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline not in preferred format %Y-%m-%d %H:%M but accepting alternate format with seconds")
+                except:
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline missing or incorrectly formatted; Omitting deadline")
 
         points = self.options.get('points', 0)
 
diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 609514ab9..eae288d85 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -218,11 +218,11 @@ def run(self):
             try:
                 deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M')
             except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline not in preferred format %Y-%m-%d %H:%M")
-            try:
-                deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
-            except:
-                self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline also not in alternate format %Y-%m-%d %H:%M:%S\n Omitting deadline")
+                try:
+                    deadline = datetime.strptime(self.options['deadline'], '%Y-%m-%d %H:%M:%S')
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline not in preferred format %Y-%m-%d %H:%M but accepting alternate format with seconds")
+                except:
+                    self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline missing or incorrectly formatted; Omitting deadline")
 
         points = self.options.get('points', 0)
 

From a2d95af67b91aaa2c9f4dd9287924eabe2b861fd Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Mon, 5 Sep 2016 11:53:38 -0400
Subject: [PATCH 34/46] remove customization in bookfuncs.js

---
 runestone/common/js/bookfuncs.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runestone/common/js/bookfuncs.js b/runestone/common/js/bookfuncs.js
index 7a3ef3792..5445b0ba2 100644
--- a/runestone/common/js/bookfuncs.js
+++ b/runestone/common/js/bookfuncs.js
@@ -120,8 +120,7 @@ function gotUser(data, status, whatever) {
         }
     } else {
         if (!caughtErr) {
-            // mess = "username: " + d.nick;
-            mess = "user: " + d.email;
+            mess = "username: " + d.nick;
             eBookConfig.email = d.email;
             eBookConfig.isLoggedIn = true;
             eBookConfig.cohortId = d.cohortId;

From cfd993fc6ca826afcde737dc525e36ce83ce55e4 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Tue, 6 Sep 2016 19:19:10 -0400
Subject: [PATCH 35/46] show email instead of username

---
 runestone/common/js/bookfuncs.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runestone/common/js/bookfuncs.js b/runestone/common/js/bookfuncs.js
index 5445b0ba2..7a3ef3792 100644
--- a/runestone/common/js/bookfuncs.js
+++ b/runestone/common/js/bookfuncs.js
@@ -120,7 +120,8 @@ function gotUser(data, status, whatever) {
         }
     } else {
         if (!caughtErr) {
-            mess = "username: " + d.nick;
+            // mess = "username: " + d.nick;
+            mess = "user: " + d.email;
             eBookConfig.email = d.email;
             eBookConfig.isLoggedIn = true;
             eBookConfig.cohortId = d.cohortId;

From 11a5a1742807961dc12fb38f3d0e75404501b1d4 Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Sat, 17 Sep 2016 14:14:36 -0400
Subject: [PATCH 36/46] Reformat subchapter appearance and indent for more
 neatness

---
 runestone/usageAssignment/__init__.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 954625dc9..5b5739039 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 from __future__ import print_function
+import logging
 
 __author__ = 'Paul Resnick'
 
@@ -57,10 +58,18 @@ def visit_ua_node(self,node):
         chapter_data = None
 
     s = ""
+    chapters_and_subchapters = {}
     if chapter_data and course_name:
-        for d in chapter_data:
+        for d in chapter_data: # Set up Chapter-Subchs dictionary
             ch_name, sub_chs = d['ch'], d['sub_chs']
-            s += '<div class="panel-heading">'
+            if d['ch'] not in chapters_and_subchapters:
+                chapters_and_subchapters[d['ch']] = [x for x in d['sub_chs']]
+            else:
+                for subch in d['sub_chs']:
+                    chapters_and_subchapters[d['ch']].append(subch)
+
+        for ch_name,sub_chs in chapters_and_subchapters.items():
+            s += '<div style="margin-left:150px;" class="panel-heading">'
             s += ch_name
             s += '<ul class="list-group">'
             for sub_ch_name in sub_chs:
@@ -70,7 +79,7 @@ def visit_ua_node(self,node):
             s += '</ul>'
             s += '</div>'
 
-    # is this needed??
+    # is this needed?? 
     s = s.replace("u'","'")  # hack:  there must be a better way to include the list and avoid unicode strings
 
     self.body.append(s)

From 220cd73ba411de5cedbacdfee628f6121d6646af Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Sat, 17 Sep 2016 14:18:21 -0400
Subject: [PATCH 37/46] Remove unnecessary import post-debug

---
 runestone/usageAssignment/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index 5b5739039..b46b18997 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 from __future__ import print_function
-import logging
 
 __author__ = 'Paul Resnick'
 

From e99e44abc6f8eba326ff03f9a8dfbea4bf1aa8a0 Mon Sep 17 00:00:00 2001
From: jczetta <jczetta@gmail.com>
Date: Sat, 17 Sep 2016 19:28:12 -0400
Subject: [PATCH 38/46] Ordered dictionary.

---
 runestone/usageAssignment/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index b46b18997..4ef6ce48c 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -25,6 +25,7 @@
 from runestone.common.runestonedirective import RunestoneDirective
 from runestone.server.componentdb import addAssignmentToDB, getOrCreateAssignmentType, getCourseID, addAssignmentQuestionToDB, getOrInsertQuestionForPage
 from datetime import datetime
+from collections import OrderedDict
 import os
 
 def setup(app):
@@ -57,13 +58,14 @@ def visit_ua_node(self,node):
         chapter_data = None
 
     s = ""
-    chapters_and_subchapters = {}
+    chapters_and_subchapters = OrderedDict()
     if chapter_data and course_name:
         for d in chapter_data: # Set up Chapter-Subchs dictionary
             ch_name, sub_chs = d['ch'], d['sub_chs']
             if d['ch'] not in chapters_and_subchapters:
-                chapters_and_subchapters[d['ch']] = [x for x in d['sub_chs']]
+                chapters_and_subchapters[d['ch']] = d['sub_chs']
             else:
+                # The order matters with respect to the list wherein they're added to the dictionary. 
                 for subch in d['sub_chs']:
                     chapters_and_subchapters[d['ch']].append(subch)
 

From 97371af4ae60357c84bdeb1a471d091e38b621e1 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sun, 18 Sep 2016 08:56:09 -0400
Subject: [PATCH 39/46] better saving of htmlsource; autograde field for
 assignment_questions; updates to assignment directive

---
 runestone/activecode/activecode.py    |  3 +-
 runestone/activecode/js/activecode.js | 18 ++++++---
 runestone/assess/assessbase.py        |  1 +
 runestone/assess/multiplechoice.py    | 14 ++++++-
 runestone/assignment/__init__.py      | 57 ++++++++++++++++++++++++---
 runestone/external/external.py        | 16 ++++----
 runestone/server/componentdb.py       | 15 ++++++-
 7 files changed, 101 insertions(+), 23 deletions(-)

diff --git a/runestone/activecode/activecode.py b/runestone/activecode/activecode.py
index 393d1560b..f79631751 100644
--- a/runestone/activecode/activecode.py
+++ b/runestone/activecode/activecode.py
@@ -192,8 +192,9 @@ def run(self):
             env.activecodecounter = 0
         env.activecodecounter += 1
         self.options['name'] = self.arguments[0].strip()
-
+        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
         self.options['divid'] = self.arguments[0]
+
         if not self.options['divid']:
             raise Exception("No divid for ..activecode or ..actex in activecode.py")
 
diff --git a/runestone/activecode/js/activecode.js b/runestone/activecode/js/activecode.js
index 4113a0bb0..b90913973 100755
--- a/runestone/activecode/js/activecode.js
+++ b/runestone/activecode/js/activecode.js
@@ -443,11 +443,19 @@ ActiveCode.prototype.createGradeSummary = function () {
         var report = eval(data)[0];
         // check for report['message']
         if (report) {
-            body = "<h4>Grade Report</h4>" +
-                   "<p>This assignment: " + report['grade'] + "</p>" +
-                   "<p>" + report['comment'] + "</p>" +
-                   "<p>Number of graded assignments: " + report['count'] + "</p>" +
-                   "<p>Average score: " +  report['avg'] + "</p>"
+            if (report['version'] == 2){
+                // new version; would be better to embed this in HTML for the activecode
+                body = "<h4>Grade Report</h4>" +
+                       "<p>This question: " + report['grade'] + " out of " + report['max'] + "</p>" +
+                       "<p>" + report['comment'] + "</p>"
+            }
+            else{
+                body = "<h4>Grade Report</h4>" +
+                       "<p>This assignment: " + report['grade'] + "</p>" +
+                       "<p>" + report['comment'] + "</p>" +
+                       "<p>Number of graded assignments: " + report['count'] + "</p>" +
+                       "<p>Average score: " +  report['avg'] + "</p>"
+            }
 
         } else {
             body = "<h4>The server did not return any grade information</h4>";
diff --git a/runestone/assess/assessbase.py b/runestone/assess/assessbase.py
index 662465c20..349e84335 100644
--- a/runestone/assess/assessbase.py
+++ b/runestone/assess/assessbase.py
@@ -78,6 +78,7 @@ def run(self):
         self.options['qnumber'] = self.getNumber()
         
         self.options['divid'] = self.arguments[0]
+        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
 
         if self.content[0][:2] == '..':  # first line is a directive
             self.content[0] = self.options['qnumber'] + ': \n\n' + self.content[0]
diff --git a/runestone/assess/multiplechoice.py b/runestone/assess/multiplechoice.py
index 8a8b5bfe4..74297c2a1 100644
--- a/runestone/assess/multiplechoice.py
+++ b/runestone/assess/multiplechoice.py
@@ -21,7 +21,7 @@
 from .assessbase import *
 import json
 import random
-from runestone.server.componentdb import addQuestionToDB
+from runestone.server.componentdb import addQuestionToDB, addHTMLToDB
 
 
 class MChoiceNode(nodes.General, nodes.Element):
@@ -36,6 +36,10 @@ def __init__(self,content):
         self.mc_options = content
 
 def visit_mc_node(self,node):
+
+    node.delimiter = "_start__{}_".format(node.mc_options['divid'])
+    self.body.append(node.delimiter)
+
     res = ""
     if 'random' in node.mc_options:
         node.mc_options['random'] = 'data-random'
@@ -70,9 +74,15 @@ def depart_mc_node(self,node):
 
 
     res += node.template_end % node.mc_options
-
     self.body.append(res)
 
+    addHTMLToDB(node.mc_options['divid'],
+                node.mc_options['basecourse'],
+                "".join(self.body[self.body.index(node.delimiter) + 1:]))
+
+    self.body.remove(node.delimiter)
+
+
 
 
 
diff --git a/runestone/assignment/__init__.py b/runestone/assignment/__init__.py
index 979db8957..69376a1c0 100644
--- a/runestone/assignment/__init__.py
+++ b/runestone/assignment/__init__.py
@@ -20,16 +20,43 @@
 from docutils import nodes
 from docutils.parsers.rst import directives
 from docutils.parsers.rst import Directive
-from runestone.server.componentdb import addAssignmentToDB, addAssignmentQuestionToDB, getCourseID, getOrCreateAssignmentType, getQuestionID
+from runestone.server.componentdb import addAssignmentToDB, addAssignmentQuestionToDB, getCourseID, getOrCreateAssignmentType, getQuestionID, get_HTML_from_DB
 from runestone.common.runestonedirective import RunestoneDirective
 from datetime import datetime
 
 def setup(app):
     app.add_directive('assignment',Assignment)
+    app.add_node(AssignmentNode, html=(visit_a_node, depart_a_node))
 
     app.connect('doctree-resolved',process_nodes)
     app.connect('env-purge-doc', purge)
 
+class AssignmentNode(nodes.General, nodes.Element):
+    def __init__(self, content):
+        """
+
+        Arguments:
+        - `self`:
+        - `content`:
+        """
+        super(AssignmentNode, self).__init__(name=content['name'])
+        self.a_components = content
+
+
+def visit_a_node(self, node):
+    pass
+
+def depart_a_node(self, node):
+    question_ids = node.a_components['question_ids']
+    basecourse = node.a_components['basecourse']
+    for q_id in question_ids:
+        src = get_HTML_from_DB(q_id, basecourse)
+        if src:
+            self.body.append(src)
+        else:
+            self.body.append("<p>Missing HTML source for {}</p>".format(q_id))
+            print("No HTML source saved for {}; can't include that kind of question until code writing to HTML is implemented for that directive".format(q_id))
+
 def process_nodes(app,env,docname):
     pass
 
@@ -38,6 +65,7 @@ def purge(app,env,docname):
     pass
 
 
+
 class Assignment(RunestoneDirective):
     """
         .. assignment:
@@ -56,7 +84,10 @@ class Assignment(RunestoneDirective):
         'assignment_type': directives.unchanged,
         'questions': directives.unchanged,
         'deadline':directives.unchanged,
-        'points':directives.positive_int
+        'points':directives.positive_int,
+        'threshold': directives.positive_int,
+        'autograde': directives.unchanged,
+        'generate_html': directives.flag
     })
 
     def run(self):
@@ -67,12 +98,16 @@ def run(self):
                 :questions: (divid_1 50), (divid_2 100), ...
                 :deadline: 23-09-2016 15:30
                 :points: integer
+                :threshold: integer
+                :autograde: visited
+                :generate_html:
         """
 
         course_name = self.state.document.settings.env.config.html_context['course_id']
         self.options['course_name'] = course_name
         course_id = getCourseID(course_name)
         basecourse_name = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
+        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
 
         name = self.options.get('name') # required; error if missing
         assignment_type_name = self.options.get('assignment_type')
@@ -91,27 +126,37 @@ def run(self):
                     self.state.document.settings.env.warn(self.state.document.settings.env.docname, "deadline missing or incorrectly formatted; Omitting deadline")
 
         points = self.options.get('points', 0)
+        threshold = self.options.get('threshold', None)
+        if threshold:
+            threshold = int(threshold)
+        autograde = self.options.get('autograde', None)
 
         assignment_id = addAssignmentToDB(name = name,
                           course_id = course_id,
                           assignment_type_id = assignment_type_id,
                           deadline = deadline,
-                          points = points)
+                          points = points,
+                          threshold = threshold)
 
         unparsed = self.options.get('questions', None)
+        question_names = []
         if unparsed:
             summative_type_id = getOrCreateAssignmentType("summative")
             q_strings = unparsed.split(',')
             for q in q_strings:
                 (question_name, points) = q.strip().split()
-
+                question_names.append(question_name)
                 # first get the question_id associated with question_name
                 question_id = getQuestionID(basecourse_name, question_name)
                 if question_id:
-                    addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = summative_type_id)
+                    addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = summative_type_id, autograde = autograde)
                 else:
                     self.state.document.settings.env.warn(self.state.document.settings.env.docname, "Question {} is not in the database for basecourse {}".format(question_name, basecourse_name))
         else:
             self.state.document.settings.env.warn(self.state.document.settings.env.docname, "No questions for assignment {}".format(name))
+        self.options['question_ids'] = question_names
 
-        return []
+        if 'generate_html' in self.options:
+            return [AssignmentNode(self.options)]
+        else:
+            return []
diff --git a/runestone/external/external.py b/runestone/external/external.py
index 2706ded9e..ba297f52d 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -49,23 +49,25 @@ def visit_external_node(self, node):
     # Set options and format templates accordingly
     # env = node.document.settings.env
 
-    res = TEMPLATE_START % node.external_options
-    self.body.append(res)
+    node.delimiter = "_start__{}_".format(node.ac_components['divid'])
 
-    addHTMLToDB(node.external_options['divid'],
-                node.external_options['basecourse'],
-                "".join(""))
+    self.body.append(node.delimiter)
 
-    # self.body.remove(node.delimiter)
+    res = TEMPLATE_START % node.external_options
+    self.body.append(res)
 
 
 def depart_external_node(self, node):
     # Set options and format templates accordingly
     res = TEMPLATE_END % node.external_options
-    delimiter = "_start__{}_".format(node.external_options['divid'])
 
     self.body.append(res)
 
+    addHTMLToDB(node.external_options['divid'],
+                node.external_options['basecourse'],
+                "".join(""))
+
+    self.body.remove(node.delimiter)
 
 # Templates to be formatted by node options
 TEMPLATE_START = '''
diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index a77c4423f..3560d5975 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -163,7 +163,7 @@ def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_po
         res = engine.execute(ins)
         return res.inserted_primary_key[0]
 
-def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = None, timed=None):
+def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = None, timed=None, autograde=None):
     meta = MetaData()
     questions = Table('questions', meta, autoload=True, autoload_with=engine)
     assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
@@ -179,7 +179,8 @@ def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_typ
             question_id = question_id,
             points = points,
             timed= timed,
-            assessment_type = assessment_type
+            assessment_type = assessment_type,
+            autograde = autograde
             )
         engine.execute(stmt)
     else:
@@ -257,3 +258,13 @@ def addHTMLToDB(divid, basecourse, htmlsrc):
         except:
             print("Error while trying to add directive {} to the DB".format(divid))
 
+def get_HTML_from_DB(divid, basecourse):
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+    sel = select([questions]).where(and_(questions.c.name == divid,
+                                          questions.c.base_course == basecourse))
+    res = engine.execute(sel).first()
+    if res:
+        return res['htmlsrc']
+    else:
+        return ""

From 77f3b013de3be3c1dece4cc5f6e8d04c18fea80c Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sun, 18 Sep 2016 09:09:09 -0400
Subject: [PATCH 40/46] Revert "show email instead of username"

This reverts commit cfd993fc6ca826afcde737dc525e36ce83ce55e4.
---
 runestone/common/js/bookfuncs.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runestone/common/js/bookfuncs.js b/runestone/common/js/bookfuncs.js
index 7a3ef3792..5445b0ba2 100644
--- a/runestone/common/js/bookfuncs.js
+++ b/runestone/common/js/bookfuncs.js
@@ -120,8 +120,7 @@ function gotUser(data, status, whatever) {
         }
     } else {
         if (!caughtErr) {
-            // mess = "username: " + d.nick;
-            mess = "user: " + d.email;
+            mess = "username: " + d.nick;
             eBookConfig.email = d.email;
             eBookConfig.isLoggedIn = true;
             eBookConfig.cohortId = d.cohortId;

From cedaaf2981ae8eddeb63cd166012fbabf8329560 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sun, 18 Sep 2016 10:40:44 -0400
Subject: [PATCH 41/46] external directive bug fix

---
 runestone/external/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runestone/external/external.py b/runestone/external/external.py
index ba297f52d..8ba2ab76d 100644
--- a/runestone/external/external.py
+++ b/runestone/external/external.py
@@ -49,7 +49,7 @@ def visit_external_node(self, node):
     # Set options and format templates accordingly
     # env = node.document.settings.env
 
-    node.delimiter = "_start__{}_".format(node.ac_components['divid'])
+    node.delimiter = "_start__{}_".format(node.external_options['divid'])
 
     self.body.append(node.delimiter)
 

From bf54a587eed86596daf1b139ef97fa0e3070231a Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Sun, 18 Sep 2016 15:46:56 -0400
Subject: [PATCH 42/46] save autograde marking in the question_assignment table

---
 runestone/usageAssignment/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runestone/usageAssignment/__init__.py b/runestone/usageAssignment/__init__.py
index ff1630db5..7245ac2e3 100644
--- a/runestone/usageAssignment/__init__.py
+++ b/runestone/usageAssignment/__init__.py
@@ -245,6 +245,6 @@ def run(self):
 
         for acid in paths + active_codes:
             q_id = getOrInsertQuestionForPage(base_course=basecourse_name, name=acid, is_private='F', question_type="page", autograde = "visited", difficulty=1,chapter=None)
-            addAssignmentQuestionToDB(q_id, assignment_id, 1)
+            addAssignmentQuestionToDB(q_id, assignment_id, 1, autograde="visited")
 
         return [usageAssignmentNode(self.options)]

From 5663e2b9a684c093396ddf73a5ceea1f26781a12 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Tue, 4 Oct 2016 09:41:31 -0400
Subject: [PATCH 43/46] delete assignment_questions when adding assignment

---
 runestone/server/componentdb.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 3560d5975..be7f79d27 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -221,7 +221,12 @@ def addAssignmentToDB(name = None, course_id = None, assignment_type_id = None,
             threshold = threshold
         )
         engine.execute(stmt)
-        return res['id']
+        a_id = res['id']
+        # delete all existing AssignmentQuestions, so that you don't have any leftovers
+        # this is safe because grades and comments are associated with div_ids and course_names, not assignment_questions rows.
+        stmt2 = assignment_questions.delete().where(assignment_questions.c.assignment_id == a_id)
+        engine.execute(stmt2)
+
     else:
         ins = assignments.insert().values(
             course=course_id,
@@ -231,8 +236,9 @@ def addAssignmentToDB(name = None, course_id = None, assignment_type_id = None,
             points = points,
             threshold = threshold)
         res = engine.execute(ins)
-        return res.inserted_primary_key[0]
+        a_id = res.inserted_primary_key[0]
 
+    return a_id
 
 def addHTMLToDB(divid, basecourse, htmlsrc):
     if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):

From c39459a9d2c9571167f6de23f244f67821a4dc18 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 5 Oct 2016 05:58:52 -0400
Subject: [PATCH 44/46] add autograde flag in assignment_question insertion

---
 runestone/server/componentdb.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index be7f79d27..84b9ce26b 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -190,7 +190,8 @@ def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_typ
             question_id = question_id,
             points = points,
             timed=timed,
-            assessment_type = assessment_type
+            assessment_type = assessment_type,
+            autograde = autograde
             )
         engine.execute(ins)
 

From b664204c09690ddb7b15040005afffd6498975b3 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Wed, 5 Oct 2016 05:58:52 -0400
Subject: [PATCH 45/46] add autograde flag in assignment_question insertion

Conflicts:
	runestone/server/componentdb.py
---
 runestone/server/componentdb.py | 169 ++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)

diff --git a/runestone/server/componentdb.py b/runestone/server/componentdb.py
index 686262a38..84b9ce26b 100644
--- a/runestone/server/componentdb.py
+++ b/runestone/server/componentdb.py
@@ -22,6 +22,14 @@
 import os
 from sqlalchemy import create_engine, Table, MetaData, select, delete, update, and_
 
+# create a global DB query engine to share for the rest of the file
+if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
+    dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
+    engine = create_engine(dburl)
+else:
+    dburl = None
+    engine = None
+    print("Skipping all DB operations because environment variables not set up")
 
 def logSource(self):
     sourcelog = self.state.document.settings.env.config.html_context.get('dsource', None)
@@ -66,8 +74,10 @@ def addQuestionToDB(self):
         srcpath, line = self.state_machine.get_source_and_line()
         subchapter = os.path.basename(srcpath).replace('.rst','')
         chapter = srcpath.split(os.path.sep)[-2]
+
         autograde = self.options.get('autograde', None)
 
+
         sel = select([questions]).where(and_(questions.c.name == self.arguments[0],
                                               questions.c.base_course == basecourse))
         res = engine.execute(sel).first()
@@ -82,6 +92,155 @@ def addQuestionToDB(self):
         except UnicodeEncodeError:
             raise self.severe("Bad character in directive {} in {}/{} this will not be saved to the DB".format(self.arguments[0], self.chapter, self.subchapter))
 
+def getQuestionID(base_course, name):
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+
+
+    sel = select([questions]).where(and_(questions.c.name == name,
+                                          questions.c.base_course == base_course))
+    res = engine.execute(sel).first()
+    if res:
+        return res['id']
+    else:
+        return None
+
+def getOrInsertQuestionForPage(base_course=None, name=None, is_private='F', question_type="page", autograde = "visited", author=None, difficulty=1,chapter=None):
+    last_changed = datetime.now()
+
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+
+
+    sel = select([questions]).where(and_(questions.c.name == name,
+                                          questions.c.base_course == base_course))
+    res = engine.execute(sel).first()
+
+    if res:
+        id = res['id']
+        stmt = questions.update().where(questions.c.id == id).values(
+            timestamp=last_changed,
+            is_private= is_private,
+            question_type=question_type,
+            autograde = autograde,
+            author=author,
+            difficulty=difficulty,
+            chapter=chapter)
+        res = engine.execute(stmt)
+        return id
+    else:
+        ins = questions.insert().values(
+            base_course= base_course,
+            name= name,
+            timestamp=last_changed,
+            is_private= is_private,
+            question_type=question_type,
+            autograde = autograde,
+            author=author,
+            difficulty=difficulty,
+            chapter=chapter)
+        res = engine.execute(ins)
+        return res.inserted_primary_key[0]
+
+def getOrCreateAssignmentType(assignment_type_name, grade_type = None, points_possible = None, assignments_count = None, assignments_dropped = None):
+
+    meta = MetaData()
+    assignment_types = Table('assignment_types', meta, autoload=True, autoload_with=engine)
+
+    # search for it in the DB
+    sel = select([assignment_types]).where(assignment_types.c.name == assignment_type_name)
+    res = engine.execute(sel).first()
+    if res:
+        return res['id']
+    else:
+        # create the assignment type
+        ins = assignment_types.insert().values(
+            name=assignment_type_name,
+            grade_type = grade_type,
+            points_possible = points_possible,
+            assignments_count = assignments_count,
+            assignments_dropped = assignments_dropped)
+        res = engine.execute(ins)
+        return res.inserted_primary_key[0]
+
+def addAssignmentQuestionToDB(question_id, assignment_id, points, assessment_type = None, timed=None, autograde=None):
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+    assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
+
+    # now insert or update the assignment_questions row
+    sel = select([assignment_questions]).where(and_(assignment_questions.c.assignment_id == assignment_id,
+                                          assignment_questions.c.question_id == question_id))
+    res = engine.execute(sel).first()
+    if res:
+        #update
+        stmt = assignment_questions.update().where(assignment_questions.c.id == res['id']).values( \
+            assignment_id = assignment_id,
+            question_id = question_id,
+            points = points,
+            timed= timed,
+            assessment_type = assessment_type,
+            autograde = autograde
+            )
+        engine.execute(stmt)
+    else:
+        #insert
+        ins = assignment_questions.insert().values(
+            assignment_id = assignment_id,
+            question_id = question_id,
+            points = points,
+            timed=timed,
+            assessment_type = assessment_type,
+            autograde = autograde
+            )
+        engine.execute(ins)
+
+def getCourseID(coursename):
+    meta = MetaData()
+    courses = Table('courses', meta, autoload=True, autoload_with=engine)
+
+    sel = select([courses]).where(courses.c.course_name == coursename)
+    res = engine.execute(sel).first()
+    return res['id']
+
+def addAssignmentToDB(name = None, course_id = None, assignment_type_id = None, deadline = None, points = None, threshold = None):
+
+    last_changed = datetime.now()
+
+    meta = MetaData()
+    assignments = Table('assignments', meta, autoload=True, autoload_with=engine)
+    assignment_questions = Table('assignment_questions', meta, autoload=True, autoload_with=engine)
+
+    sel = select([assignments]).where(and_(assignments.c.name == name,
+                                          assignments.c.course == course_id))
+    res = engine.execute(sel).first()
+    if res:
+        stmt = assignments.update().where(assignments.c.id == res['id']).values(
+            assignment_type = assignment_type_id,
+            duedate = deadline,
+            points = points,
+            threshold = threshold
+        )
+        engine.execute(stmt)
+        a_id = res['id']
+        # delete all existing AssignmentQuestions, so that you don't have any leftovers
+        # this is safe because grades and comments are associated with div_ids and course_names, not assignment_questions rows.
+        stmt2 = assignment_questions.delete().where(assignment_questions.c.assignment_id == a_id)
+        engine.execute(stmt2)
+
+    else:
+        ins = assignments.insert().values(
+            course=course_id,
+            name=name,
+            assignment_type = assignment_type_id,
+            duedate = deadline,
+            points = points,
+            threshold = threshold)
+        res = engine.execute(ins)
+        a_id = res.inserted_primary_key[0]
+
+    return a_id
+
 def addHTMLToDB(divid, basecourse, htmlsrc):
     if all(name in os.environ for name in ['DBHOST', 'DBPASS', 'DBUSER', 'DBNAME']):
         dburl = 'postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}'.format(**os.environ)
@@ -106,3 +265,13 @@ def addHTMLToDB(divid, basecourse, htmlsrc):
         except:
             print("Error while trying to add directive {} to the DB".format(divid))
 
+def get_HTML_from_DB(divid, basecourse):
+    meta = MetaData()
+    questions = Table('questions', meta, autoload=True, autoload_with=engine)
+    sel = select([questions]).where(and_(questions.c.name == divid,
+                                          questions.c.base_course == basecourse))
+    res = engine.execute(sel).first()
+    if res:
+        return res['htmlsrc']
+    else:
+        return ""

From 3ce03a35ae4a5604c49fb2547574be8dff4728b1 Mon Sep 17 00:00:00 2001
From: Paul Resnick <presnick@umich.edu>
Date: Thu, 6 Oct 2016 11:46:23 -0400
Subject: [PATCH 46/46] remove redundant setting of basecourse which is handled
 more generically in runestonedirective.py

---
 runestone/activecode/activecode.py | 1 -
 runestone/assess/assessbase.py     | 1 -
 2 files changed, 2 deletions(-)

diff --git a/runestone/activecode/activecode.py b/runestone/activecode/activecode.py
index f79631751..dba2690ff 100644
--- a/runestone/activecode/activecode.py
+++ b/runestone/activecode/activecode.py
@@ -192,7 +192,6 @@ def run(self):
             env.activecodecounter = 0
         env.activecodecounter += 1
         self.options['name'] = self.arguments[0].strip()
-        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
         self.options['divid'] = self.arguments[0]
 
         if not self.options['divid']:
diff --git a/runestone/assess/assessbase.py b/runestone/assess/assessbase.py
index 349e84335..662465c20 100644
--- a/runestone/assess/assessbase.py
+++ b/runestone/assess/assessbase.py
@@ -78,7 +78,6 @@ def run(self):
         self.options['qnumber'] = self.getNumber()
         
         self.options['divid'] = self.arguments[0]
-        self.options['basecourse'] = self.state.document.settings.env.config.html_context.get('basecourse', "unknown")
 
         if self.content[0][:2] == '..':  # first line is a directive
             self.content[0] = self.options['qnumber'] + ': \n\n' + self.content[0]