From 7ed12fe56742996ca194ba4d8412aad94dc85768 Mon Sep 17 00:00:00 2001
From: Kristin Stephens <ksteph@cs.berkeley.edu>
Date: Fri, 2 Aug 2013 14:41:42 -0700
Subject: [PATCH] New queryable django app

Makes new tables that are queryable to get aggregate data on student per
problem information and student grades at three levels: overall course,
assignment type, and assignment.
---
 lms/djangoapps/queryable/__init__.py          |   0
 .../queryable/management/__init__.py          |   0
 .../queryable/management/commands/__init__.py |   0
 .../commands/populate_studentgrades.py        | 346 +++++++++
 .../commands/populate_studentmoduleexpand.py  | 108 +++
 .../queryable/migrations/0001_initial.py      | 222 ++++++
 .../queryable/migrations/__init__.py          |   0
 lms/djangoapps/queryable/models.py            | 104 +++
 .../tests/test_populate_studentgrades.py      | 734 ++++++++++++++++++
 .../test_populate_studentmoduleexpand.py      | 186 +++++
 lms/djangoapps/queryable/tests/test_util.py   | 256 ++++++
 lms/djangoapps/queryable/util.py              |  40 +
 lms/envs/dev.py                               |   3 +
 lms/envs/test.py                              |   3 +
 14 files changed, 2002 insertions(+)
 create mode 100644 lms/djangoapps/queryable/__init__.py
 create mode 100644 lms/djangoapps/queryable/management/__init__.py
 create mode 100644 lms/djangoapps/queryable/management/commands/__init__.py
 create mode 100644 lms/djangoapps/queryable/management/commands/populate_studentgrades.py
 create mode 100644 lms/djangoapps/queryable/management/commands/populate_studentmoduleexpand.py
 create mode 100644 lms/djangoapps/queryable/migrations/0001_initial.py
 create mode 100644 lms/djangoapps/queryable/migrations/__init__.py
 create mode 100644 lms/djangoapps/queryable/models.py
 create mode 100644 lms/djangoapps/queryable/tests/test_populate_studentgrades.py
 create mode 100644 lms/djangoapps/queryable/tests/test_populate_studentmoduleexpand.py
 create mode 100644 lms/djangoapps/queryable/tests/test_util.py
 create mode 100644 lms/djangoapps/queryable/util.py

diff --git a/lms/djangoapps/queryable/__init__.py b/lms/djangoapps/queryable/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/lms/djangoapps/queryable/management/__init__.py b/lms/djangoapps/queryable/management/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/lms/djangoapps/queryable/management/commands/__init__.py b/lms/djangoapps/queryable/management/commands/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/lms/djangoapps/queryable/management/commands/populate_studentgrades.py b/lms/djangoapps/queryable/management/commands/populate_studentgrades.py
new file mode 100644
index 000000000000..655dcf6e3ada
--- /dev/null
+++ b/lms/djangoapps/queryable/management/commands/populate_studentgrades.py
@@ -0,0 +1,346 @@
+# ======== Populate Student Grades  ====================================================================================
+#
+# Populates the student grade tables of the queryable_table model (CourseGrade, AssignmentTypeGrade, AssignmentGrade).
+#
+# For the provided course_id, it will find all students that may have changed their grade since the last populate. Of
+# these students rows for the course grade and assignment type are created only if the student has submitted at
+# least one answer to any problem in the course. Rows for assignments are only created if the student has submitted an
+# answer to one of the problems in that assignment. Updates only occur if there is a change in the values the row should
+# be storing. 
+
+import json
+import re
+
+from datetime import datetime
+from pytz import UTC
+from optparse import make_option
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+
+from courseware import grades
+from courseware.courses import get_course_by_id
+from courseware.models import StudentModule
+
+from queryable.models import Log, CourseGrade, AssignmentTypeGrade, AssignmentGrade
+from queryable import util
+
+
+################## Helper Functions ##################
+def update_course_grade(course_grade, gradeset):
+    """
+    Returns true if the course grade needs to be updated.
+    """
+    return (not util.approx_equal(course_grade.percent, gradeset['percent'])) or (course_grade.grade != gradeset['grade'])
+
+
+def get_assignment_index(assignment):
+    """
+    Returns the assignment's index, -1 if an index can't be found.
+
+    `assignment` is a string formatted like this "HW 02" and this function returns 2 in this case.
+
+    The string is the 'label' for each section in the 'section_breakdown' of the dictionary returned by the grades.grade
+    function.
+    """
+
+    m = re.search('.* (\d+)', assignment)
+    index = -1
+    if m:
+        index = int(m.group(1))-1
+
+    return index
+
+
+def assignment_exists_and_has_problems(assignment_problems_map, category, index):
+    """
+    Returns True if the assignment for the category and index exists and has problems
+
+    `assignment_problems_map` a dictionary returned by get_assignment_to_problem_map(course_id)
+
+    `category` string specifying the category or assignment type for this assignment
+
+    `index` zero-based indexing into the array of assignments for that category
+    """
+
+    if index < 0:
+        return False
+
+    if category not in assignment_problems_map:
+        return False
+
+    if index >= len(assignment_problems_map[category]):
+        return False
+
+    return len(assignment_problems_map[category][index]) > 0
+
+
+def get_student_problems(course_id, student):
+    """
+    Returns an array of problem ids that the student has answered for this course.
+
+    `course_id` the course ID for the course interested in
+
+    `student` the student want to get his/her problems
+
+    Queries the database to get the problems the student has submitted an answer to for the course specified.
+    """
+
+    query = StudentModule.objects.filter(
+        course_id__exact=course_id,
+        student=student,
+        grade__isnull=False,
+        module_type__exact='problem',
+    ).values('module_state_key').distinct()
+    
+    student_problems = []
+    for problem in query:
+        student_problems.append(problem['module_state_key'])
+
+    return student_problems
+
+
+def student_did_problems(student_problems, problem_set):
+    """
+    Returns true if `student_problems` and `problem_set` share problems.
+
+    `student_problems` array of problem ids the student has done
+
+    `problem_set` array of problem ids
+    """
+
+    return (set(student_problems) & set(problem_set))
+
+
+def store_course_grade_if_need(student, course_id, gradeset):
+    """
+    Stores the course grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `gradeset` the values returned by grades.grade
+
+    The course grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    update_course_grade is true.
+    """
+
+    course_grade, created = CourseGrade.objects.get_or_create(user=student, course_id=course_id)
+
+    if created or update_course_grade(course_grade, gradeset):
+        course_grade.percent = gradeset['percent']
+        course_grade.grade = gradeset['grade']
+        course_grade.save()
+        return True
+
+    return False
+
+
+def store_assignment_type_grade_if_need(student, course_id, category, percent):
+    """
+    Stores the assignment type grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `category` the category for the assignment type, found in the return value of the grades.grade function
+
+    `percent` the percent grade the student received for this assignment
+
+    The assignment type grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    if the percent value is different than what is currently in the database.
+    """
+
+    assign_type_grade, created = AssignmentTypeGrade.objects.get_or_create(
+        user=student,
+        course_id=course_id,
+        category=category,
+    )
+
+    if created or not util.approx_equal(assign_type_grade.percent, percent):
+        assign_type_grade.percent = percent
+        assign_type_grade.save()
+        return True
+
+    return False
+
+
+def store_assignment_grade_if_need(student, course_id, label, percent):
+    """
+    Stores the assignment grade for the student and course if needed, returns True if it was stored
+
+    `student` is a User object representing the student
+
+    `course_id` the course's ID
+
+    `label` the label for the assignment, found in the return value of the grades.grade function
+
+    `percent` the percent grade the student received for this assignment
+
+    The assignment grade is stored if it has never been stored before (i.e. this is a new row in the database) or
+    if the percent value is different than what is currently in the database.
+    """
+
+    assign_grade, created = AssignmentGrade.objects.get_or_create(
+        user=student,
+        course_id=course_id,
+        label=label,
+    )
+
+    if created or not util.approx_equal(assign_grade.percent, percent):
+        assign_grade.percent = percent
+        assign_grade.save()
+        return True
+
+    return False
+
+################## Actual Command ##################
+class Command(BaseCommand):
+    help = "Populates the queryable.StudentGrades table.\n"
+    help += "Usage: populate_studentgrades course_id\n"
+    help += "   course_id: course's ID, such as Medicine/HRP258/Statistics_in_Medicine\n"
+
+    option_list = BaseCommand.option_list + (
+        make_option('-f', '--force',
+                    action='store_true',
+                    dest='force',
+                    default=False,
+                    help='Forces a full populate for all students and rows, rather than iterative.'),
+        )
+
+    def handle(self, *args, **options):
+        script_id = "studentgrades"
+
+        print "args = ", args
+
+        if len(args) > 0:
+            course_id = args[0]
+        else:
+            print self.help
+            return
+
+        assignment_problems_map = util.get_assignment_to_problem_map(course_id)
+
+        print "--------------------------------------------------------------------------------"
+        print "Populating queryable.StudentGrades table for course {0}".format(course_id)
+        print "--------------------------------------------------------------------------------"
+
+        # Grab when we start, to log later
+        tstart = datetime.now(UTC)
+
+        iterative_populate = True
+        if options['force']:
+            print "--------------------------------------------------------------------------------"
+            print "Full populate: Forced full populate"
+            print "--------------------------------------------------------------------------------"
+            iterative_populate = False
+
+        if iterative_populate:
+            # Get when this script was last run for this course
+            last_log_run = Log.objects.filter(script_id__exact=script_id, course_id__exact=course_id)
+
+            length = len(last_log_run)
+            print "--------------------------------------------------------------------------------"
+            if length > 0:
+                print "Iterative populate: Last log run", last_log_run[0].created
+            else:
+                print "Full populate: Can't find log of last run"
+                iterative_populate = False
+            print "--------------------------------------------------------------------------------"
+        
+        # If iterative populate get all students since last populate, otherwise get all students that fit the criteria.
+        # Criteria: match course_id, module_type is 'problem', grade is not null because it means they have submitted an
+        # answer to a problem that might effect their grade.
+        if iterative_populate:
+            students = User.objects.filter(studentmodule__course_id=course_id,
+                                           studentmodule__module_type='problem',
+                                           studentmodule__grade__isnull=False,
+                                           studentmodule__modified__gte=last_log_run[0].created).distinct()
+        else:
+            students = User.objects.filter(studentmodule__course_id=course_id,
+                                           studentmodule__module_type='problem',
+                                           studentmodule__grade__isnull=False).distinct()
+
+        # Create a dummy request to pass to the grade function.
+        # Code originally from lms/djangoapps/instructor/offline_gradecalc.py
+        # Copying instead of using that code so everything is self contained in this django app.
+        class DummyRequest(object):
+            META = {}
+            def __init__(self):
+                return
+            def get_host(self):
+                return 'edx.mit.edu'
+            def is_secure(self):
+                return False
+
+        # Get course using the id, to pass to the grade function
+        course = get_course_by_id(course_id)
+
+        c_updated_students = 0
+        for student in students:
+            updated = False
+            student_problems = None
+
+            # Create dummy request and set its user and session
+            request = DummyRequest()
+            request.user = student
+            request.session = {}
+
+            # Call grade to get the gradeset
+            gradeset = grades.grade(student, request, course, keep_raw_scores=False)
+
+            updated = store_course_grade_if_need(student, course_id, gradeset)
+
+            # Iterate through the section_breakdown
+            for section in gradeset['section_breakdown']:
+                # If the dict has 'prominent' and it's True this is at the assignment type level, store it if need
+                if ('prominent' in section) and section['prominent']:
+                    updated = store_assignment_type_grade_if_need(
+                        student, course_id, section['category'], section['percent']
+                    )
+
+                else: #If no 'prominent' or it's False this is at the assignment level
+                    store = False
+
+                    # If the percent is 0, there are three possibilities:
+                    # 1. There are no problems for that assignment yet -> skip section
+                    # 2. The student hasn't submitted an answer to any problem for that assignment -> skip section
+                    # 3. The student has submitted answers and got zero -> record
+                    # Only store for #3
+                    if section['percent'] > 0:
+                        store = True
+                    else:
+                        # Find which assignment this is for this type/category
+                        index = get_assignment_index(section['label'])
+                        if index < 0:
+                            print "WARNING: Can't find index for the following section, skipping"
+                            print section
+                        else:
+                            if assignment_exists_and_has_problems(assignment_problems_map, section['category'], index):
+
+                                # Get problems student has done, only do this database call if needed
+                                if student_problems == None:
+                                    student_problems = get_student_problems(course_id, student)
+                                    
+                                curr_assignment_problems = assignment_problems_map[section['category']][index]
+                                    
+                                if student_did_problems(student_problems, curr_assignment_problems):
+                                    store = True
+
+                    if store:
+                        updated = store_assignment_grade_if_need(
+                            student, course_id, section['label'], section['percent']
+                        )
+
+            if updated:
+                c_updated_students += 1
+
+        c_all_students = len(students)
+        print "--------------------------------------------------------------------------------"
+        print "Done! Updated {0} students' grades out of {1}".format(c_updated_students, c_all_students)
+        print "--------------------------------------------------------------------------------"
+
+        # Save since everything finished successfully, log latest run.
+        q_log = Log(script_id=script_id, course_id=course_id, created=tstart)
+        q_log.save()
diff --git a/lms/djangoapps/queryable/management/commands/populate_studentmoduleexpand.py b/lms/djangoapps/queryable/management/commands/populate_studentmoduleexpand.py
new file mode 100644
index 000000000000..90d26dc15d88
--- /dev/null
+++ b/lms/djangoapps/queryable/management/commands/populate_studentmoduleexpand.py
@@ -0,0 +1,108 @@
+# ======== Populate StudentModuleExpand  ===============================================================================
+#
+# Populates the StudentModuleExpand table of the queryable_table model.
+#
+# For the provided course_id, it will find all rows in the StudentModule table of the courseware model that have
+# module_type 'problem' and the grade is not null. Then for any rows that have changed since the last populate or do not
+# have a corresponding row, update the attempts value.
+
+import json
+
+from datetime import datetime
+from pytz import UTC
+from optparse import make_option
+from django.core.management.base import BaseCommand
+
+from xmodule.modulestore.django import modulestore
+
+from courseware.models import StudentModule
+from queryable.models import Log
+from queryable.models import StudentModuleExpand
+
+class Command(BaseCommand):
+    help = "Populates the queryable.StudentModuleExpand table.\n"
+    help += "Usage: populate_studentmoduleexpand course_id\n"
+    help += "   course_id: course's ID, such as Medicine/HRP258/Statistics_in_Medicine\n"
+
+    option_list = BaseCommand.option_list + (
+        make_option('-f', '--force',
+                    action='store_true',
+                    dest='force',
+                    default=False,
+                    help='Forces a full populate for all students and rows, rather than iterative.'),
+        )
+
+    def handle(self, *args, **options):
+        script_id = "studentmoduleexpand"
+
+        print "args = ", args
+
+        if len(args) > 0:
+            course_id = args[0]
+        else:
+            print self.help
+            return
+
+        print "--------------------------------------------------------------------------------"
+        print "Populating queryable.StudentModuleExpand table for course {0}".format(course_id)
+        print "--------------------------------------------------------------------------------"
+
+        # Grab when we start, to log later
+        tstart = datetime.now(UTC)
+
+        iterative_populate = True
+        if options['force']:
+            print "--------------------------------------------------------------------------------"
+            print "Full populate: Forced full populate"
+            print "--------------------------------------------------------------------------------"
+            iterative_populate = False
+
+        if iterative_populate:
+            # Get when this script was last run for this course
+            last_log_run = Log.objects.filter(script_id__exact=script_id, course_id__exact=course_id)
+
+            length = len(last_log_run)
+            print "--------------------------------------------------------------------------------"
+            if length > 0:
+                print "Iterative populate: Last log run", last_log_run[0].created
+            else:
+                print "Full populate: Can't find log of last run"
+                iterative_populate = False
+            print "--------------------------------------------------------------------------------"
+        
+        # If iterative populate, get all the problems that students have submitted an answer to for this course,
+        # since the last run
+        if iterative_populate:
+            sm_rows = StudentModule.objects.filter(course_id__exact=course_id, grade__isnull=False,
+                                                  module_type__exact="problem", modified__gte=last_log_run[0].created)
+        else:
+            sm_rows = StudentModule.objects.filter(course_id__exact=course_id, grade__isnull=False,
+                                                  module_type__exact="problem")
+
+        c_updated_rows = 0
+        # For each problem, get or create the corresponding StudentModuleExpand row
+        for sm in sm_rows:
+            sme, created = StudentModuleExpand.objects.get_or_create(student=sm.student, course_id=course_id,
+                                                                     module_state_key=sm.module_state_key,
+                                                                     student_module=sm)
+
+            # If the StudentModuleExpand row is new or the StudentModule row was
+            # more recently updated than the StudentModuleExpand row, fill in/update
+            # everything and save
+            if created or (sme.modified < sm.modified):
+                c_updated_rows += 1
+                sme.grade = sm.grade
+                sme.max_grade = sm.max_grade
+                state = json.loads(sm.state)
+                sme.attempts = state["attempts"]
+                sme.save()
+
+        c_all_rows = len(sm_rows)
+        print "--------------------------------------------------------------------------------"
+        print "Done! Updated/Created {0} queryable rows out of {1} from courseware_studentmodule".format(
+            c_updated_rows, c_all_rows)
+        print "--------------------------------------------------------------------------------"
+
+        # Save since everything finished successfully, log latest run.
+        q_log = Log(script_id=script_id, course_id=course_id, created=tstart)
+        q_log.save()
diff --git a/lms/djangoapps/queryable/migrations/0001_initial.py b/lms/djangoapps/queryable/migrations/0001_initial.py
new file mode 100644
index 000000000000..13bf9ea4654a
--- /dev/null
+++ b/lms/djangoapps/queryable/migrations/0001_initial.py
@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'StudentModuleExpand'
+        db.create_table('queryable_studentmoduleexpand', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('student_module', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['courseware.StudentModule'])),
+            ('attempts', self.gf('django.db.models.fields.IntegerField')(db_index=True, null=True, blank=True)),
+            ('module_type', self.gf('django.db.models.fields.CharField')(default='problem', max_length=32, db_index=True)),
+            ('module_state_key', self.gf('django.db.models.fields.CharField')(max_length=255, db_column='module_id', db_index=True)),
+            ('student', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('grade', self.gf('django.db.models.fields.FloatField')(db_index=True, null=True, blank=True)),
+            ('max_grade', self.gf('django.db.models.fields.FloatField')(null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable', ['StudentModuleExpand'])
+
+        # Adding unique constraint on 'StudentModuleExpand', fields ['student', 'module_state_key', 'course_id']
+        db.create_unique('queryable_studentmoduleexpand', ['student_id', 'module_id', 'course_id'])
+
+        # Adding model 'CourseGrade'
+        db.create_table('queryable_coursegrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('grade', self.gf('django.db.models.fields.CharField')(max_length=32, null=True, db_index=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable', ['CourseGrade'])
+
+        # Adding unique constraint on 'CourseGrade', fields ['user', 'course_id']
+        db.create_unique('queryable_coursegrade', ['user_id', 'course_id'])
+
+        # Adding model 'AssignmentTypeGrade'
+        db.create_table('queryable_assignmenttypegrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable', ['AssignmentTypeGrade'])
+
+        # Adding unique constraint on 'AssignmentTypeGrade', fields ['user', 'course_id', 'category']
+        db.create_unique('queryable_assignmenttypegrade', ['user_id', 'course_id', 'category'])
+
+        # Adding model 'AssignmentGrade'
+        db.create_table('queryable_assignmentgrade', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('category', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('percent', self.gf('django.db.models.fields.FloatField')(null=True, db_index=True)),
+            ('label', self.gf('django.db.models.fields.CharField')(max_length=32, db_index=True)),
+            ('detail', self.gf('django.db.models.fields.CharField')(max_length=255, null=True, blank=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, db_index=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, db_index=True, blank=True)),
+        ))
+        db.send_create_signal('queryable', ['AssignmentGrade'])
+
+        # Adding unique constraint on 'AssignmentGrade', fields ['user', 'course_id', 'label']
+        db.create_unique('queryable_assignmentgrade', ['user_id', 'course_id', 'label'])
+
+        # Adding model 'Log'
+        db.create_table('queryable_log', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('script_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+        ))
+        db.send_create_signal('queryable', ['Log'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'AssignmentGrade', fields ['user', 'course_id', 'label']
+        db.delete_unique('queryable_assignmentgrade', ['user_id', 'course_id', 'label'])
+
+        # Removing unique constraint on 'AssignmentTypeGrade', fields ['user', 'course_id', 'category']
+        db.delete_unique('queryable_assignmenttypegrade', ['user_id', 'course_id', 'category'])
+
+        # Removing unique constraint on 'CourseGrade', fields ['user', 'course_id']
+        db.delete_unique('queryable_coursegrade', ['user_id', 'course_id'])
+
+        # Removing unique constraint on 'StudentModuleExpand', fields ['student', 'module_state_key', 'course_id']
+        db.delete_unique('queryable_studentmoduleexpand', ['student_id', 'module_id', 'course_id'])
+
+        # Deleting model 'StudentModuleExpand'
+        db.delete_table('queryable_studentmoduleexpand')
+
+        # Deleting model 'CourseGrade'
+        db.delete_table('queryable_coursegrade')
+
+        # Deleting model 'AssignmentTypeGrade'
+        db.delete_table('queryable_assignmenttypegrade')
+
+        # Deleting model 'AssignmentGrade'
+        db.delete_table('queryable_assignmentgrade')
+
+        # Deleting model 'Log'
+        db.delete_table('queryable_log')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'courseware.studentmodule': {
+            'Meta': {'unique_together': "(('student', 'module_state_key', 'course_id'),)", 'object_name': 'StudentModule'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'done': ('django.db.models.fields.CharField', [], {'default': "'na'", 'max_length': '8', 'db_index': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_state_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_column': "'module_id'", 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'queryable.assignmentgrade': {
+            'Meta': {'unique_together': "(('user', 'course_id', 'label'),)", 'object_name': 'AssignmentGrade'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'detail': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'queryable.assignmenttypegrade': {
+            'Meta': {'unique_together': "(('user', 'course_id', 'category'),)", 'object_name': 'AssignmentTypeGrade'},
+            'category': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'queryable.coursegrade': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'CourseGrade'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'percent': ('django.db.models.fields.FloatField', [], {'null': 'True', 'db_index': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'queryable.log': {
+            'Meta': {'ordering': "['-created']", 'object_name': 'Log'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'script_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'})
+        },
+        'queryable.studentmoduleexpand': {
+            'Meta': {'unique_together': "(('student', 'module_state_key', 'course_id'),)", 'object_name': 'StudentModuleExpand'},
+            'attempts': ('django.db.models.fields.IntegerField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_state_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_column': "'module_id'", 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'student_module': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['courseware.StudentModule']"})
+        }
+    }
+
+    complete_apps = ['queryable']
\ No newline at end of file
diff --git a/lms/djangoapps/queryable/migrations/__init__.py b/lms/djangoapps/queryable/migrations/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/lms/djangoapps/queryable/models.py b/lms/djangoapps/queryable/models.py
new file mode 100644
index 000000000000..1f02b15d4061
--- /dev/null
+++ b/lms/djangoapps/queryable/models.py
@@ -0,0 +1,104 @@
+from django.contrib.auth.models import User
+from django.db import models
+from courseware.models import StudentModule
+
+class StudentModuleExpand(models.Model):
+    """
+    Expanded version of courseware's model StudentModule. This is only for
+    instances of module type 'problem'. Adds attribute 'attempts' that is pulled
+    out of the json in the state attribute.
+    """
+    
+    EXPAND_TYPES = {'problem'}
+
+    student_module = models.ForeignKey(StudentModule, db_index=True)
+
+    # The value mapped to 'attempts' in the json in state
+    attempts = models.IntegerField(null=True, blank=True, db_index=True)
+
+    # Values from StudentModule
+    module_type = models.CharField(max_length=32, default='problem', db_index=True)
+    module_state_key = models.CharField(max_length=255, db_index=True, db_column='module_id')
+    student = models.ForeignKey(User, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    class Meta:
+        unique_together = (('student', 'module_state_key', 'course_id'),)
+
+    grade = models.FloatField(null=True, blank=True, db_index=True)
+    max_grade = models.FloatField(null=True, blank=True)
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    modified = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class CourseGrade(models.Model):
+    """
+    Holds student's overall course grade as a percentage and letter grade (if letter grade present).
+    """
+
+    user = models.ForeignKey(User, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    percent = models.FloatField(db_index=True, null=True)
+    grade = models.CharField(max_length=32, db_index=True, null=True)
+
+    class Meta:
+        unique_together = (('user', 'course_id'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class AssignmentTypeGrade(models.Model):
+    """
+    Holds student's average grade for each assignment type per course.
+    """
+
+    user = models.ForeignKey(User, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    category = models.CharField(max_length=255, db_index=True)
+    percent = models.FloatField(db_index=True, null=True)
+
+    class Meta:
+        unique_together = (('user', 'course_id', 'category'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+
+class AssignmentGrade(models.Model):
+    """
+    Holds student's assignment grades per course.
+    """
+
+    user = models.ForeignKey(User, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+
+    category = models.CharField(max_length=255, db_index=True)
+    percent = models.FloatField(db_index=True, null=True)
+    label = models.CharField(max_length=32, db_index=True)
+    detail = models.CharField(max_length=255, blank=True, null=True)
+
+    class Meta:
+        unique_together = (('user', 'course_id', 'label'), )
+
+    created = models.DateTimeField(auto_now_add=True, db_index=True)
+    updated = models.DateTimeField(auto_now=True, db_index=True)
+
+
+class Log(models.Model):
+    """
+    Log of when a script in this django app was last run. Use to filter out students or rows that don't need to be
+    processed in the populate scripts and show instructors how fresh the data is.
+    """
+
+    script_id = models.CharField(max_length=255, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
+    created = models.DateTimeField(null=True, db_index=True)
+
+    class Meta:
+        ordering = ["-created"]
+        get_latest_by = "created"
diff --git a/lms/djangoapps/queryable/tests/test_populate_studentgrades.py b/lms/djangoapps/queryable/tests/test_populate_studentgrades.py
new file mode 100644
index 000000000000..56b482e3e45b
--- /dev/null
+++ b/lms/djangoapps/queryable/tests/test_populate_studentgrades.py
@@ -0,0 +1,734 @@
+import json
+from datetime import datetime
+from pytz import UTC
+from mock import Mock, patch
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from django.core.management import call_command
+
+from courseware import grades
+from courseware.tests.factories import StudentModuleFactory
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from student.tests.factories import UserFactory as StudentUserFactory
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+
+from queryable.models import Log, CourseGrade, AssignmentTypeGrade, AssignmentGrade
+from queryable.management.commands import populate_studentgrades
+
+
+class TestPopulateStudentGradesUpdateCourseGrade(TestCase):
+    """
+    Tests the helper fuction update_course_grade in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.course_grade = CourseGrade(percent=0.9, grade='A')
+        self.gradeset = {'percent' : 0.9, 'grade' : 'A'}
+
+
+    def test_no_update(self):
+        """
+        Values are the same, so no update
+        """
+        self.assertFalse(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+
+    def test_percents_not_equal(self):
+        """
+        Update because the percents don't equal
+        """
+        self.course_grade.percent = 1.0
+
+        self.assertTrue(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+
+    def test_different_grade(self):
+        """
+        Update because the grade is different
+        """
+        self.course_grade.grade = 'Foo'
+
+        self.assertTrue(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+
+    def test_grade_as_null(self):
+        """
+        Percent is the same and grade are both null, so no update
+        """
+        self.course_grade.grade = None
+        self.gradeset['grade'] = None
+
+        self.assertFalse(populate_studentgrades.update_course_grade(self.course_grade, self.gradeset))
+
+
+class TestPopulateStudentGradesGetAssignmentIndex(TestCase):
+    """
+    Tests the helper fuction get_assignment_index in the populate_studentgrades custom command
+    """
+
+    def test_simple(self):
+        """
+        Simple test if returns correct index.
+        """
+
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 3"),2)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 02"),1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 11"),10)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW 001"),0)
+
+
+    def test_no_index(self):
+        """
+        Test if returns -1 for badly formed input
+        """
+
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW Avg"),-1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW"),-1)
+        self.assertEquals(populate_studentgrades.get_assignment_index("HW "),-1)
+
+
+class TestPopulateStudentGradesGetStudentProblems(TestCase):
+    """
+    Tests the helper fuction get_student_problems in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student_module = StudentModuleFactory(
+            module_type='problem',
+            module_state_key='one',
+            grade=1,
+            max_grade=1,
+            )
+
+    def test_single_problem(self):
+        """
+        Test returns a single problem
+        """
+
+        problem_set = populate_studentgrades.get_student_problems(
+            self.student_module.course_id,
+            self.student_module.student,
+        )
+
+        self.assertEquals(len(problem_set),1)
+        self.assertEquals(problem_set[0], self.student_module.module_state_key)
+
+
+    def test_problem_with_no_submission(self):
+        """
+        Test to make sure only returns the problems with a submission.
+        """
+
+        student_module_no_submission = StudentModuleFactory(
+            course_id=self.student_module.course_id,
+            student=self.student_module.student,
+            module_type='problem',
+            module_state_key='no_submission',
+            grade=None,
+            max_grade=None,
+        )
+
+        problem_set = populate_studentgrades.get_student_problems(
+            self.student_module.course_id,
+            self.student_module.student,
+        )
+
+        self.assertEquals(len(problem_set),1)
+        self.assertEquals(problem_set[0], self.student_module.module_state_key)
+
+
+class TestPopulateStudentGradesAssignmentExistsAndHasProblems(TestCase):
+    """
+    Tests the helper fuction assignment_exists_and_has_problems in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.category = 'HW'
+        self.assignment_problems_map = {
+            self.category : [
+                ['cat_1_problem_id_1'],
+            ]
+        }
+
+
+    def test_simple(self):
+        """
+        Test where assignment does exist and has problems
+        """
+
+        self.assertTrue(populate_studentgrades.assignment_exists_and_has_problems(
+                self.assignment_problems_map,
+                self.category,
+                len(self.assignment_problems_map[self.category])-1,
+            ))
+
+
+    def test_assignment_exist_no_problems(self):
+        """
+        Test where assignment exists but has no problems
+        """
+
+        self.assignment_problems_map['Final'] = [[]]
+        
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems(
+                self.assignment_problems_map, 'Final', 0
+            ))
+
+
+    def test_negative_index(self):
+        """
+        Test handles negative indexes well by returning False
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems({},"",-1))
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems({},"",-5))
+
+
+    def test_non_existing_category(self):
+        """
+        Test handled a category that doesn't actually exist by returning False
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems({},"Foo",0))
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems(self.assignment_problems_map,"Foo",0))
+
+
+    def test_index_too_high(self):
+        """
+        Test that if the index is higher than the actual number of assignments
+        """
+
+        self.assertFalse(populate_studentgrades.assignment_exists_and_has_problems(
+                self.assignment_problems_map, self.category, len(self.assignment_problems_map[self.category])
+            ))
+
+
+class TestPopulateStudentGradesStudentDidProblems(TestCase):
+    """
+    Tests the helper fuction student_did_problems in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student_problems = ['cat_1_problem_1']
+
+    def test_student_did_do_problems(self):
+        """
+        Test where student did do some of the problems
+        """
+
+        self.assertTrue(populate_studentgrades.student_did_problems(self.student_problems, self.student_problems))
+
+        problem_set = list(self.student_problems)
+        problem_set.append('cat_2_problem_1')
+        self.assertTrue(populate_studentgrades.student_did_problems(self.student_problems, problem_set))
+
+
+    def test_student_did_not_do_problems(self):
+        """
+        Test where student didn't do any problems in the list
+        """
+
+        self.assertFalse(populate_studentgrades.student_did_problems(self.student_problems, []))
+        self.assertFalse(populate_studentgrades.student_did_problems([],self.student_problems))
+
+        problem_set = ['cat_1_problem_2']
+        self.assertFalse(populate_studentgrades.student_did_problems(self.student_problems, problem_set))
+
+
+class TestPopulateStudentGradesStoreCourseGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_course_grade_if_need in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.gradeset = {
+            'percent' : 1.0,
+            'grade' : 'A',
+        }
+        self.course_grade = CourseGrade(
+            user=self.student,
+            course_id=self.course_id,
+            percent=self.gradeset['percent'],
+            grade=self.gradeset['grade'],
+        )
+        self.course_grade.save()
+
+
+    def test_new_course_grade_store(self):
+        """
+        Test stores because it's a new CourseGrade
+        """
+
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course_id)),1)
+        student = StudentUserFactory()
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            student, self.course_id, self.gradeset
+        )
+        
+        self.assertTrue(return_value)
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course_id)),2)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.update_course_grade')
+    def test_update_store(self, mock_update_course_grade):
+        """
+        Test stores because update_course_grade returns True
+        """
+        mock_update_course_grade.return_value = True
+
+        updated_time = self.course_grade.updated
+
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            self.student, self.course_id, self.gradeset
+        )
+        
+        self.assertTrue(return_value)
+
+        course_grades = CourseGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+        )
+        self.assertEqual(len(course_grades),1)
+        self.assertNotEqual(updated_time, course_grades[0].updated)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.update_course_grade')
+    def test_no_update_no_store(self, mock_update_course_grade):
+        """
+        Test doesn't touch the row because it is not newly created and update_course_grade returns False
+        """
+        mock_update_course_grade.return_value = False
+
+        updated_time = self.course_grade.updated
+
+        return_value = populate_studentgrades.store_course_grade_if_need(
+            self.student, self.course_id, self.gradeset
+        )
+        
+        self.assertFalse(return_value)
+
+        course_grades = CourseGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+        )
+        self.assertEqual(len(course_grades),1)
+        self.assertEqual(updated_time, course_grades[0].updated)
+
+
+class TestPopulateStudentGradesStoreAssignmentTypeGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_assignment_type_grade_if_need in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.category = 'Homework'
+        self.percent = 1.0
+        self.assignment_type_grade = AssignmentTypeGrade(
+            user=self.student,
+            course_id=self.course_id,
+            category=self.category,
+            percent=self.percent,
+        )
+        self.assignment_type_grade.save()
+
+
+    def test_new_assignment_type_grade_store(self):
+        """
+        Test the function both stores the new assignment type grade and returns True meaning that it had
+        """
+
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course_id)),1)
+        return_value = populate_studentgrades.store_assignment_type_grade_if_need(
+            self.student, self.course_id, 'Foo 01', 1.0
+        )
+        
+        self.assertTrue(return_value)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course_id)),2)
+
+
+    def test_difference_percent_store(self):
+        """
+        Test updates the percent value when it is different
+        """
+
+        new_percent = self.percent-0.1
+        return_value = populate_studentgrades.store_assignment_type_grade_if_need(
+            self.student, self.course_id, self.category, new_percent
+        )
+        
+        self.assertTrue(return_value)
+
+        assignment_type_grades = AssignmentTypeGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+            category=self.category,
+        )
+        self.assertEqual(len(assignment_type_grades),1)
+        self.assertEqual(assignment_type_grades[0].percent, new_percent)
+
+
+    def test_same_percent_no_store(self):
+        """
+        Test does not touch row if the row exists and the precent is not different
+        """
+        updated_time = self.assignment_type_grade.updated
+
+        return_value = populate_studentgrades.store_assignment_type_grade_if_need(
+            self.student, self.course_id, self.category, self.percent
+        )
+
+        self.assertFalse(return_value)
+
+        assignment_type_grades = AssignmentTypeGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+            category=self.category,
+        )
+        self.assertEqual(len(assignment_type_grades),1)
+        self.assertEqual(assignment_type_grades[0].percent, self.percent)
+        self.assertEqual(assignment_type_grades[0].updated, updated_time)
+
+
+class TestPopulateStudentGradesStoreAssignmentGradeIfNeed(TestCase):
+    """
+    Tests the helper fuction store_assignment_grade_if_need in the populate_studentgrades custom command
+    """
+
+    def setUp(self):
+        self.student = StudentUserFactory()
+        self.course_id = 'test/test/test'
+        self.label = 'HW 01'
+        self.percent = 1.0
+        self.assignment_grade = AssignmentGrade(
+            user=self.student,
+            course_id=self.course_id,
+            label=self.label,
+            percent=self.percent,
+        )
+        self.assignment_grade.save()
+
+
+    def test_new_assignment_grade_store(self):
+        """
+        Test the function both stores the new assignment grade and returns True meaning that it had
+        """
+
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course_id)),1)
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, 'Foo 01', 1.0
+        )
+        
+        self.assertTrue(return_value)
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course_id)),2)
+
+
+    def test_difference_percent_store(self):
+        """
+        Test updates the percent value when it is different
+        """
+
+        new_percent = self.percent-0.1
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, self.label, new_percent
+        )
+        
+        self.assertTrue(return_value)
+
+        assignment_grades = AssignmentGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+            label=self.label,
+        )
+        self.assertEqual(len(assignment_grades),1)
+        self.assertEqual(assignment_grades[0].percent, new_percent)
+
+
+    def test_same_percent_no_store(self):
+        """
+        Test does not touch row if the row exists and the precent is not different
+        """
+        updated_time = self.assignment_grade.updated
+
+        return_value = populate_studentgrades.store_assignment_grade_if_need(
+            self.student, self.course_id, self.label, self.percent
+        )
+
+        self.assertFalse(return_value)
+
+        assignment_grades = AssignmentGrade.objects.filter(
+            course_id__exact=self.course_id,
+            user=self.student,
+            label=self.label,
+        )
+        self.assertEqual(len(assignment_grades),1)
+        self.assertEqual(assignment_grades[0].percent, self.percent)
+        self.assertEqual(assignment_grades[0].updated, updated_time)
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestPopulateStudentGradesCommand(TestCase):
+
+    def create_studentmodule(self):
+        """
+        Creates a StudentModule. This can't be in setUp because some functions can't have one in the database.
+        """
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+
+
+    def create_log_entry(self):
+        """
+        Adds a queryable log entry to the database
+        """
+        log = Log(script_id=self.script_id, course_id=self.course.id, created=datetime.now(UTC))
+        log.save()
+
+
+    def setUp(self):
+        self.command = 'populate_studentgrades'
+        self.script_id = 'studentgrades'
+        self.course = CourseFactory.create()
+        self.category = 'Homework'
+        self.gradeset = {
+            'percent' : 1.0,
+            'grade' : 'A',
+            'section_breakdown' : [
+                {'category':self.category, 'label':'HW Avg', 'percent':1.0, 'prominent':True},
+                {'category':self.category, 'label':'HW 01', 'percent':1.0},
+            ],
+        }
+        # Make sure these are correct with the above gradeset
+        self.assignment_type_index = 0
+        self.assignment_index = 1
+
+
+    def test_missing_input(self):
+        """
+        Fails safely when not given enough input
+        """
+        try:
+            call_command(self.command)
+            self.assertTrue(True)
+        except:
+            self.assertTrue(False)
+
+
+    def test_just_logs_if_empty_course(self):
+        """
+        If the course has nothing in it, just logs the run in the log table.
+        """
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 1)
+        self.assertEqual(len(CourseGrade.objects.filter(course_id__exact=self.course.id)), 0)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(course_id__exact=self.course.id)), 0)
+        self.assertEqual(len(AssignmentGrade.objects.filter(course_id__exact=self.course.id)), 0)
+
+
+    @patch('courseware.grades.grade')
+    def test_force_update(self, mock_grade):
+        """
+        Even if there is a log entry for incremental update, force a full update
+
+        This may be done because something happened in the last update.
+        """
+        mock_grade.return_value = self.gradeset
+
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+
+        self.create_log_entry()
+
+        call_command(self.command, self.course.id, force=True)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course.id)), 2)
+        self.assertEqual(len(CourseGrade.objects.filter(user=sm.student, course_id__exact=self.course.id)), 1)
+        self.assertEqual(len(AssignmentTypeGrade.objects.filter(
+                    user=sm.student, course_id__exact=self.course.id, category=self.category
+                )), 1)
+        self.assertEqual(len(AssignmentGrade.objects.filter(
+                    user=sm.student,
+                    course_id__exact=self.course.id,
+                    label=self.gradeset['section_breakdown'][self.assignment_index]['label'],
+                )), 1)
+
+        
+    @patch('courseware.grades.grade')
+    def test_incremental_update_if_log_exists(self, mock_grade):
+        """
+        Make sure it uses the log entry if it exists and we aren't forcing a full update
+        """
+        mock_grade.return_value = self.gradeset
+
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+        sm.student.last_name = "Student1"
+        sm.student.save()
+
+        self.create_log_entry()
+
+        # Create a StudentModule that is after the log entry, different name
+        sm = StudentModuleFactory(
+            course_id=self.course.id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+        sm.student.last_name = "Student2"
+        sm.student.save()
+
+        call_command(self.command, self.course.id)
+
+        self.assertEqual(mock_grade.call_count, 1)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.store_course_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_store_course_grade(self, mock_grade, mock_method):
+        """
+        Calls store_course_grade_if_need for all students
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_method.call_count, 1)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.store_assignment_type_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_store_assignment_type_grade(self, mock_grade, mock_method):
+        """
+        Calls store_assignment_type_grade_if_need when such a section exists
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_method.call_count, 1)
+
+
+
+    @patch('queryable.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_store_assignment_grade_percent_not_zero(self, mock_grade, mock_method):
+        """
+        Calls store_assignment_grade_if_need when the percent for that assignment is not zero
+        """
+        mock_grade.return_value = self.gradeset
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_method.call_count, 1)
+
+
+
+    @patch('queryable.management.commands.populate_studentgrades.get_assignment_index')
+    @patch('queryable.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_bad_index(self, mock_grade, mock_method, mock_assign_index):
+        """
+        Does not call store_assignment_grade_if_need when the percent is zero because get_assignment_index returns a
+        negative number.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_index.return_value = -1
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_grade.call_count, 1)
+        self.assertEqual(mock_method.call_count, 0)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.get_student_problems')
+    @patch('queryable.management.commands.populate_studentgrades.assignment_exists_and_has_problems')
+    @patch('queryable.util.get_assignment_to_problem_map')
+    @patch('queryable.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_no_student_problems(self, mock_grade, mock_method, mock_assign_problem_map,\
+                                                                   mock_assign_exists, mock_student_problems):
+        """
+        Does not call store_assignment_grade_if_need when the percent is zero because the student did not submit
+        answers to any problems in that assignment.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_problem_map.return_value = {
+            self.gradeset['section_breakdown'][self.assignment_index]['category'] : [[]]
+        }
+
+        mock_assign_exists.return_value = True
+
+        mock_student_problems.return_value = []
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_method.call_count, 0)
+
+
+    @patch('queryable.management.commands.populate_studentgrades.get_student_problems')
+    @patch('queryable.management.commands.populate_studentgrades.assignment_exists_and_has_problems')
+    @patch('queryable.util.get_assignment_to_problem_map')
+    @patch('queryable.management.commands.populate_studentgrades.store_assignment_grade_if_need')
+    @patch('courseware.grades.grade')
+    def test_assignment_grade_percent_zero_has_student_problems(self, mock_grade, mock_method, mock_assign_problem_map,\
+                                                                   mock_assign_exists, mock_student_problems):
+        """
+        Calls store_assignment_grade_if_need when the percent is zero because the student did submit answers to
+        problems in that assignment.
+        """
+        self.gradeset['section_breakdown'][self.assignment_index]['percent'] = 0.0
+        mock_grade.return_value = self.gradeset
+
+        mock_assign_problem_map.return_value = {
+            self.gradeset['section_breakdown'][self.assignment_index]['category'] : [['problem_1']]
+        }
+
+        mock_assign_exists.return_value = True
+
+        mock_student_problems.return_value = ['problem_1']
+
+        self.create_studentmodule()
+        
+        call_command(self.command, self.course.id)
+        
+        self.assertEqual(mock_method.call_count, 1)
+
+
diff --git a/lms/djangoapps/queryable/tests/test_populate_studentmoduleexpand.py b/lms/djangoapps/queryable/tests/test_populate_studentmoduleexpand.py
new file mode 100644
index 000000000000..a2043f22396c
--- /dev/null
+++ b/lms/djangoapps/queryable/tests/test_populate_studentmoduleexpand.py
@@ -0,0 +1,186 @@
+import json
+from datetime import datetime
+from pytz import UTC
+from StringIO import StringIO
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from django.core import management
+
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+
+from courseware.tests.factories import StudentModuleFactory
+from queryable.models import Log, StudentModuleExpand
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestPopulateStudentModuleExpand(TestCase):
+
+    def setUp(self):
+        self.command = 'populate_studentmoduleexpand'
+        self.script_id = "studentmoduleexpand"
+        self.course_id = 'test/test/test'
+
+
+    def test_missing_input(self):
+        """
+        Fails safely when not given enough input
+        """
+        try:
+            management.call_command(self.command)
+            self.assertTrue(True)
+        except:
+            self.assertTrue(False)
+
+
+    def test_just_logs_if_empty_course(self):
+        """
+        If the course has nothing in it, just logs the run in the log table
+        """
+        management.call_command(self.command, self.course_id)
+
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course_id)), 1)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)), 0)
+
+
+    def test_force_update(self):
+        """
+        Even if there is a log entry for incremental update, force a full update
+
+        This may be done because something happened in the last update.
+        """
+        
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course_id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+        
+        # Create the log entry
+        log = Log(script_id=self.script_id, course_id=self.course_id, created=datetime.now(UTC))
+        log.save()
+
+        # Create a StudentModuleExpand that is after the log entry and has a different attempts value
+        sme = StudentModuleExpand(
+            course_id=self.course_id,
+            module_state_key=sm.module_state_key,
+            student_module=sm,
+            attempts=0,
+        )
+
+        # Call command with the -f flag
+        management.call_command(self.command, self.course_id, force=True)
+
+        # Check to see if new rows have been added
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course_id)), 2)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)), 1)
+        self.assertEqual(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)[0].attempts, 1)
+
+
+    def test_incremental_update_if_log_exists(self):
+        """
+        Make sure it uses the log entry if it exists and we aren't forcing a full update
+        """
+        # Create a StudentModule that is before the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course_id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+        
+        # Create the log entry
+        log = Log(script_id=self.script_id, course_id=self.course_id, created=datetime.now(UTC))
+        log.save()
+
+        # Create a StudentModule that is after the log entry
+        sm = StudentModuleFactory(
+            course_id=self.course_id,
+            module_type='problem',
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':1}),
+        )
+        
+        # Call command
+        management.call_command(self.command, self.course_id)
+
+        # Check to see if new row has been added to log
+        self.assertEqual(len(Log.objects.filter(script_id__exact=self.script_id, course_id__exact=self.course_id)), 2)
+
+        # Even though there are two studentmodules only one row should be created
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)), 1)
+
+
+    def test_update_only_if_row_modified(self):
+        """
+        Test populate does not update a row if it is not necessary
+
+        For example the problem may have a more recent modified date but the attempts value has not changed.
+        """
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)), 0)
+
+        # Create a StudentModule
+        sm1 = StudentModuleFactory(
+            course_id=self.course_id,
+            module_type='problem',
+            module_state_key=1,
+            grade=1,
+            max_grade=1,
+        )
+        # Create a StudentModuleExpand
+        sme1 = StudentModuleExpand(
+            course_id=self.course_id,
+            student_module=sm1,
+            module_state_key=sm1.module_state_key,
+            student=sm1.student,
+            attempts=0,
+        )
+        sme1.save()
+
+        # Touch the StudentModule row so it has a later modified time
+        sm1.state=json.dumps({'attempts':1})
+        sm1.save()
+
+        # Create a StudentModule
+        sm2 = StudentModuleFactory(
+            course_id=self.course_id,
+            module_type='problem',
+            module_state_key=2,
+            grade=1,
+            max_grade=1,
+            state=json.dumps({'attempts':2}),
+        )
+        # Create a StudentModuleExpand that has the same attempts value
+        sme2 = StudentModuleExpand(
+            course_id=self.course_id,
+            student_module=sm2,
+            module_state_key=sm2.module_state_key,
+            student=sm2.student,
+            attempts=2,
+        )
+        sme2.save()
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(course_id__exact=self.course_id)), 2)
+
+        # Call command
+        management.call_command(self.command, self.course_id)
+
+        self.assertEqual(len(StudentModuleExpand.objects.filter(
+                    course_id__exact=self.course_id, module_state_key__exact=sme1.module_state_key
+                )), 1)
+        self.assertEqual(len(StudentModuleExpand.objects.filter(
+                    course_id__exact=self.course_id, module_state_key__exact=sme2.module_state_key
+                )), 1)
+
+        self.assertEqual(StudentModuleExpand.objects.filter(
+                course_id__exact=self.course_id, module_state_key__exact=sme1.module_state_key
+            )[0].attempts, 1)
+        self.assertEqual(StudentModuleExpand.objects.filter(
+                course_id__exact=self.course_id, module_state_key__exact=sme2.module_state_key
+            )[0].attempts, 2)
+
diff --git a/lms/djangoapps/queryable/tests/test_util.py b/lms/djangoapps/queryable/tests/test_util.py
new file mode 100644
index 000000000000..e07fd81442d4
--- /dev/null
+++ b/lms/djangoapps/queryable/tests/test_util.py
@@ -0,0 +1,256 @@
+from django.test import TestCase
+from django.test.utils import override_settings
+
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+
+from queryable import util
+
+
+class TestUtilApproxEqual(TestCase):
+    """
+    Check the approx_equal function
+    """
+
+    def test_default_tolerance(self):
+        """
+        Check that function with default tolerance
+        """
+        self.assertTrue(util.approx_equal(1.00001,1.0))
+        self.assertTrue(util.approx_equal(1.0,1.00001))
+
+        self.assertFalse(util.approx_equal(1.0,2.0))
+        self.assertFalse(util.approx_equal(1.0,1.0002))
+
+
+    def test_smaller_default_tolerance(self):
+        """
+        Set tolerance smaller than default and check if still correct
+        """
+        
+        self.assertTrue(util.approx_equal(1.0,1.0,1))
+        self.assertTrue(util.approx_equal(1.0,1.000001,0.000001))
+
+
+    def test_bigger_default_tolerance(self):
+        """
+        Set tolerance bigger than default and check if still correct
+        """
+
+        self.assertFalse(util.approx_equal(1.0,2.0,0.75))
+        self.assertFalse(util.approx_equal(2.0,1.0,0.75))
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+class TestUtilGetAssignmentToProblemMap(TestCase):
+    """
+    Tests the get_assignemnt_to_problem_map
+    """
+
+    def setUp(self):
+        self.course = CourseFactory.create()
+
+
+    def test_empty_course(self):
+        """
+        Test for course with nothing in it
+        """
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+        
+        self.assertEqual(problems_map, {})
+
+
+    def test_single_assignment(self):
+        """
+        Test returns the problems for a course with a single assignment
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential",
+        )
+        subsection_metadata = own_metadata(subsection)
+        subsection_metadata['graded'] = True
+        subsection_metadata['format'] = "Homework"
+        modulestore().update_metadata(subsection.location, subsection_metadata)
+
+        unit = ItemFactory.create(
+            parent_location=subsection.location.url(),
+            category="vertical",
+        )
+
+        problem1 = ItemFactory.create(
+            parent_location=unit.location.url(),
+            category="problem",
+        )
+        problem2 = ItemFactory.create(
+            parent_location=unit.location.url(),
+            category="problem",
+        )
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {
+            'Homework' : [
+                [problem1.location.url(), problem2.location.url()],
+                ],
+            }
+
+        self.assertEqual(problems_map,answer)
+
+
+    def test_two_assignments_same_type(self):
+        """
+        Test if has two assignments
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['graded'] = True
+        subsection_metadata2['format'] = "Homework"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {
+            'Homework' : [
+                [problem1.location.url()],
+                [problem2.location.url()],
+                ],
+            }
+
+        self.assertEqual(problems_map,answer)
+
+
+    def test_two_assignments_different_types(self):
+        """
+        Creates two assignments of different types
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['graded'] = True
+        subsection_metadata2['format'] = "Quiz"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {
+            'Homework' : [
+                [problem1.location.url()],
+                ],
+            'Quiz' : [
+                [problem2.location.url()],
+                ],
+            }
+
+        self.assertEqual(problems_map,answer)
+
+
+
+    def test_return_only_graded_subsections(self):
+        """
+        Make sure only returns problems and assignments that are graded
+        """
+        section = ItemFactory.create(
+            parent_location=self.course.location.url(),
+            category="chapter")
+
+        subsection1 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata1 = own_metadata(subsection1)
+        subsection_metadata1['graded'] = True
+        subsection_metadata1['format'] = "Homework"
+        modulestore().update_metadata(subsection1.location, subsection_metadata1)
+
+        unit1 = ItemFactory.create(
+            parent_location=subsection1.location.url(),
+            category="vertical")
+        problem1 = ItemFactory.create(
+            parent_location=unit1.location.url(),
+            category="problem")
+
+        subsection2 = ItemFactory.create(
+            parent_location=section.location.url(),
+            category="sequential")
+        subsection_metadata2 = own_metadata(subsection2)
+        subsection_metadata2['format'] = "Quiz"
+        modulestore().update_metadata(subsection2.location, subsection_metadata2)
+
+        unit2 = ItemFactory.create(
+            parent_location=subsection2.location.url(),
+            category="vertical")
+        problem2 = ItemFactory.create(
+            parent_location=unit2.location.url(),
+            category="problem")
+
+        problems_map = util.get_assignment_to_problem_map(self.course.id)
+
+        answer = {
+            'Homework' : [
+                [problem1.location.url()],
+                ],
+            }
+
+        self.assertEqual(problems_map,answer)
+
diff --git a/lms/djangoapps/queryable/util.py b/lms/djangoapps/queryable/util.py
new file mode 100644
index 000000000000..788c01d38809
--- /dev/null
+++ b/lms/djangoapps/queryable/util.py
@@ -0,0 +1,40 @@
+# ======== Utility functions to help with population ===================================================================
+
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import own_metadata
+
+def get_assignment_to_problem_map(course_id):
+    """
+    Returns a dictionary with assignment types/categories as keys and the value is an array of arrays. Each inner array
+    holds problem ids for an assignment. The arrays are ordered in the outer array as they are seen in the course, which
+    is how they are numbered in a student's progress page.
+    """
+
+    course = modulestore().get_instance(course_id, CourseDescriptor.id_to_location(course_id), depth=4)
+
+    assignment_problems_map = {}
+    for section in course.get_children():
+        for subsection in section.get_children():
+            subsection_metadata = own_metadata(subsection)
+            if ('graded' in subsection_metadata) and subsection_metadata['graded']:
+                category = subsection_metadata['format']
+                if category not in assignment_problems_map:
+                    assignment_problems_map[category] = []
+                    
+                problems = []
+                for unit in subsection.get_children():
+                    for child in unit.get_children():
+                        if child.location.category == 'problem':
+                            problems.append(child.location.url())
+                
+                assignment_problems_map[category].append(problems)
+
+    return assignment_problems_map
+
+
+def approx_equal(a,b,tolerance=0.0001):
+    """
+    Checks if a and b are at most the specified tolerance away from each other.
+    """
+    return abs(a-b) <= tolerance;
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index a8a9778ac4ce..589f6ae16fb2 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -260,6 +260,9 @@
 ########################## USER API ########################
 EDX_API_KEY = ''
 
+########################## QUERYABLE TABLES ########################
+INSTALLED_APPS += ('queryable',)
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 225947d28cf4..c560105e7585 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -211,3 +211,6 @@
 
 import openid.oidutil
 openid.oidutil.log = lambda message, level=0: None
+
+### QUERYABLE APP ###
+INSTALLED_APPS += ('queryable',)