Unicode changes to support R to L script

cms/djangoapps/contentstore/utils.py

@@ -143,7 +143,7 @@ def get_lms_link_for_item(location, preview=False, course_id=None):
         else:
             lms_base = settings.LMS_BASE
 
-        lms_link = "//{lms_base}/courses/{course_id}/jump_to/{location}".format(
+        lms_link = u"//{lms_base}/courses/{course_id}/jump_to/{location}".format(
             lms_base=lms_base,
             course_id=course_id,
             location=Location(location)
@@ -179,7 +179,7 @@ def get_lms_link_for_about_page(location):
         about_base = None
 
     if about_base is not None:
-        lms_link = "//{about_base_url}/courses/{course_id}/about".format(
+        lms_link = u"//{about_base_url}/courses/{course_id}/about".format(
             about_base_url=about_base,
             course_id=Location(location).course_id
         )

cms/djangoapps/contentstore/views/component.py

@@ -267,8 +267,7 @@ def unit_handler(request, tag=None, package_id=None, branch=None, version_guid=N
         preview_lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE')
 
         preview_lms_link = (
-            '//{preview_lms_base}/courses/{org}/{course}/'
-            '{course_name}/courseware/{section}/{subsection}/{index}'
+            u'//{preview_lms_base}/courses/{org}/{course}/{course_name}/courseware/{section}/{subsection}/{index}'
         ).format(
             preview_lms_base=preview_lms_base,
             lms_base=settings.LMS_BASE,

cms/djangoapps/contentstore/views/course.py

@@ -251,7 +251,7 @@ def create_new_course(request):
     run = request.json.get('run')
 
     try:
-        dest_location = Location('i4x', org, number, 'course', run)
+        dest_location = Location(u'i4x', org, number, u'course', run)
     except InvalidLocationError as error:
         return JsonResponse({
             "ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(
@@ -286,8 +286,10 @@ def create_new_course(request):
     course_search_location = bson.son.SON({
         '_id.tag': 'i4x',
         # cannot pass regex to Location constructor; thus this hack
-        '_id.org': re.compile('^{}$'.format(dest_location.org), re.IGNORECASE),
-        '_id.course': re.compile('^{}$'.format(dest_location.course), re.IGNORECASE),
+        # pylint: disable=E1101
+        '_id.org': re.compile(u'^{}$'.format(dest_location.org), re.IGNORECASE | re.UNICODE),
+        # pylint: disable=E1101
+        '_id.course': re.compile(u'^{}$'.format(dest_location.course), re.IGNORECASE | re.UNICODE),
         '_id.category': 'course',
     })
     courses = modulestore().collection.find(course_search_location, fields=('_id'))

cms/djangoapps/contentstore/views/preview.py

@@ -41,7 +41,7 @@ def handler_prefix(block, handler='', suffix=''):
     Trailing `/`s are removed from the returned url.
     """
     return reverse('preview_handler', kwargs={
-        'usage_id': quote_slashes(str(block.scope_ids.usage_id)),
+        'usage_id': quote_slashes(unicode(block.scope_ids.usage_id).encode('utf-8')),
         'handler': handler,
         'suffix': suffix,
     }).rstrip('/?')

cms/static/js/index.js

@@ -84,8 +84,8 @@ require(["domReady", "jquery", "underscore", "js/utils/cancel_on_escape"],
                 if (required) {
                     return required;
                 }
-                if (item !== encodeURIComponent(item)) {
-                    return gettext('Please do not use any spaces or special characters in this field.');
+                if (/\s/g.test(item)) {
+                    return gettext('Please do not use any spaces in this field.');
                 }
                 return '';
             };

cms/templates/unit.html

@@ -164,7 +164,7 @@ <h4 class="header">${_("Unit Settings")}</h4>
               % endif
               ${_("with the subsection {link_start}{name}{link_end}").format(
                   name=subsection.display_name_with_default,
-                  link_start='<a href="{url}">'.format(url=subsection_url),
+                  link_start=u'<a href="{url}">'.format(url=subsection_url),
                   link_end='</a>',
                 )}
               </p>

common/djangoapps/cache_toolbox/core.py

@@ -110,12 +110,12 @@ def instance_key(model, instance_or_pk):
 
 
 def set_cached_content(content):
-    cache.set(str(content.location), content)
+    cache.set(unicode(content.location).encode("utf-8"), content)
 
 
 def get_cached_content(location):
-    return cache.get(str(location))
+    return cache.get(unicode(location).encode("utf-8"))
 
 
 def del_cached_content(location):
-    cache.delete(str(location))
+    cache.delete(unicode(location).encode("utf-8"))

common/djangoapps/course_groups/cohorts.py

@@ -77,7 +77,7 @@ def is_commentable_cohorted(course_id, commentable_id):
         # inline discussions are cohorted by default
         ans = True
 
-    log.debug("is_commentable_cohorted({0}, {1}) = {2}".format(course_id,
+    log.debug(u"is_commentable_cohorted({0}, {1}) = {2}".format(course_id,
                                                                commentable_id,
                                                                ans))
     return ans

common/djangoapps/static_replace/__init__.py

@@ -19,7 +19,7 @@ def _url_replace_regex(prefix):
     To anyone contemplating making this more complicated:
     http://xkcd.com/1171/
     """
-    return r"""
+    return ur"""
         (?x)                      # flags=re.VERBOSE
         (?P<quote>\\?['"])        # the opening quotes
         (?P<prefix>{prefix})      # the prefix
@@ -152,7 +152,7 @@ def replace_static_url(match):
         return "".join([quote, url, quote])
 
     return re.sub(
-        _url_replace_regex('(?:{static_url}|/static/)(?!{data_dir})'.format(
+        _url_replace_regex(u'(?:{static_url}|/static/)(?!{data_dir})'.format(
             static_url=settings.STATIC_URL,
             data_dir=static_asset_path or data_directory
         )),

common/djangoapps/student/roles.py

@@ -159,30 +159,30 @@ def __init__(self, role, location, course_context=None):
 
         if isinstance(self.location, Location):
             try:
-                groupnames.append('{0}_{1}'.format(role, self.location.course_id))
+                groupnames.append(u'{0}_{1}'.format(role, self.location.course_id))
                 course_context = self.location.course_id  # course_id is valid for translation
             except InvalidLocationError:  # will occur on old locations where location is not of category course
                 if course_context is None:
                     raise CourseContextRequired()
                 else:
-                    groupnames.append('{0}_{1}'.format(role, course_context))
+                    groupnames.append(u'{0}_{1}'.format(role, course_context))
             try:
                 locator = loc_mapper().translate_location_to_course_locator(course_context, self.location)
-                groupnames.append('{0}_{1}'.format(role, locator.package_id))
+                groupnames.append(u'{0}_{1}'.format(role, locator.package_id))
             except (InvalidLocationError, ItemNotFoundError):
                 # if it's never been mapped, the auth won't be via the Locator syntax
                 pass
             # least preferred legacy role_course format
-            groupnames.append('{0}_{1}'.format(role, self.location.course))
+            groupnames.append(u'{0}_{1}'.format(role, self.location.course))  # pylint: disable=E1101, E1103
         elif isinstance(self.location, CourseLocator):
-            groupnames.append('{0}_{1}'.format(role, self.location.package_id))
+            groupnames.append(u'{0}_{1}'.format(role, self.location.package_id))
             # handle old Location syntax
             old_location = loc_mapper().translate_locator_to_location(self.location, get_course=True)
             if old_location:
                 # the slashified version of the course_id (myu/mycourse/myrun)
-                groupnames.append('{0}_{1}'.format(role, old_location.course_id))
+                groupnames.append(u'{0}_{1}'.format(role, old_location.course_id))
                 # add the least desirable but sometimes occurring format.
-                groupnames.append('{0}_{1}'.format(role, old_location.course))
+                groupnames.append(u'{0}_{1}'.format(role, old_location.course))  # pylint: disable=E1101, E1103
 
         super(CourseRole, self).__init__(groupnames)
 
@@ -193,12 +193,13 @@ class OrgRole(GroupBasedRole):
     """
     def __init__(self, role, location):
         location = Location(location)
-        super(OrgRole, self).__init__(['{}_{}'.format(role, location.org)])
+        super(OrgRole, self).__init__([u'{}_{}'.format(role, location.org)])
 
 
 class CourseStaffRole(CourseRole):
     """A Staff member of a course"""
     ROLE = 'staff'
+
     def __init__(self, *args, **kwargs):
         super(CourseStaffRole, self).__init__(self.ROLE, *args, **kwargs)
 

common/djangoapps/student/views.py

@@ -353,7 +353,7 @@ def dashboard(request):
 
             course_enrollment_pairs.append((course, enrollment))
         except ItemNotFoundError:
-            log.error("User {0} enrolled in non-existent course {1}"
+            log.error(u"User {0} enrolled in non-existent course {1}"
                       .format(user.username, enrollment.course_id))
 
     course_optouts = Optout.objects.filter(user=user).values_list('course_id', flat=True)
@@ -495,9 +495,9 @@ def change_enrollment(request):
         org, course_num, run = course_id.split("/")
         dog_stats_api.increment(
             "common.student.enrollment",
-            tags=["org:{0}".format(org),
-                  "course:{0}".format(course_num),
-                  "run:{0}".format(run)]
+            tags=[u"org:{0}".format(org),
+                  u"course:{0}".format(course_num),
+                  u"run:{0}".format(run)]
         )
 
         CourseEnrollment.enroll(user, course.id, mode=current_mode.slug)

common/lib/capa/capa/capa_problem.py

@@ -372,7 +372,7 @@ def _grade_answers(self, student_answers):
             # TODO: figure out where to get file submissions when rescoring.
             if 'filesubmission' in responder.allowed_inputfields and student_answers is None:
                 _ = self.capa_system.i18n.ugettext
-                raise Exception(_("Cannot rescore problems with possible file submissions"))
+                raise Exception(_(u"Cannot rescore problems with possible file submissions"))
 
             # use 'student_answers' only if it is provided, and if it might contain a file
             # submission that would not exist in the persisted "student_answers".

common/lib/capa/capa/correctmap.py

@@ -50,7 +50,7 @@ def set(
     ):
 
         if answer_id is not None:
-            self.cmap[str(answer_id)] = {
+            self.cmap[answer_id] = {
                 'correctness': correctness,
                 'npoints': npoints,
                 'msg': msg,

common/lib/capa/capa/xqueue_interface.py

@@ -62,7 +62,7 @@ class XQueueInterface(object):
     """
 
     def __init__(self, url, django_auth, requests_auth=None):
-        self.url = url
+        self.url = unicode(url)
         self.auth = django_auth
         self.session = requests.Session()
         self.session.auth = requests_auth

common/lib/xmodule/xmodule/contentstore/content.py

@@ -34,7 +34,9 @@ def is_thumbnail(self):
 
     @staticmethod
     def generate_thumbnail_name(original_name):
-        return ('{0}' + XASSET_THUMBNAIL_TAIL_NAME).format(os.path.splitext(original_name)[0])
+        return u"{name_root}{extension}".format(
+            name_root=os.path.splitext(original_name)[0],
+            extension=XASSET_THUMBNAIL_TAIL_NAME,)
 
     @staticmethod
     def compute_location(org, course, name, revision=None, is_thumbnail=False):
@@ -64,7 +66,7 @@ def is_c4x_path(path_string):
         """
         Returns a boolean if a path is believed to be a c4x link based on the leading element
         """
-        return path_string.startswith('/{0}/'.format(XASSET_LOCATION_TAG))
+        return path_string.startswith(u'/{0}/'.format(XASSET_LOCATION_TAG))
 
     @staticmethod
     def renamespace_c4x_path(path_string, target_location):
@@ -86,14 +88,14 @@ def get_static_path_from_location(location):
         the actual /c4x/... path which the client needs to reference static content
         """
         if location is not None:
-            return "/static/{name}".format(**location.dict())
+            return u"/static/{name}".format(**location.dict())
         else:
             return None
 
     @staticmethod
     def get_base_url_path_for_course_assets(loc):
         if loc is not None:
-            return "/c4x/{org}/{course}/asset".format(**loc.dict())
+            return u"/c4x/{org}/{course}/asset".format(**loc.dict())
 
     @staticmethod
     def get_id_from_location(location):
@@ -237,6 +239,6 @@ def generate_thumbnail(self, content, tempfile_path=None):
 
             except Exception, e:
                 # log and continue as thumbnails are generally considered as optional
-                logging.exception("Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(e)))
+                logging.exception(u"Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(e)))
 
         return thumbnail_content, thumbnail_file_location

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -30,12 +30,12 @@
 
 # TODO (cpennington): We should decide whether we want to expand the
 # list of valid characters in a location
-INVALID_CHARS = re.compile(r"[^\w.-]")
+INVALID_CHARS = re.compile(r"[^\w.%-]", re.UNICODE)
 # Names are allowed to have colons.
-INVALID_CHARS_NAME = re.compile(r"[^\w.:-]")
+INVALID_CHARS_NAME = re.compile(r"[^\w.:%-]", re.UNICODE)
 
 # html ids can contain word chars and dashes
-INVALID_HTML_CHARS = re.compile(r"[^\w-]")
+INVALID_HTML_CHARS = re.compile(r"[^\w-]", re.UNICODE)
 
 _LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
 
@@ -186,14 +186,14 @@ def check_list(list_):
         elif isinstance(location, basestring):
             match = URL_RE.match(location)
             if match is None:
-                log.debug("location %r doesn't match URL", location)
+                log.debug(u"location %r doesn't match URL", location)
                 raise InvalidLocationError(location)
             groups = match.groupdict()
             check_dict(groups)
             return _LocationBase.__new__(_cls, **groups)
         elif isinstance(location, (list, tuple)):
             if len(location) not in (5, 6):
-                log.debug('location has wrong length')
+                log.debug(u'location has wrong length')
                 raise InvalidLocationError(location)
 
             if len(location) == 5:
@@ -216,17 +216,17 @@ def url(self):
         """
         Return a string containing the URL for this location
         """
-        url = "{0.tag}://{0.org}/{0.course}/{0.category}/{0.name}".format(self)
+        url = u"{0.tag}://{0.org}/{0.course}/{0.category}/{0.name}".format(self)
         if self.revision:
-            url += "@" + self.revision
+            url += u"@{rev}".format(rev=self.revision)  # pylint: disable=E1101
         return url
 
     def html_id(self):
         """
         Return a string with a version of the location that is safe for use in
         html id attributes
         """
-        id_string = "-".join(str(v) for v in self.list() if v is not None)
+        id_string = u"-".join(v for v in self.list() if v is not None)
         return Location.clean_for_html(id_string)
 
     def dict(self):
@@ -240,6 +240,9 @@ def list(self):
         return list(self)
 
     def __str__(self):
+        return str(self.url().encode("utf-8"))
+
+    def __unicode__(self):
         return self.url()
 
     def __repr__(self):
@@ -254,7 +257,7 @@ def course_id(self):
         Throws an InvalidLocationError is this location does not represent a course.
         """
         if self.category != 'course':
-            raise InvalidLocationError('Cannot call course_id for {0} because it is not of category course'.format(self))
+            raise InvalidLocationError(u'Cannot call course_id for {0} because it is not of category course'.format(self))
 
         return "/".join([self.org, self.course, self.name])