From 121eec35e2f794b928aab6f3364f1c4b228e2d90 Mon Sep 17 00:00:00 2001 From: Didrik Pinte Date: Mon, 16 Mar 2015 19:35:15 +0100 Subject: [PATCH 1/4] switching to using a full unicode layer in preferences.py --- apptools/preferences/preferences.py | 8 ++++++ apptools/preferences/preferences_helper.py | 9 ++++--- .../tests/preferences_helper_test_case.py | 25 +++++++++++-------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/apptools/preferences/preferences.py b/apptools/preferences/preferences.py index 9a64d6ebc..746b1625f 100644 --- a/apptools/preferences/preferences.py +++ b/apptools/preferences/preferences.py @@ -541,6 +541,14 @@ def _remove_preferences_listener(self, listener): def _set(self, key, value): """ Set the value of a preference in this node. """ + + # everything must be unicode encoded so that ConfigObj configuration + # can properly serialize the data. If we receive regular str, they are + # supposed to be utf-8 encoded + if isinstance(value, str): + value = value.decode('utf-8') + else: + value = unicode(value) self._lk.acquire() old = self._preferences.get(key) diff --git a/apptools/preferences/preferences_helper.py b/apptools/preferences/preferences_helper.py index 227bac943..f57a7517c 100644 --- a/apptools/preferences/preferences_helper.py +++ b/apptools/preferences/preferences_helper.py @@ -130,12 +130,13 @@ def _get_value(self, trait_name, value): # If the trait type is 'Str' then we just take the raw value. if isinstance(handler, Str) or trait.is_str: - pass + # Just in case we get back a unicode object, convert it to a + # utf-8 encoded object. + value = value.encode('utf-8') elif isinstance(handler, Unicode): - # Just in case we get back an ASCII `str` object, convert it to a - # `unicode` object. - value = unicode(value) + + pass # Otherwise, we eval it! else: diff --git a/apptools/preferences/tests/preferences_helper_test_case.py b/apptools/preferences/tests/preferences_helper_test_case.py index 305a7a47d..792908343 100644 --- a/apptools/preferences/tests/preferences_helper_test_case.py +++ b/apptools/preferences/tests/preferences_helper_test_case.py @@ -201,7 +201,7 @@ class AcmeUIPreferencesHelper(PreferencesHelper): return def test_real_unicode_values(self): - """ default values """ + """ Test with real life unicode values """ p = self.preferences p.load(self.example) @@ -217,25 +217,30 @@ class AcmeUIPreferencesHelper(PreferencesHelper): width = Int(50) ratio = Float(1.0) visible = Bool(True) - description = Unicode(u'U\xdc\xf2ser') + description = Unicode(u'') offsets = List(Int, [1, 2, 3, 4]) names = List(Str, ['joe', 'fred', 'jane']) helper = AcmeUIPreferencesHelper() + first_unicode_str = u'U\xdc\xf2ser' + first_utf8_str = 'U\xc3\x9c\xc3\xb2ser' + original_description = helper.description - helper.description = u'U\xdc\xf2ser' - self.assertEqual(u'U\xdc\xf2ser', helper.description) + helper.description = first_unicode_str + self.assertEqual(first_unicode_str, helper.description) - helper.description = u'caf\xe9' - self.assertEqual(u'caf\xe9', helper.description) - self.assertEqual(u'caf\xe9', p.get('acme.ui.description')) + second_unicode_str = u'caf\xe9' + second_utf8_str = 'caf\xc3\xa9' + helper.description = second_unicode_str + self.assertEqual(second_unicode_str, helper.description) + self.assertEqual(second_unicode_str, p.get('acme.ui.description')) p.save(self.example) p.load(self.example) - self.assertEqual(u'caf\xe9', p.get('acme.ui.description')) + self.assertEqual(second_unicode_str, p.get('acme.ui.description')) self.assertEqual(u'True', p.get('acme.ui.visible')) self.assertEqual(True, helper.visible) @@ -496,11 +501,11 @@ def _width_changed(self, trait_name, old, new): # ratio to get set via the static trait change handler on the helper. p.set('acme.ui.width', 42) self.assertEqual(42, helper.width) - self.assertEqual(42, p.get('acme.ui.width')) + self.assertEqual('42', p.get('acme.ui.width')) # Did the ratio get changed? self.assertEqual(3.0, helper.ratio) - self.assertEqual(3.0, p.get('acme.ui.ratio')) + self.assertEqual('3.0', p.get('acme.ui.ratio')) return From 5bcbcc93250aaec78e324f681cf53c243061b022 Mon Sep 17 00:00:00 2001 From: Didrik Pinte Date: Tue, 17 Mar 2015 11:16:37 +0100 Subject: [PATCH 2/4] remove wrong assumption on utf-8 encoding for str --- apptools/preferences/preferences.py | 9 +++------ apptools/preferences/preferences_helper.py | 10 ++-------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/apptools/preferences/preferences.py b/apptools/preferences/preferences.py index 746b1625f..8a4b8c115 100644 --- a/apptools/preferences/preferences.py +++ b/apptools/preferences/preferences.py @@ -543,12 +543,9 @@ def _set(self, key, value): """ Set the value of a preference in this node. """ # everything must be unicode encoded so that ConfigObj configuration - # can properly serialize the data. If we receive regular str, they are - # supposed to be utf-8 encoded - if isinstance(value, str): - value = value.decode('utf-8') - else: - value = unicode(value) + # can properly serialize the data. Python str are supposed to be ASCII + # encoded. + value = unicode(value) self._lk.acquire() old = self._preferences.get(key) diff --git a/apptools/preferences/preferences_helper.py b/apptools/preferences/preferences_helper.py index f57a7517c..dfe643e7e 100644 --- a/apptools/preferences/preferences_helper.py +++ b/apptools/preferences/preferences_helper.py @@ -128,14 +128,8 @@ def _get_value(self, trait_name, value): trait = self.trait(trait_name) handler = trait.handler - # If the trait type is 'Str' then we just take the raw value. - if isinstance(handler, Str) or trait.is_str: - # Just in case we get back a unicode object, convert it to a - # utf-8 encoded object. - value = value.encode('utf-8') - - elif isinstance(handler, Unicode): - + # If the trait type is 'Str' or Unicode then we just take the raw value. + if isinstance(handler, (Str, Unicode)) or trait.is_str: pass # Otherwise, we eval it! From 0d421cf5f3dce6a320d3f3971f85762c114b82fd Mon Sep 17 00:00:00 2001 From: Didrik Pinte Date: Tue, 17 Mar 2015 11:18:42 +0100 Subject: [PATCH 3/4] updating changelog --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index e4cbdca8f..d248de535 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,7 +8,7 @@ Enhancements * Apptools now have a changelog! * Preferences system defaults to utf-8 encoded string with ConfigObj providing - better support for unicode in the PreferenceHelper (#41). + better support for unicode in the PreferenceHelper (#45). * Added a traitsified backport of Python 3's lru_cache (#39). * Added PyTables support to the io submodule (#19, #20, and #24 through #34). * Added a SelectionService for managing selections within an application (#15, #16, #17, From b440934284531ee7cef6bb02b4fc083eb3c67d21 Mon Sep 17 00:00:00 2001 From: Corran Webster Date: Tue, 17 Mar 2015 10:23:03 +0000 Subject: [PATCH 4/4] Update CHANGES.txt --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index d248de535..d8f5c6b48 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,7 +8,7 @@ Enhancements * Apptools now have a changelog! * Preferences system defaults to utf-8 encoded string with ConfigObj providing - better support for unicode in the PreferenceHelper (#45). + better support for unicode in the PreferenceHelper (#41, #45). * Added a traitsified backport of Python 3's lru_cache (#39). * Added PyTables support to the io submodule (#19, #20, and #24 through #34). * Added a SelectionService for managing selections within an application (#15, #16, #17,