From 83d7516a97b81adcabc35cbde5b17212e296e11e Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 31 Oct 2013 16:29:11 +0100 Subject: [PATCH 01/19] cow: add nose unit tests --- lib/bb/COW.py | 233 ++++++++++------------------ lib/bb/tests/cow.py | 136 ----------------- lib/bb/tests/test_cow.py | 319 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 402 insertions(+), 286 deletions(-) delete mode 100644 lib/bb/tests/cow.py create mode 100644 lib/bb/tests/test_cow.py diff --git a/lib/bb/COW.py b/lib/bb/COW.py index 6917ec378a3..7de46df3907 100644 --- a/lib/bb/COW.py +++ b/lib/bb/COW.py @@ -18,7 +18,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -#Please Note: +# Please Note: # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. # Assign a file to __warn__ to get warnings about slow operations. # @@ -40,46 +40,63 @@ MUTABLE = "__mutable__" +IGNORELIST = ['__module__', '__doc__', # Python's default builtins + '__count__', # from class COWDictBase, COWSetBase + '__hasmutable__', # from COWDictMeta + ] + + class COWMeta(type): pass + class COWDictMeta(COWMeta): - __warn__ = False + __warn__ = None __hasmutable__ = False __marker__ = tuple() - def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) - 3) - __repr__ = __str__ - def cow(cls): - class C(cls): + class COWDict(cls): __count__ = cls.__count__ + 1 - return C + return COWDict copy = cow __call__ = cow + def count(cls): + return cls.__count__ + def __setitem__(cls, key, value): + if not isinstance(key, str): + raise TypeError("%s: user key must be a string" % cls.__name__) + if key.startswith('__'): + # It does not make sense to let the user enter keys starting with + # '__' since we risk to overwrite existing Python builtins or + # even our own builtins + raise TypeError("%s: user key is not allowed to start with '__'" % + cls.__name__) if not isinstance(value, ImmutableTypes): if not isinstance(value, COWMeta): cls.__hasmutable__ = True - key += MUTABLE + key += MUTABLE # mutable keys will be suffixed by "__mutable__" setattr(cls, key, value) def __getmutable__(cls, key, readonly=False): + # Add the __mutable__ suffix to the key nkey = key + MUTABLE try: return cls.__dict__[nkey] except KeyError: pass + # Get nkey's value otherwise will raise AttributeError value = getattr(cls, nkey) if readonly: return value - if not cls.__warn__ is False and not isinstance(value, COWMeta): - print("Warning: Doing a copy because %s is a mutable type." % key, file=cls.__warn__) + if cls.__warn__ and not isinstance(value, COWMeta): + print("Warning: Doing a copy because %s is a mutable type." % + key, file=cls.__warn__) + try: value = value.copy() except AttributeError as e: @@ -88,17 +105,21 @@ def __getmutable__(cls, key, readonly=False): return value __getmarker__ = [] + def __getreadonly__(cls, key, default=__getmarker__): - """\ + """ Get a value (even if mutable) which you promise not to change. """ return cls.__getitem__(key, default, True) def __getitem__(cls, key, default=__getmarker__, readonly=False): + """ This method is called when calling obj[key] """ try: try: + # Check if the key is present in the attribute list value = getattr(cls, key) except AttributeError: + # Check if the key is mutable (with '__mutable__' suffix) value = cls.__getmutable__(key, readonly) # This is for values which have been deleted @@ -129,15 +150,17 @@ def has_key(cls, key): return False return True - def iter(cls, type, readonly=False): + def iter(cls, type_str, readonly=False): for key in dir(cls): + # We skip Python's builtins and the ones in IGNORELIST if key.startswith("__"): continue + # Mutable keys have a __mutable__ suffix that we remove if key.endswith(MUTABLE): key = key[:-len(MUTABLE)] - if type == "keys": + if type_str == "keys": yield key try: @@ -148,33 +171,58 @@ def iter(cls, type, readonly=False): except KeyError: continue - if type == "values": + if type_str == "values": yield value - if type == "items": + if type_str == "items": yield (key, value) raise StopIteration() - def iterkeys(cls): - return cls.iter("keys") + def iterkeys(cls, readonly=False): + return cls.iter("keys", readonly) + + # The default iterator is 'readonly' + def __iter__(cls): + return cls.iter("keys", readonly=True) + def itervalues(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + if cls.__warn__ and cls.__hasmutable__ and readonly is False: + print( + "Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("values", readonly) + def iteritems(cls, readonly=False): - if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + if cls.__warn__ and cls.__hasmutable__ and readonly is False: + print( + "Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("items", readonly) -class COWSetMeta(COWDictMeta): def __str__(cls): - # FIXME: I have magic numbers! - return "" % (cls.__count__, len(cls.__dict__) -3) + """ + Returns a string representation of this object + The number of keys is only showing keys in the current 'level' + """ + return ("<%s Level: %i Number of keys: %i>" % + (cls.__name__, cls.__count__, cls.__len__())) __repr__ = __str__ + def __len__(cls): + """ Returns the number of 'keys' in the COWDict """ + # cls.__dict__ is the default module namespace as a dictionary + # We skip keys found in IGNORELIST + i = 0 + for x in cls.__dict__: + if x in IGNORELIST: + continue + i += 1 + return i + + +class COWSetMeta(COWDictMeta): + def cow(cls): - class C(cls): + class COWSet(cls): __count__ = cls.__count__ + 1 - return C + return COWSet def add(cls, value): COWDictMeta.__setitem__(cls, repr(hash(value)), value) @@ -182,8 +230,8 @@ def add(cls, value): def remove(cls, value): COWDictMeta.__delitem__(cls, repr(hash(value))) - def __in__(cls, value): - return COWDictMeta.has_key(repr(hash(value))) + def __contains__(cls, value): + return COWDictMeta.has_key(cls, repr(hash(value))) def iterkeys(cls): raise TypeError("sets don't have keys") @@ -192,132 +240,17 @@ def iteritems(cls): raise TypeError("sets don't have 'items'") # These are the actual classes you use! + + class COWDictBase(object): __metaclass__ = COWDictMeta __count__ = 0 + class COWSetBase(object): __metaclass__ = COWSetMeta __count__ = 0 + if __name__ == "__main__": - import sys - COWDictBase.__warn__ = sys.stderr - a = COWDictBase() - print("a", a) - - a['a'] = 'a' - a['b'] = 'b' - a['dict'] = {} - - b = a.copy() - print("b", b) - b['c'] = 'b' - - print() - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - b['dict']['a'] = 'b' - b['a'] = 'c' - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - try: - b['dict2'] - except KeyError as e: - print("Okay!") - - a['set'] = COWSetBase() - a['set'].add("o1") - a['set'].add("o1") - a['set'].add("o2") - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - b['set'].add('o3') - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - a['set2'] = set() - a['set2'].add("o1") - a['set2'].add("o1") - a['set2'].add("o2") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - del b['b'] - try: - print(b['b']) - except KeyError: - print("Yay! deleted key raises error") - - if b.has_key('b'): - print("Boo!") - else: - print("Yay - has_key with delete works!") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('b') - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('dict') - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() + print("The unit tests in test_cow.py show how COWDict/SetBase are used") diff --git a/lib/bb/tests/cow.py b/lib/bb/tests/cow.py deleted file mode 100644 index 35c5841f322..00000000000 --- a/lib/bb/tests/cow.py +++ /dev/null @@ -1,136 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Tests for Copy-on-Write (cow.py) -# -# Copyright 2006 Holger Freyther -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import unittest -import os - -class COWTestCase(unittest.TestCase): - """ - Test case for the COW module from mithro - """ - - def testGetSet(self): - """ - Test and set - """ - from bb.COW import COWDictBase - a = COWDictBase.copy() - - self.assertEquals(False, a.has_key('a')) - - a['a'] = 'a' - a['b'] = 'b' - self.assertEquals(True, a.has_key('a')) - self.assertEquals(True, a.has_key('b')) - self.assertEquals('a', a['a'] ) - self.assertEquals('b', a['b'] ) - - def testCopyCopy(self): - """ - Test the copy of copies - """ - - from bb.COW import COWDictBase - - # create two COW dict 'instances' - b = COWDictBase.copy() - c = COWDictBase.copy() - - # assign some keys to one instance, some keys to another - b['a'] = 10 - b['c'] = 20 - c['a'] = 30 - - # test separation of the two instances - self.assertEquals(False, c.has_key('c')) - self.assertEquals(30, c['a']) - self.assertEquals(10, b['a']) - - # test copy - b_2 = b.copy() - c_2 = c.copy() - - self.assertEquals(False, c_2.has_key('c')) - self.assertEquals(10, b_2['a']) - - b_2['d'] = 40 - self.assertEquals(False, c_2.has_key('d')) - self.assertEquals(True, b_2.has_key('d')) - self.assertEquals(40, b_2['d']) - self.assertEquals(False, b.has_key('d')) - self.assertEquals(False, c.has_key('d')) - - c_2['d'] = 30 - self.assertEquals(True, c_2.has_key('d')) - self.assertEquals(True, b_2.has_key('d')) - self.assertEquals(30, c_2['d']) - self.assertEquals(40, b_2['d']) - self.assertEquals(False, b.has_key('d')) - self.assertEquals(False, c.has_key('d')) - - # test copy of the copy - c_3 = c_2.copy() - b_3 = b_2.copy() - b_3_2 = b_2.copy() - - c_3['e'] = 4711 - self.assertEquals(4711, c_3['e']) - self.assertEquals(False, c_2.has_key('e')) - self.assertEquals(False, b_3.has_key('e')) - self.assertEquals(False, b_3_2.has_key('e')) - self.assertEquals(False, b_2.has_key('e')) - - b_3['e'] = 'viel' - self.assertEquals('viel', b_3['e']) - self.assertEquals(4711, c_3['e']) - self.assertEquals(False, c_2.has_key('e')) - self.assertEquals(True, b_3.has_key('e')) - self.assertEquals(False, b_3_2.has_key('e')) - self.assertEquals(False, b_2.has_key('e')) - - def testCow(self): - from bb.COW import COWDictBase - c = COWDictBase.copy() - c['123'] = 1027 - c['other'] = 4711 - c['d'] = { 'abc' : 10, 'bcd' : 20 } - - copy = c.copy() - - self.assertEquals(1027, c['123']) - self.assertEquals(4711, c['other']) - self.assertEquals({'abc':10, 'bcd':20}, c['d']) - self.assertEquals(1027, copy['123']) - self.assertEquals(4711, copy['other']) - self.assertEquals({'abc':10, 'bcd':20}, copy['d']) - - # cow it now - copy['123'] = 1028 - copy['other'] = 4712 - copy['d']['abc'] = 20 - - - self.assertEquals(1027, c['123']) - self.assertEquals(4711, c['other']) - self.assertEquals({'abc':10, 'bcd':20}, c['d']) - self.assertEquals(1028, copy['123']) - self.assertEquals(4712, copy['other']) - self.assertEquals({'abc':20, 'bcd':20}, copy['d']) diff --git a/lib/bb/tests/test_cow.py b/lib/bb/tests/test_cow.py new file mode 100644 index 00000000000..140f70dba53 --- /dev/null +++ b/lib/bb/tests/test_cow.py @@ -0,0 +1,319 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for Copy-on-Write (cow.py) +# +# Copyright 2006 Holger Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import os +from bb.COW import COWDictBase, COWSetBase +from nose.tools import raises + + +class TestCOWDictBase(unittest.TestCase): + + """ + Test case for the COW module from mithro + """ + + def test_cow(self): + # Test base COW functionality + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + + self.assertEquals(c.count(), 1) # Level: 1 + self.assertEquals(len(c), 3) # c has 3 keys in Level #1 + + c_2 = c.copy() + + self.assertEquals(c_2.count(), 2) # c_2 Level: 2 (copy of c) + self.assertEquals(len(c_2), 0) # c_2 has 0 keys in Level #2 + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + self.assertEquals(d, c['d']) + self.assertEquals(1027, c_2['123']) + self.assertEquals(4711, c_2['other']) + + # The two dictionary objects must be identical at this point: + # We must use getattr() instead of c_2['d'] + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + # c_2['d'] will result in calling __getmutable__ and since 'd__mutable__' + # is not an attribute of c_2 (but only of c), and since readonly=False, + # it will do a d.copy() aka a 'shallow copy' of the dictionary 'd' + # So the copy on write behaviour actually happens here: + self.assertEquals(d, c_2['d']) + + # At this point, the two dictionary objects must be different: + # We must use getattr() instead of c_2['d'] + self.assertNotEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + # At this point c_2 has 1 key in Level #2 + self.assertEquals(c_2.count(), 2) + self.assertEquals(len(c_2), 1) + + # Change the immutable values for the same keys as in c + c_2['123'] = 1028 + c_2['other'] = 4712 + + # Since we only changed c_2, verify again that that c is unchanged + self.assertEquals(1027, c['123']) + self.assertEquals(4711, c['other']) + # However, c_2 values must have changed + self.assertEquals(1028, c_2['123']) + self.assertEquals(4712, c_2['other']) + + # Change the mutable values for the same keys as in c + c_2['d']['abc'] = 20 + + # Since we only changed c_2, verify again that that c is unchanged + self.assertEquals(d, c['d']) + # However, c_2 is changed + self.assertEquals({'abc': 20, 'bcd': 20}, c_2['d']) + + # At this point c_2 has 3 keys in Level #2 + self.assertEquals(c_2.count(), 2) + self.assertEquals(len(c_2), 3) + + def test_iter_readonly(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + expected_values = (1027, 4711, d) + expected_items = (('123', 1027), ('other', 4711), ('d', d)) + + i = 0 + for x in c.iterkeys(readonly=True): + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + i = 0 + for x in c.itervalues(readonly=True): + i += 1 + self.assertTrue(x in expected_values) + self.assertTrue(i == 3) + + i = 0 + for x in c.iteritems(readonly=True): + i += 1 + self.assertTrue(x in expected_items) + self.assertTrue(i == 3) + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c_2.iterkeys(readonly=True): + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c_2.itervalues(readonly=True): + i += 1 + self.assertTrue(x in expected_values) + self.assertTrue(i == 3) + + i = 0 + for x in c.iteritems(readonly=True): + i += 1 + self.assertTrue(x in expected_items) + self.assertTrue(i == 3) + + def test_default_ro_iter(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for x in c: + i += 1 + self.assertTrue(x in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + def test_nonro_iteritems(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + expected_keys = ('123', 'other', 'd') + + c_2 = c.copy() + + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + i = 0 + for k, v in c_2.iteritems(): + i += 1 + self.assertTrue(k in expected_keys) + self.assertTrue(i == 3) + + # Check that the mutable dict 'd' _has been_ shallow copied + self.assertNotEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + def test_cow_get_set(self): + a = COWDictBase() + self.assertEquals(False, a.has_key('a')) + + a['a'] = 'a' + a['b'] = 'b' + self.assertEquals(True, a.has_key('a')) + self.assertEquals(True, a.has_key('b')) + self.assertEquals('a', a['a']) + self.assertEquals('b', a['b']) + + def test_cow_copy_of_copy(self): + # Test the copy of copies + + # create two COW dict 'instances' + b = COWDictBase() + c = COWDictBase() + + # assign some keys to one instance, some keys to another + b['a'] = 10 + b['c'] = 20 + c['a'] = 30 + + # test separation of the two instances + self.assertEquals(False, c.has_key('c')) + self.assertEquals(30, c['a']) + self.assertEquals(10, b['a']) + + # test copy + b_2 = b.copy() + c_2 = c.copy() + + self.assertEquals(False, c_2.has_key('c')) + self.assertEquals(10, b_2['a']) + + b_2['d'] = 40 + self.assertEquals(False, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + c_2['d'] = 30 + self.assertEquals(True, c_2.has_key('d')) + self.assertEquals(True, b_2.has_key('d')) + self.assertEquals(30, c_2['d']) + self.assertEquals(40, b_2['d']) + self.assertEquals(False, b.has_key('d')) + self.assertEquals(False, c.has_key('d')) + + # test copy of the copy + c_3 = c_2.copy() + b_3 = b_2.copy() + b_3_2 = b_2.copy() + + c_3['e'] = 4711 + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(False, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + b_3['e'] = 'viel' + self.assertEquals('viel', b_3['e']) + self.assertEquals(4711, c_3['e']) + self.assertEquals(False, c_2.has_key('e')) + self.assertEquals(True, b_3.has_key('e')) + self.assertEquals(False, b_3_2.has_key('e')) + self.assertEquals(False, b_2.has_key('e')) + + def test_contains_ro(self): + c = COWDictBase() + c['xyz'] = 1 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + self.assertTrue('xyz' in c) + c_2 = c.copy() + self.assertTrue('d' in c) + self.assertTrue('d' in c_2) + # Check that the mutable dict 'd' has not been shallow copied + self.assertEquals(id(d), id(getattr(c_2, 'd__mutable__'))) + + @raises(KeyError) + def test_raise_keyerror(self): + c = COWDictBase() + c['123'] = 1027 + c['other'] = 4711 + a = c['gfgd'] + + def test_revertitem(self): + c = COWDictBase() + c['xyz'] = 1 + d = {'abc': 10, 'bcd': 20} + c['d'] = d + c_2 = c.copy() + c_2['xyz'] = 2 + self.assertTrue(c_2['xyz'], 2) + c_2.__revertitem__('xyz') + self.assertTrue(c_2['xyz'], 1) + + c_2['d']['abc'] = 20 + self.assertTrue(c_2['d']['abc'], 20) + c_2.__revertitem__('d') + self.assertTrue(c_2['d']['abc'], 10) + + def test_cowset(self): + c = COWDictBase() + c['set'] = COWSetBase() + c['set'].add("o1") + c['set'].add("o1") + self.assertTrue(len(c['set']), 1) + c['set'].add("o2") + self.assertTrue(len(c['set']), 2) + c['set'].remove("o1") + self.assertTrue(len(c['set']), 1) + self.assertTrue('o2' in c['set']) + + def test_cow_copy_anything(self): + class Anything: + var = 0 + pass + a = Anything() + c = COWDictBase() + c['any'] = a + self.assertEquals(id(a), id(getattr(c, 'any__mutable__'))) + c_2 = c.copy() + self.assertEquals(id(a), id(getattr(c_2, 'any__mutable__'))) + c_2['any'].var = 1 + self.assertNotEquals(id(a), id(getattr(c_2, 'any__mutable__'))) From ad9d2f925eaa781d9b0ffceef53735d91531c77a Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Mon, 4 Nov 2013 11:01:03 +0100 Subject: [PATCH 02/19] fixing stuff --- lib/bb/tests/{data.py => test_data.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/bb/tests/{data.py => test_data.py} (100%) diff --git a/lib/bb/tests/data.py b/lib/bb/tests/test_data.py similarity index 100% rename from lib/bb/tests/data.py rename to lib/bb/tests/test_data.py From 24662323df561a172b6c11bb8510c69a46328118 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Tue, 5 Nov 2013 16:19:01 +0100 Subject: [PATCH 03/19] small pep8 changes --- lib/bb/data_smart.py | 203 ++++++++++++++++++++++++------------------- 1 file changed, 115 insertions(+), 88 deletions(-) diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index a1cbaba62b2..72556dc1c64 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -28,22 +28,28 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re, sys, traceback +import copy +import re +import sys +import traceback from collections import MutableMapping import logging import hashlib -import bb, bb.codeparser -from bb import utils -from bb.COW import COWDictBase +import bb +import bb.codeparser +from bb import utils +from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") __setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') +__setvar_regexp__ = re.compile( + '(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") -def infer_caller_details(loginfo, parent = False, varval = True): + +def infer_caller_details(loginfo, parent=False, varval=True): """Save the caller the trouble of specifying everything.""" # Save effort. if 'ignore' in loginfo and loginfo['ignore']: @@ -52,7 +58,7 @@ def infer_caller_details(loginfo, parent = False, varval = True): if not loginfo: loginfo['ignore'] = True return - # Infer caller's likely values for variable (var) and value (value), + # Infer caller's likely values for variable (var) and value (value), # to reduce clutter in the rest of the code. if varval and ('variable' not in loginfo or 'detail' not in loginfo): try: @@ -71,17 +77,19 @@ def infer_caller_details(loginfo, parent = False, varval = True): loginfo['variable'] = v # Infer file/line/function from traceback if 'file' not in loginfo: - depth = 3 + depth = 3 if parent: depth = 4 - file, line, func, text = traceback.extract_stack(limit = depth)[0] - loginfo['file'] = file + file_, line, func, text = traceback.extract_stack(limit=depth)[0] + loginfo['file'] = file_ loginfo['line'] = line if func not in loginfo: loginfo['func'] = func + class VariableParse: - def __init__(self, varname, d, val = None): + + def __init__(self, varname, d, val=None): self.varname = varname self.d = d self.value = val @@ -90,41 +98,43 @@ def __init__(self, varname, d, val = None): self.execs = set() def var_sub(self, match): - key = match.group()[2:-1] - if self.varname and key: - if self.varname == key: - raise Exception("variable %s references itself!" % self.varname) - if key in self.d.expand_cache: - varparse = self.d.expand_cache[key] - var = varparse.value - else: - var = self.d.getVar(key, True) - self.references.add(key) - if var is not None: - return var - else: - return match.group() + key = match.group()[2:-1] + if self.varname and key: + if self.varname == key: + raise Exception( + "variable %s references itself!" % self.varname) + if key in self.d.expand_cache: + varparse = self.d.expand_cache[key] + var = varparse.value + else: + var = self.d.getVar(key, True) + self.references.add(key) + if var is not None: + return var + else: + return match.group() def python_sub(self, match): - code = match.group()[3:-1] - codeobj = compile(code.strip(), self.varname or "", "eval") - - parser = bb.codeparser.PythonParser(self.varname, logger) - parser.parse_python(code) - if self.varname: - vardeps = self.d.getVarFlag(self.varname, "vardeps", True) - if vardeps is None: - parser.log.flush() - else: + code = match.group()[3:-1] + codeobj = compile(code.strip(), self.varname or "", "eval") + + parser = bb.codeparser.PythonParser(self.varname, logger) + parser.parse_python(code) + if self.varname: + vardeps = self.d.getVarFlag(self.varname, "vardeps", True) + if vardeps is None: parser.log.flush() - self.references |= parser.references - self.execs |= parser.execs + else: + parser.log.flush() + self.references |= parser.references + self.execs |= parser.execs - value = utils.better_eval(codeobj, DataContext(self.d)) - return str(value) + value = utils.better_eval(codeobj, DataContext(self.d)) + return str(value) class DataContext(dict): + def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) @@ -137,25 +147,33 @@ def __missing__(self, key): else: return value + class ExpansionError(Exception): + def __init__(self, varname, expression, exception): self.expression = expression self.variablename = varname self.exception = exception if varname: if expression: - self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % ( + varname, expression, type(exception).__name__, exception) else: - self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s: %s: %s" % ( + varname, type(exception).__name__, exception) else: - self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) + self.msg = "Failure expanding expression %s which triggered exception %s: %s" % ( + expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) + def __str__(self): return self.msg + class IncludeHistory(object): - def __init__(self, parent = None, filename = '[TOP LEVEL]'): + + def __init__(self, parent=None, filename='[TOP LEVEL]'): self.parent = parent self.filename = filename self.children = [] @@ -180,10 +198,11 @@ def __exit__(self, a, b, c): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % filename) + bb.warn("Include log: Tried to finish '%s' at top level." % + self.filename) return False - def emit(self, o, level = 0): + def emit(self, o, level=0): """Emit an include history file, and its children.""" if level: spaces = " " * (level - 1) @@ -197,10 +216,12 @@ def emit(self, o, level = 0): o.write("\n") child.emit(o, level) + class VariableHistory(object): + def __init__(self, dataroot): self.dataroot = dataroot - self.variables = COWDictBase.copy() + self.variables = COWDictBase() def copy(self): new = VariableHistory(self.dataroot) @@ -212,7 +233,7 @@ def record(self, *kwonly, **loginfo): return if len(kwonly) > 0: raise TypeError - infer_caller_details(loginfo, parent = True) + infer_caller_details(loginfo, parent=True) if 'ignore' in loginfo and loginfo['ignore']: return if 'op' not in loginfo or not loginfo['op']: @@ -244,9 +265,9 @@ def emit(self, var, oval, val, o): for event in history: # o.write("# %s\n" % str(event)) if 'func' in event: - # If we have a function listed, this is internal - # code, not an operation in a config file, and the - # full path is distracting. + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. event['file'] = re.sub('.*/', '', event['file']) display_func = ' [%s]' % event['func'] else: @@ -255,7 +276,8 @@ def emit(self, var, oval, val, o): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % + (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) if len(history) > 1: o.write("# computed:\n") o.write('# "%s"\n' % (commentVal)) @@ -276,7 +298,7 @@ def get_variable_lines(self, var, f): var_history = self.variable(var) lines = [] for event in var_history: - if f== event['file']: + if f == event['file']: line = event['line'] lines.append(line) return lines @@ -285,12 +307,15 @@ def del_var_history(self, var, f=None, line=None): """If file f and line are not given, the entire history of var is deleted""" if var in self.variables: if f and line: - self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] + self.variables[var] = [x for x in self.variables[ + var] if x['file'] != f and x['line'] != line] else: self.variables[var] = [] + class DataSmart(MutableMapping): - def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): + + def __init__(self, special=None, seen=None): self.dict = {} self.inchistory = IncludeHistory() @@ -298,8 +323,8 @@ def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): self._tracking = False # cookie monster tribute - self._special_values = special - self._seen_overrides = seen + self._special_values = special or COWDictBase() + self._seen_overrides = seen or COWDictBase() self.expand_cache = {} @@ -311,7 +336,7 @@ def disableTracking(self): def expandWithRefs(self, s, varname): - if not isinstance(s, basestring): # sanity check + if not isinstance(s, basestring): # sanity check return VariableParse(varname, self, s) if varname and varname in self.expand_cache: @@ -340,18 +365,17 @@ def expandWithRefs(self, s, varname): return varparse - def expand(self, s, varname = None): + def expand(self, s, varname=None): return self.expandWithRefs(s, varname).value - - def finalize(self, parent = False): + def finalize(self, parent=False): """Performs final steps upon the datastore, including application of overrides""" overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] finalize_caller = { 'op': 'finalize', } - infer_caller_details(finalize_caller, parent = parent, varval = False) + infer_caller_details(finalize_caller, parent=parent, varval=False) # # Well let us see what breaks here. We used to iterate @@ -380,15 +404,16 @@ def finalize(self, parent = False): if o not in self._seen_overrides: continue - vars = self._seen_overrides[o].copy() - for var in vars: + vars_ = self._seen_overrides[o].copy() + for var in vars_: name = var[:-l] try: # Report only once, even if multiple changes. if name not in finalizes_reported: finalizes_reported[name] = True finalize_caller['variable'] = name - finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) + finalize_caller['detail'] = 'was: ' + str( + self.getVar(name, False)) self.varhistory.record(**finalize_caller) # Copy history of the override over. for event in self.varhistory.variable(var): @@ -396,7 +421,8 @@ def finalize(self, parent = False): loginfo['variable'] = name loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) self.varhistory.record(**loginfo) - self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') + self.setVar( + name, self.getVar(var, False), op='finalize', file='override[%s]' % o, line='') self.delVar(var) except Exception: logger.info("Untracked delVar") @@ -414,7 +440,7 @@ def finalize(self, parent = False): if not o2 in overrides: match = False if not match: - keep.append((a ,o)) + keep.append((a, o)) continue if op == "_append": @@ -425,9 +451,11 @@ def finalize(self, parent = False): sval = a + (self.getVar(append, False) or "") self.setVar(append, sval) elif op == "_remove": - removes = self.getVarFlag(append, "_removeactive", False) or [] + removes = self.getVarFlag( + append, "_removeactive", False) or [] removes.extend(a.split()) - self.setVarFlag(append, "_removeactive", removes, ignore=True) + self.setVarFlag( + append, "_removeactive", removes, ignore=True) # We save overrides that may be applied at some later stage if keep: @@ -461,13 +489,12 @@ def _makeShadowCopy(self, var): else: self.initVar(var) - def setVar(self, var, value, **loginfo): - #print("var=" + str(var) + " val=" + str(value)) + # print("var=" + str(var) + " val=" + str(value)) if 'op' not in loginfo: loginfo['op'] = "set" self.expand_cache = {} - match = __setvar_regexp__.match(var) + match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: base = match.group('base') keyword = match.group("keyword") @@ -506,11 +533,11 @@ def setVar(self, var, value, **loginfo): def _setvar_update_overrides(self, var): # aka pay the cookie monster - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if len(override) > 0: if override not in self._seen_overrides: self._seen_overrides[override] = set() - self._seen_overrides[override].add( var ) + self._seen_overrides[override].add(var) def getVar(self, var, expand=False, noweakdefault=False): return self.getVarFlag(var, "_content", expand, noweakdefault) @@ -565,7 +592,7 @@ def delVar(self, var, **loginfo): self.expand_cache = {} self.dict[var] = {} if '_' in var: - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) @@ -652,7 +679,7 @@ def setVarFlags(self, var, flags, **loginfo): self.varhistory.record(**loginfo) self.dict[var][i] = flags[i] - def getVarFlags(self, var, expand = False, internalflags=False): + def getVarFlags(self, var, expand=False, internalflags=False): local_var = self._findVar(var) flags = {} @@ -667,7 +694,6 @@ def getVarFlags(self, var, expand = False, internalflags=False): return None return flags - def delVarFlags(self, var, **loginfo): if not var in self.dict: self._makeShadowCopy(var) @@ -680,19 +706,19 @@ def delVarFlags(self, var, **loginfo): # try to save the content if "_content" in self.dict[var]: - content = self.dict[var]["_content"] - self.dict[var] = {} + content = self.dict[var]["_content"] + self.dict[var] = {} self.dict[var]["_content"] = content else: del self.dict[var] - def createCopy(self): """ Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) + data = DataSmart( + seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data @@ -724,7 +750,7 @@ def localkeys(self): yield key def __iter__(self): - def keylist(d): + def keylist(d): klist = set() for key in d: if key == "_data": @@ -739,7 +765,7 @@ def keylist(d): return klist for k in keylist(self.dict): - yield k + yield k def __len__(self): return len(frozenset(self)) @@ -763,32 +789,33 @@ def get_hash(self): bb.data.expandKeys(d) bb.data.update_data(d) - config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) + config_whitelist = set( + (d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: if key in config_whitelist: continue value = d.getVar(key, False) or "" - data.update({key:value}) + data.update({key: value}) - varflags = d.getVarFlags(key, internalflags = True) + varflags = d.getVarFlags(key, internalflags=True) if not varflags: continue for f in varflags: if f == "_content": continue - data.update({'%s[%s]' % (key, f):varflags[f]}) + data.update({'%s[%s]' % (key, f): varflags[f]}) for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: bb_list = d.getVar(key, False) or [] bb_list.sort() - data.update({key:str(bb_list)}) + data.update({key: str(bb_list)}) if key == "__BBANONFUNCS": for i in bb_list: value = d.getVar(i, True) or "" - data.update({i:value}) + data.update({i: value}) data_str = str([(k, data[k]) for k in sorted(data.keys())]) return hashlib.md5(data_str).hexdigest() From a251ef08bf520a882ee4bf8de54cf4b33c0dd80e Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Wed, 6 Nov 2013 11:34:25 +0100 Subject: [PATCH 04/19] ignore coverage files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 78aa674d322..f28691d1504 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ pyshtables.py /bin/bitbakec *.swp tags +lib/BAR +lib/FOO +.coverage +htmlcov/ From af536cd5061eea9a39b4a088dc7598451c905e96 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Wed, 6 Nov 2013 12:18:25 +0100 Subject: [PATCH 05/19] one step closer to understanding --- lib/bb/COW.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/bb/COW.py b/lib/bb/COW.py index 7de46df3907..dad03b3ed64 100644 --- a/lib/bb/COW.py +++ b/lib/bb/COW.py @@ -55,12 +55,12 @@ class COWDictMeta(COWMeta): __hasmutable__ = False __marker__ = tuple() - def cow(cls): + def copy(cls): class COWDict(cls): __count__ = cls.__count__ + 1 return COWDict - copy = cow - __call__ = cow + + __call__ = copy def count(cls): return cls.__count__ @@ -219,7 +219,7 @@ def __len__(cls): class COWSetMeta(COWDictMeta): - def cow(cls): + def copy(cls): class COWSet(cls): __count__ = cls.__count__ + 1 return COWSet @@ -239,9 +239,8 @@ def iterkeys(cls): def iteritems(cls): raise TypeError("sets don't have 'items'") -# These are the actual classes you use! - +# These are the actual classes you use! class COWDictBase(object): __metaclass__ = COWDictMeta __count__ = 0 From 36f0f32fceae2c932f848cf4e2c42d545dc88cb1 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 13:21:10 +0100 Subject: [PATCH 06/19] nosify more --- bin/bitbake-nose-selftest | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 bin/bitbake-nose-selftest diff --git a/bin/bitbake-nose-selftest b/bin/bitbake-nose-selftest new file mode 100755 index 00000000000..a45f8f28f3d --- /dev/null +++ b/bin/bitbake-nose-selftest @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import nose +#sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + '/lib') + +import inspect +import __builtin__ + +print sys.path +cdir = os.path.abspath(os.path.curdir) +for d in sys.path: + if cdir in d: + print d + +savimp = __builtin__.__import__ + +def newimp(name, *x, **xx): + caller = inspect.currentframe().f_back + print "module: [%s]" % name, "caller: [%s]" % caller.f_globals.get('__name__') + return savimp(name, *x, **xx) + +__builtin__.__import__ = newimp + +#sys.exit(0) +#import unittest +#try: + #import bb +#except RuntimeError as exc: + #sys.exit(str(exc)) + +#tests = ["bb.tests.codeparser", + #"bb.tests.cow", + #"bb.tests.data", + #"bb.tests.fetch", + #"bb.tests.utils"] + +#for t in tests: + #__import__(t) + +#unittest.main(argv=["bitbake-selftest"] + tests) + + +if __name__ == '__main__': + nose.run(argv=sys.argv) From d4ed0e80639e6bfea321eab58763d7ef67e10416 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 14:27:36 +0100 Subject: [PATCH 07/19] Revert "small pep8 changes" This reverts commit 24662323df561a172b6c11bb8510c69a46328118. --- lib/bb/data_smart.py | 203 +++++++++++++++++++------------------------ 1 file changed, 88 insertions(+), 115 deletions(-) diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index 72556dc1c64..a1cbaba62b2 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -28,28 +28,22 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy -import re -import sys -import traceback +import copy, re, sys, traceback from collections import MutableMapping import logging import hashlib -import bb -import bb.codeparser -from bb import utils -from bb.COW import COWDictBase +import bb, bb.codeparser +from bb import utils +from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") __setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile( - '(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') +__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") - -def infer_caller_details(loginfo, parent=False, varval=True): +def infer_caller_details(loginfo, parent = False, varval = True): """Save the caller the trouble of specifying everything.""" # Save effort. if 'ignore' in loginfo and loginfo['ignore']: @@ -58,7 +52,7 @@ def infer_caller_details(loginfo, parent=False, varval=True): if not loginfo: loginfo['ignore'] = True return - # Infer caller's likely values for variable (var) and value (value), + # Infer caller's likely values for variable (var) and value (value), # to reduce clutter in the rest of the code. if varval and ('variable' not in loginfo or 'detail' not in loginfo): try: @@ -77,19 +71,17 @@ def infer_caller_details(loginfo, parent=False, varval=True): loginfo['variable'] = v # Infer file/line/function from traceback if 'file' not in loginfo: - depth = 3 + depth = 3 if parent: depth = 4 - file_, line, func, text = traceback.extract_stack(limit=depth)[0] - loginfo['file'] = file_ + file, line, func, text = traceback.extract_stack(limit = depth)[0] + loginfo['file'] = file loginfo['line'] = line if func not in loginfo: loginfo['func'] = func - class VariableParse: - - def __init__(self, varname, d, val=None): + def __init__(self, varname, d, val = None): self.varname = varname self.d = d self.value = val @@ -98,43 +90,41 @@ def __init__(self, varname, d, val=None): self.execs = set() def var_sub(self, match): - key = match.group()[2:-1] - if self.varname and key: - if self.varname == key: - raise Exception( - "variable %s references itself!" % self.varname) - if key in self.d.expand_cache: - varparse = self.d.expand_cache[key] - var = varparse.value - else: - var = self.d.getVar(key, True) - self.references.add(key) - if var is not None: - return var - else: - return match.group() + key = match.group()[2:-1] + if self.varname and key: + if self.varname == key: + raise Exception("variable %s references itself!" % self.varname) + if key in self.d.expand_cache: + varparse = self.d.expand_cache[key] + var = varparse.value + else: + var = self.d.getVar(key, True) + self.references.add(key) + if var is not None: + return var + else: + return match.group() def python_sub(self, match): - code = match.group()[3:-1] - codeobj = compile(code.strip(), self.varname or "", "eval") - - parser = bb.codeparser.PythonParser(self.varname, logger) - parser.parse_python(code) - if self.varname: - vardeps = self.d.getVarFlag(self.varname, "vardeps", True) - if vardeps is None: + code = match.group()[3:-1] + codeobj = compile(code.strip(), self.varname or "", "eval") + + parser = bb.codeparser.PythonParser(self.varname, logger) + parser.parse_python(code) + if self.varname: + vardeps = self.d.getVarFlag(self.varname, "vardeps", True) + if vardeps is None: + parser.log.flush() + else: parser.log.flush() - else: - parser.log.flush() - self.references |= parser.references - self.execs |= parser.execs + self.references |= parser.references + self.execs |= parser.execs - value = utils.better_eval(codeobj, DataContext(self.d)) - return str(value) + value = utils.better_eval(codeobj, DataContext(self.d)) + return str(value) class DataContext(dict): - def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) @@ -147,33 +137,25 @@ def __missing__(self, key): else: return value - class ExpansionError(Exception): - def __init__(self, varname, expression, exception): self.expression = expression self.variablename = varname self.exception = exception if varname: if expression: - self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % ( - varname, expression, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) else: - self.msg = "Failure expanding variable %s: %s: %s" % ( - varname, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) else: - self.msg = "Failure expanding expression %s which triggered exception %s: %s" % ( - expression, type(exception).__name__, exception) + self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) - def __str__(self): return self.msg - class IncludeHistory(object): - - def __init__(self, parent=None, filename='[TOP LEVEL]'): + def __init__(self, parent = None, filename = '[TOP LEVEL]'): self.parent = parent self.filename = filename self.children = [] @@ -198,11 +180,10 @@ def __exit__(self, a, b, c): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % - self.filename) + bb.warn("Include log: Tried to finish '%s' at top level." % filename) return False - def emit(self, o, level=0): + def emit(self, o, level = 0): """Emit an include history file, and its children.""" if level: spaces = " " * (level - 1) @@ -216,12 +197,10 @@ def emit(self, o, level=0): o.write("\n") child.emit(o, level) - class VariableHistory(object): - def __init__(self, dataroot): self.dataroot = dataroot - self.variables = COWDictBase() + self.variables = COWDictBase.copy() def copy(self): new = VariableHistory(self.dataroot) @@ -233,7 +212,7 @@ def record(self, *kwonly, **loginfo): return if len(kwonly) > 0: raise TypeError - infer_caller_details(loginfo, parent=True) + infer_caller_details(loginfo, parent = True) if 'ignore' in loginfo and loginfo['ignore']: return if 'op' not in loginfo or not loginfo['op']: @@ -265,9 +244,9 @@ def emit(self, var, oval, val, o): for event in history: # o.write("# %s\n" % str(event)) if 'func' in event: - # If we have a function listed, this is internal - # code, not an operation in a config file, and the - # full path is distracting. + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. event['file'] = re.sub('.*/', '', event['file']) display_func = ' [%s]' % event['func'] else: @@ -276,8 +255,7 @@ def emit(self, var, oval, val, o): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % - (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) if len(history) > 1: o.write("# computed:\n") o.write('# "%s"\n' % (commentVal)) @@ -298,7 +276,7 @@ def get_variable_lines(self, var, f): var_history = self.variable(var) lines = [] for event in var_history: - if f == event['file']: + if f== event['file']: line = event['line'] lines.append(line) return lines @@ -307,15 +285,12 @@ def del_var_history(self, var, f=None, line=None): """If file f and line are not given, the entire history of var is deleted""" if var in self.variables: if f and line: - self.variables[var] = [x for x in self.variables[ - var] if x['file'] != f and x['line'] != line] + self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] else: self.variables[var] = [] - class DataSmart(MutableMapping): - - def __init__(self, special=None, seen=None): + def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): self.dict = {} self.inchistory = IncludeHistory() @@ -323,8 +298,8 @@ def __init__(self, special=None, seen=None): self._tracking = False # cookie monster tribute - self._special_values = special or COWDictBase() - self._seen_overrides = seen or COWDictBase() + self._special_values = special + self._seen_overrides = seen self.expand_cache = {} @@ -336,7 +311,7 @@ def disableTracking(self): def expandWithRefs(self, s, varname): - if not isinstance(s, basestring): # sanity check + if not isinstance(s, basestring): # sanity check return VariableParse(varname, self, s) if varname and varname in self.expand_cache: @@ -365,17 +340,18 @@ def expandWithRefs(self, s, varname): return varparse - def expand(self, s, varname=None): + def expand(self, s, varname = None): return self.expandWithRefs(s, varname).value - def finalize(self, parent=False): + + def finalize(self, parent = False): """Performs final steps upon the datastore, including application of overrides""" overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] finalize_caller = { 'op': 'finalize', } - infer_caller_details(finalize_caller, parent=parent, varval=False) + infer_caller_details(finalize_caller, parent = parent, varval = False) # # Well let us see what breaks here. We used to iterate @@ -404,16 +380,15 @@ def finalize(self, parent=False): if o not in self._seen_overrides: continue - vars_ = self._seen_overrides[o].copy() - for var in vars_: + vars = self._seen_overrides[o].copy() + for var in vars: name = var[:-l] try: # Report only once, even if multiple changes. if name not in finalizes_reported: finalizes_reported[name] = True finalize_caller['variable'] = name - finalize_caller['detail'] = 'was: ' + str( - self.getVar(name, False)) + finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) self.varhistory.record(**finalize_caller) # Copy history of the override over. for event in self.varhistory.variable(var): @@ -421,8 +396,7 @@ def finalize(self, parent=False): loginfo['variable'] = name loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) self.varhistory.record(**loginfo) - self.setVar( - name, self.getVar(var, False), op='finalize', file='override[%s]' % o, line='') + self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') self.delVar(var) except Exception: logger.info("Untracked delVar") @@ -440,7 +414,7 @@ def finalize(self, parent=False): if not o2 in overrides: match = False if not match: - keep.append((a, o)) + keep.append((a ,o)) continue if op == "_append": @@ -451,11 +425,9 @@ def finalize(self, parent=False): sval = a + (self.getVar(append, False) or "") self.setVar(append, sval) elif op == "_remove": - removes = self.getVarFlag( - append, "_removeactive", False) or [] + removes = self.getVarFlag(append, "_removeactive", False) or [] removes.extend(a.split()) - self.setVarFlag( - append, "_removeactive", removes, ignore=True) + self.setVarFlag(append, "_removeactive", removes, ignore=True) # We save overrides that may be applied at some later stage if keep: @@ -489,12 +461,13 @@ def _makeShadowCopy(self, var): else: self.initVar(var) + def setVar(self, var, value, **loginfo): - # print("var=" + str(var) + " val=" + str(value)) + #print("var=" + str(var) + " val=" + str(value)) if 'op' not in loginfo: loginfo['op'] = "set" self.expand_cache = {} - match = __setvar_regexp__.match(var) + match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: base = match.group('base') keyword = match.group("keyword") @@ -533,11 +506,11 @@ def setVar(self, var, value, **loginfo): def _setvar_update_overrides(self, var): # aka pay the cookie monster - override = var[var.rfind('_') + 1:] + override = var[var.rfind('_')+1:] if len(override) > 0: if override not in self._seen_overrides: self._seen_overrides[override] = set() - self._seen_overrides[override].add(var) + self._seen_overrides[override].add( var ) def getVar(self, var, expand=False, noweakdefault=False): return self.getVarFlag(var, "_content", expand, noweakdefault) @@ -592,7 +565,7 @@ def delVar(self, var, **loginfo): self.expand_cache = {} self.dict[var] = {} if '_' in var: - override = var[var.rfind('_') + 1:] + override = var[var.rfind('_')+1:] if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) @@ -679,7 +652,7 @@ def setVarFlags(self, var, flags, **loginfo): self.varhistory.record(**loginfo) self.dict[var][i] = flags[i] - def getVarFlags(self, var, expand=False, internalflags=False): + def getVarFlags(self, var, expand = False, internalflags=False): local_var = self._findVar(var) flags = {} @@ -694,6 +667,7 @@ def getVarFlags(self, var, expand=False, internalflags=False): return None return flags + def delVarFlags(self, var, **loginfo): if not var in self.dict: self._makeShadowCopy(var) @@ -706,19 +680,19 @@ def delVarFlags(self, var, **loginfo): # try to save the content if "_content" in self.dict[var]: - content = self.dict[var]["_content"] - self.dict[var] = {} + content = self.dict[var]["_content"] + self.dict[var] = {} self.dict[var]["_content"] = content else: del self.dict[var] + def createCopy(self): """ Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart( - seen=self._seen_overrides.copy(), special=self._special_values.copy()) + data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data @@ -750,7 +724,7 @@ def localkeys(self): yield key def __iter__(self): - def keylist(d): + def keylist(d): klist = set() for key in d: if key == "_data": @@ -765,7 +739,7 @@ def keylist(d): return klist for k in keylist(self.dict): - yield k + yield k def __len__(self): return len(frozenset(self)) @@ -789,33 +763,32 @@ def get_hash(self): bb.data.expandKeys(d) bb.data.update_data(d) - config_whitelist = set( - (d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) + config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: if key in config_whitelist: continue value = d.getVar(key, False) or "" - data.update({key: value}) + data.update({key:value}) - varflags = d.getVarFlags(key, internalflags=True) + varflags = d.getVarFlags(key, internalflags = True) if not varflags: continue for f in varflags: if f == "_content": continue - data.update({'%s[%s]' % (key, f): varflags[f]}) + data.update({'%s[%s]' % (key, f):varflags[f]}) for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: bb_list = d.getVar(key, False) or [] bb_list.sort() - data.update({key: str(bb_list)}) + data.update({key:str(bb_list)}) if key == "__BBANONFUNCS": for i in bb_list: value = d.getVar(i, True) or "" - data.update({i: value}) + data.update({i:value}) data_str = str([(k, data[k]) for k in sorted(data.keys())]) return hashlib.md5(data_str).hexdigest() From 9cbee2fd86e69c271467d3ae0e6d73fa947ec8e7 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 15:03:50 +0100 Subject: [PATCH 08/19] change the version number --- lib/bb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index c0c77356372..2512b80175a 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -21,7 +21,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = "1.21.0" +__version__ = "1.21.0-p1" import sys if sys.version_info < (2, 7, 3): From d6094b15955792298f53ea8eab09d5dc1bec184a Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 15:13:41 +0100 Subject: [PATCH 09/19] small changes for pep8 compliancy --- lib/bb/data_smart.py | 198 ++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 85 deletions(-) diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index a1cbaba62b2..4aa1c9cb139 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -28,22 +28,28 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re, sys, traceback +import copy +import re +import sys +import traceback from collections import MutableMapping import logging import hashlib -import bb, bb.codeparser -from bb import utils -from bb.COW import COWDictBase +import bb +import bb.codeparser +from bb import utils +from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") __setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') +__setvar_regexp__ = re.compile( + '(?P.*?)(?P_append|_prepend|_remove)(_(?P.*))?$') __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}") __expand_python_regexp__ = re.compile(r"\${@.+?}") -def infer_caller_details(loginfo, parent = False, varval = True): + +def infer_caller_details(loginfo, parent=False, varval=True): """Save the caller the trouble of specifying everything.""" # Save effort. if 'ignore' in loginfo and loginfo['ignore']: @@ -52,7 +58,7 @@ def infer_caller_details(loginfo, parent = False, varval = True): if not loginfo: loginfo['ignore'] = True return - # Infer caller's likely values for variable (var) and value (value), + # Infer caller's likely values for variable (var) and value (value), # to reduce clutter in the rest of the code. if varval and ('variable' not in loginfo or 'detail' not in loginfo): try: @@ -71,17 +77,19 @@ def infer_caller_details(loginfo, parent = False, varval = True): loginfo['variable'] = v # Infer file/line/function from traceback if 'file' not in loginfo: - depth = 3 + depth = 3 if parent: depth = 4 - file, line, func, text = traceback.extract_stack(limit = depth)[0] - loginfo['file'] = file + file_, line, func, text = traceback.extract_stack(limit=depth)[0] + loginfo['file'] = file_ loginfo['line'] = line if func not in loginfo: loginfo['func'] = func + class VariableParse: - def __init__(self, varname, d, val = None): + + def __init__(self, varname, d, val=None): self.varname = varname self.d = d self.value = val @@ -90,41 +98,43 @@ def __init__(self, varname, d, val = None): self.execs = set() def var_sub(self, match): - key = match.group()[2:-1] - if self.varname and key: - if self.varname == key: - raise Exception("variable %s references itself!" % self.varname) - if key in self.d.expand_cache: - varparse = self.d.expand_cache[key] - var = varparse.value - else: - var = self.d.getVar(key, True) - self.references.add(key) - if var is not None: - return var - else: - return match.group() + key = match.group()[2:-1] + if self.varname and key: + if self.varname == key: + raise Exception( + "variable %s references itself!" % self.varname) + if key in self.d.expand_cache: + varparse = self.d.expand_cache[key] + var = varparse.value + else: + var = self.d.getVar(key, True) + self.references.add(key) + if var is not None: + return var + else: + return match.group() def python_sub(self, match): - code = match.group()[3:-1] - codeobj = compile(code.strip(), self.varname or "", "eval") - - parser = bb.codeparser.PythonParser(self.varname, logger) - parser.parse_python(code) - if self.varname: - vardeps = self.d.getVarFlag(self.varname, "vardeps", True) - if vardeps is None: - parser.log.flush() - else: + code = match.group()[3:-1] + codeobj = compile(code.strip(), self.varname or "", "eval") + + parser = bb.codeparser.PythonParser(self.varname, logger) + parser.parse_python(code) + if self.varname: + vardeps = self.d.getVarFlag(self.varname, "vardeps", True) + if vardeps is None: parser.log.flush() - self.references |= parser.references - self.execs |= parser.execs + else: + parser.log.flush() + self.references |= parser.references + self.execs |= parser.execs - value = utils.better_eval(codeobj, DataContext(self.d)) - return str(value) + value = utils.better_eval(codeobj, DataContext(self.d)) + return str(value) class DataContext(dict): + def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) @@ -137,25 +147,33 @@ def __missing__(self, key): else: return value + class ExpansionError(Exception): + def __init__(self, varname, expression, exception): self.expression = expression self.variablename = varname self.exception = exception if varname: if expression: - self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % ( + varname, expression, type(exception).__name__, exception) else: - self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) + self.msg = "Failure expanding variable %s: %s: %s" % ( + varname, type(exception).__name__, exception) else: - self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) + self.msg = "Failure expanding expression %s which triggered exception %s: %s" % ( + expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) + def __str__(self): return self.msg + class IncludeHistory(object): - def __init__(self, parent = None, filename = '[TOP LEVEL]'): + + def __init__(self, parent=None, filename='[TOP LEVEL]'): self.parent = parent self.filename = filename self.children = [] @@ -180,10 +198,11 @@ def __exit__(self, a, b, c): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % filename) + bb.warn("Include log: Tried to finish '%s' at top level." % + filename) return False - def emit(self, o, level = 0): + def emit(self, o, level=0): """Emit an include history file, and its children.""" if level: spaces = " " * (level - 1) @@ -197,7 +216,9 @@ def emit(self, o, level = 0): o.write("\n") child.emit(o, level) + class VariableHistory(object): + def __init__(self, dataroot): self.dataroot = dataroot self.variables = COWDictBase.copy() @@ -212,7 +233,7 @@ def record(self, *kwonly, **loginfo): return if len(kwonly) > 0: raise TypeError - infer_caller_details(loginfo, parent = True) + infer_caller_details(loginfo, parent=True) if 'ignore' in loginfo and loginfo['ignore']: return if 'op' not in loginfo or not loginfo['op']: @@ -244,9 +265,9 @@ def emit(self, var, oval, val, o): for event in history: # o.write("# %s\n" % str(event)) if 'func' in event: - # If we have a function listed, this is internal - # code, not an operation in a config file, and the - # full path is distracting. + # If we have a function listed, this is internal + # code, not an operation in a config file, and the + # full path is distracting. event['file'] = re.sub('.*/', '', event['file']) display_func = ' [%s]' % event['func'] else: @@ -255,7 +276,10 @@ def emit(self, var, oval, val, o): flag = '[%s] ' % (event['flag']) else: flag = '' - o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) + o.write("# %s %s:%s%s\n# %s\"%s\"\n" % + (event['op'], event['file'], event['line'], + display_func, flag, re.sub('\n', '\n# ', + event['detail']))) if len(history) > 1: o.write("# computed:\n") o.write('# "%s"\n' % (commentVal)) @@ -276,7 +300,7 @@ def get_variable_lines(self, var, f): var_history = self.variable(var) lines = [] for event in var_history: - if f== event['file']: + if f == event['file']: line = event['line'] lines.append(line) return lines @@ -285,12 +309,14 @@ def del_var_history(self, var, f=None, line=None): """If file f and line are not given, the entire history of var is deleted""" if var in self.variables: if f and line: - self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] + self.variables[var] = [x for x in self.variables[var] if x['file'] != f and x['line'] != line] else: self.variables[var] = [] + class DataSmart(MutableMapping): - def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): + + def __init__(self, special=COWDictBase.copy(), seen=COWDictBase.copy()): self.dict = {} self.inchistory = IncludeHistory() @@ -311,7 +337,7 @@ def disableTracking(self): def expandWithRefs(self, s, varname): - if not isinstance(s, basestring): # sanity check + if not isinstance(s, basestring): # sanity check return VariableParse(varname, self, s) if varname and varname in self.expand_cache: @@ -340,18 +366,17 @@ def expandWithRefs(self, s, varname): return varparse - def expand(self, s, varname = None): + def expand(self, s, varname=None): return self.expandWithRefs(s, varname).value - - def finalize(self, parent = False): + def finalize(self, parent=False): """Performs final steps upon the datastore, including application of overrides""" overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] finalize_caller = { 'op': 'finalize', } - infer_caller_details(finalize_caller, parent = parent, varval = False) + infer_caller_details(finalize_caller, parent=parent, varval=False) # # Well let us see what breaks here. We used to iterate @@ -380,15 +405,16 @@ def finalize(self, parent = False): if o not in self._seen_overrides: continue - vars = self._seen_overrides[o].copy() - for var in vars: + vars_ = self._seen_overrides[o].copy() + for var in vars_: name = var[:-l] try: # Report only once, even if multiple changes. if name not in finalizes_reported: finalizes_reported[name] = True finalize_caller['variable'] = name - finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) + finalize_caller['detail'] = 'was: ' + str( + self.getVar(name, False)) self.varhistory.record(**finalize_caller) # Copy history of the override over. for event in self.varhistory.variable(var): @@ -396,7 +422,8 @@ def finalize(self, parent = False): loginfo['variable'] = name loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) self.varhistory.record(**loginfo) - self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') + self.setVar( + name, self.getVar(var, False), op='finalize', file='override[%s]' % o, line='') self.delVar(var) except Exception: logger.info("Untracked delVar") @@ -414,7 +441,7 @@ def finalize(self, parent = False): if not o2 in overrides: match = False if not match: - keep.append((a ,o)) + keep.append((a, o)) continue if op == "_append": @@ -425,9 +452,11 @@ def finalize(self, parent = False): sval = a + (self.getVar(append, False) or "") self.setVar(append, sval) elif op == "_remove": - removes = self.getVarFlag(append, "_removeactive", False) or [] + removes = self.getVarFlag( + append, "_removeactive", False) or [] removes.extend(a.split()) - self.setVarFlag(append, "_removeactive", removes, ignore=True) + self.setVarFlag( + append, "_removeactive", removes, ignore=True) # We save overrides that may be applied at some later stage if keep: @@ -461,13 +490,12 @@ def _makeShadowCopy(self, var): else: self.initVar(var) - def setVar(self, var, value, **loginfo): - #print("var=" + str(var) + " val=" + str(value)) + # print("var=" + str(var) + " val=" + str(value)) if 'op' not in loginfo: loginfo['op'] = "set" self.expand_cache = {} - match = __setvar_regexp__.match(var) + match = __setvar_regexp__.match(var) if match and match.group("keyword") in __setvar_keyword__: base = match.group('base') keyword = match.group("keyword") @@ -506,11 +534,11 @@ def setVar(self, var, value, **loginfo): def _setvar_update_overrides(self, var): # aka pay the cookie monster - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if len(override) > 0: if override not in self._seen_overrides: self._seen_overrides[override] = set() - self._seen_overrides[override].add( var ) + self._seen_overrides[override].add(var) def getVar(self, var, expand=False, noweakdefault=False): return self.getVarFlag(var, "_content", expand, noweakdefault) @@ -565,7 +593,7 @@ def delVar(self, var, **loginfo): self.expand_cache = {} self.dict[var] = {} if '_' in var: - override = var[var.rfind('_')+1:] + override = var[var.rfind('_') + 1:] if override and override in self._seen_overrides and var in self._seen_overrides[override]: self._seen_overrides[override].remove(var) @@ -652,7 +680,7 @@ def setVarFlags(self, var, flags, **loginfo): self.varhistory.record(**loginfo) self.dict[var][i] = flags[i] - def getVarFlags(self, var, expand = False, internalflags=False): + def getVarFlags(self, var, expand=False, internalflags=False): local_var = self._findVar(var) flags = {} @@ -667,7 +695,6 @@ def getVarFlags(self, var, expand = False, internalflags=False): return None return flags - def delVarFlags(self, var, **loginfo): if not var in self.dict: self._makeShadowCopy(var) @@ -680,19 +707,19 @@ def delVarFlags(self, var, **loginfo): # try to save the content if "_content" in self.dict[var]: - content = self.dict[var]["_content"] - self.dict[var] = {} + content = self.dict[var]["_content"] + self.dict[var] = {} self.dict[var]["_content"] = content else: del self.dict[var] - def createCopy(self): """ Create a copy of self by setting _data to self """ # we really want this to be a DataSmart... - data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) + data = DataSmart( + seen=self._seen_overrides.copy(), special=self._special_values.copy()) data.dict["_data"] = self.dict data.varhistory = self.varhistory.copy() data.varhistory.datasmart = data @@ -724,7 +751,7 @@ def localkeys(self): yield key def __iter__(self): - def keylist(d): + def keylist(d): klist = set() for key in d: if key == "_data": @@ -739,7 +766,7 @@ def keylist(d): return klist for k in keylist(self.dict): - yield k + yield k def __len__(self): return len(frozenset(self)) @@ -763,32 +790,33 @@ def get_hash(self): bb.data.expandKeys(d) bb.data.update_data(d) - config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) + config_whitelist = set( + (d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: if key in config_whitelist: continue value = d.getVar(key, False) or "" - data.update({key:value}) + data.update({key: value}) - varflags = d.getVarFlags(key, internalflags = True) + varflags = d.getVarFlags(key, internalflags=True) if not varflags: continue for f in varflags: if f == "_content": continue - data.update({'%s[%s]' % (key, f):varflags[f]}) + data.update({'%s[%s]' % (key, f): varflags[f]}) for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: bb_list = d.getVar(key, False) or [] bb_list.sort() - data.update({key:str(bb_list)}) + data.update({key: str(bb_list)}) if key == "__BBANONFUNCS": for i in bb_list: value = d.getVar(i, True) or "" - data.update({i:value}) + data.update({i: value}) data_str = str([(k, data[k]) for k in sorted(data.keys())]) return hashlib.md5(data_str).hexdigest() From 395162a2b01094f0d3392c12814d9158d063ef33 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 15:33:50 +0100 Subject: [PATCH 10/19] removed the mutable types from __init__ --- lib/bb/data_smart.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index 4aa1c9cb139..cff62a783a5 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -221,7 +221,7 @@ class VariableHistory(object): def __init__(self, dataroot): self.dataroot = dataroot - self.variables = COWDictBase.copy() + self.variables = COWDictBase() def copy(self): new = VariableHistory(self.dataroot) @@ -316,7 +316,7 @@ def del_var_history(self, var, f=None, line=None): class DataSmart(MutableMapping): - def __init__(self, special=COWDictBase.copy(), seen=COWDictBase.copy()): + def __init__(self, special=None, seen=None): self.dict = {} self.inchistory = IncludeHistory() @@ -324,8 +324,15 @@ def __init__(self, special=COWDictBase.copy(), seen=COWDictBase.copy()): self._tracking = False # cookie monster tribute - self._special_values = special - self._seen_overrides = seen + if special is None: + self._special_values = COWDictBase() + else: + self._special_values = special + + if seen is None: + self._seen_overrides = COWDictBase() + else: + self._seen_overrides = seen self.expand_cache = {} From cf135c505f92cac20d9ba328b1044b4fce4f68ba Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Fri, 8 Nov 2013 16:04:15 +0100 Subject: [PATCH 11/19] specifying one test module also works --- bin/bitbake-nose-selftest | 27 ++++++++--------- lib/bb/__init__.py | 41 ++------------------------ lib/bb/data_smart.py | 1 + lib/bb/parse/parse_py/BBHandler.py | 47 ++++++++++++++++++++++++++++-- lib/bb/parse/parse_py/__init__.py | 4 +-- 5 files changed, 65 insertions(+), 55 deletions(-) diff --git a/bin/bitbake-nose-selftest b/bin/bitbake-nose-selftest index a45f8f28f3d..c7d286b8861 100755 --- a/bin/bitbake-nose-selftest +++ b/bin/bitbake-nose-selftest @@ -21,23 +21,24 @@ import nose #sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + '/lib') -import inspect -import __builtin__ +if False: + import inspect + import __builtin__ -print sys.path -cdir = os.path.abspath(os.path.curdir) -for d in sys.path: - if cdir in d: - print d + print sys.path + cdir = os.path.abspath(os.path.curdir) + for d in sys.path: + if cdir in d: + print d -savimp = __builtin__.__import__ + savimp = __builtin__.__import__ -def newimp(name, *x, **xx): - caller = inspect.currentframe().f_back - print "module: [%s]" % name, "caller: [%s]" % caller.f_globals.get('__name__') - return savimp(name, *x, **xx) + def newimp(name, *x, **xx): + caller = inspect.currentframe().f_back + print "module: [%s]" % name, "caller: [%s]" % caller.f_globals.get('__name__') + return savimp(name, *x, **xx) -__builtin__.__import__ = newimp + __builtin__.__import__ = newimp #sys.exit(0) #import unittest diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index 2512b80175a..98f3e29c203 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -102,42 +102,7 @@ def fatal(*args): sys.exit(1) -def deprecated(func, name=None, advice=""): - """This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emmitted - when the function is used.""" - import warnings - - if advice: - advice = ": %s" % advice - if name is None: - name = func.__name__ - - def newFunc(*args, **kwargs): - warnings.warn("Call to deprecated function %s%s." % (name, - advice), - category=DeprecationWarning, - stacklevel=2) - return func(*args, **kwargs) - newFunc.__name__ = func.__name__ - newFunc.__doc__ = func.__doc__ - newFunc.__dict__.update(func.__dict__) - return newFunc - -# For compatibility -def deprecate_import(current, modulename, fromlist, renames = None): - """Import objects from one module into another, wrapping them with a DeprecationWarning""" - import sys - - module = __import__(modulename, fromlist = fromlist) - for position, objname in enumerate(fromlist): - obj = getattr(module, objname) - newobj = deprecated(obj, "{0}.{1}".format(current, objname), - "Please use {0}.{1} instead".format(modulename, objname)) - if renames: - newname = renames[position] - else: - newname = objname - - setattr(sys.modules[current], newname, newobj) + + + diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index cff62a783a5..fbcfd311f27 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -37,6 +37,7 @@ import hashlib import bb import bb.codeparser +import bb.parse from bb import utils from bb.COW import COWDictBase diff --git a/lib/bb/parse/parse_py/BBHandler.py b/lib/bb/parse/parse_py/BBHandler.py index 01f22d3b243..acc7c1b3916 100644 --- a/lib/bb/parse/parse_py/BBHandler.py +++ b/lib/bb/parse/parse_py/BBHandler.py @@ -28,15 +28,58 @@ class for handling .bb files from __future__ import absolute_import import re, bb, os import logging +#from bb import deprecate_import import bb.build, bb.utils -from bb import data +import bb.data +#from bb import data from . import ConfHandler from .. import resolve_file, ast, logger from .ConfHandler import include, init +# I had to move deprecated and deprecate_import here to avoid an import error +# and this is actually only used in this module +def deprecated(func, name=None, advice=""): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emmitted + when the function is used.""" + import warnings + + if advice: + advice = ": %s" % advice + if name is None: + name = func.__name__ + + def newFunc(*args, **kwargs): + warnings.warn("Call to deprecated function %s%s." % (name, + advice), + category=DeprecationWarning, + stacklevel=2) + return func(*args, **kwargs) + newFunc.__name__ = func.__name__ + newFunc.__doc__ = func.__doc__ + newFunc.__dict__.update(func.__dict__) + return newFunc + +# For compatibility +def deprecate_import(current, modulename, fromlist, renames = None): + """Import objects from one module into another, wrapping them with a DeprecationWarning""" + import sys + + module = __import__(modulename, fromlist = fromlist) + for position, objname in enumerate(fromlist): + obj = getattr(module, objname) + newobj = deprecated(obj, "{0}.{1}".format(current, objname), + "Please use {0}.{1} instead".format(modulename, objname)) + if renames: + newname = renames[position] + else: + newname = objname + + setattr(sys.modules[current], newname, newobj) + # For compatibility -bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) +deprecate_import(__name__, "bb.parse", ["vars_from_file"]) __func_start_regexp__ = re.compile( r"(((?Ppython)|(?Pfakeroot))\s*)*(?P[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile( r"inherit\s+(.+)" ) diff --git a/lib/bb/parse/parse_py/__init__.py b/lib/bb/parse/parse_py/__init__.py index 3e658d0de99..75ca47c7179 100644 --- a/lib/bb/parse/parse_py/__init__.py +++ b/lib/bb/parse/parse_py/__init__.py @@ -27,7 +27,7 @@ # Based on functions from the base bb module, Copyright 2003 Holger Schurig from __future__ import absolute_import -from . import ConfHandler -from . import BBHandler +from bb.parse.parse_py import ConfHandler +from bb.parse.parse_py import BBHandler __version__ = '1.0' From e69822fdba72f9d08bf7f1101cd7bfe33e94c97b Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Wed, 13 Nov 2013 12:34:21 +0100 Subject: [PATCH 12/19] added codeparser --- .gitignore | 2 - lib/bb/tests/codeparser.py | 374 ------------------------------------- 2 files changed, 376 deletions(-) delete mode 100644 lib/bb/tests/codeparser.py diff --git a/.gitignore b/.gitignore index f28691d1504..3bcf547dd9f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,5 @@ pyshtables.py /bin/bitbakec *.swp tags -lib/BAR -lib/FOO .coverage htmlcov/ diff --git a/lib/bb/tests/codeparser.py b/lib/bb/tests/codeparser.py deleted file mode 100644 index 938b04b2c60..00000000000 --- a/lib/bb/tests/codeparser.py +++ /dev/null @@ -1,374 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Test for codeparser.py -# -# Copyright (C) 2010 Chris Larson -# Copyright (C) 2012 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import unittest -import logging -import bb - -logger = logging.getLogger('BitBake.TestCodeParser') - -# bb.data references bb.parse but can't directly import due to circular dependencies. -# Hack around it for now :( -import bb.parse -import bb.data - -class ReferenceTest(unittest.TestCase): - def setUp(self): - self.d = bb.data.init() - - def setEmptyVars(self, varlist): - for k in varlist: - self.d.setVar(k, "") - - def setValues(self, values): - for k, v in values.items(): - self.d.setVar(k, v) - - def assertReferences(self, refs): - self.assertEqual(self.references, refs) - - def assertExecs(self, execs): - self.assertEqual(self.execs, execs) - -class VariableReferenceTest(ReferenceTest): - - def parseExpression(self, exp): - parsedvar = self.d.expandWithRefs(exp, None) - self.references = parsedvar.references - - def test_simple_reference(self): - self.setEmptyVars(["FOO"]) - self.parseExpression("${FOO}") - self.assertReferences(set(["FOO"])) - - def test_nested_reference(self): - self.setEmptyVars(["BAR"]) - self.d.setVar("FOO", "BAR") - self.parseExpression("${${FOO}}") - self.assertReferences(set(["FOO", "BAR"])) - - def test_python_reference(self): - self.setEmptyVars(["BAR"]) - self.parseExpression("${@bb.data.getVar('BAR', d, True) + 'foo'}") - self.assertReferences(set(["BAR"])) - -class ShellReferenceTest(ReferenceTest): - - def parseExpression(self, exp): - parsedvar = self.d.expandWithRefs(exp, None) - parser = bb.codeparser.ShellParser("ParserTest", logger) - parser.parse_shell(parsedvar.value) - - self.references = parsedvar.references - self.execs = parser.execs - - def test_quotes_inside_assign(self): - self.parseExpression('foo=foo"bar"baz') - self.assertReferences(set([])) - - def test_quotes_inside_arg(self): - self.parseExpression('sed s#"bar baz"#"alpha beta"#g') - self.assertExecs(set(["sed"])) - - def test_arg_continuation(self): - self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc") - self.assertExecs(set(["sed"])) - - def test_dollar_in_quoted(self): - self.parseExpression('sed -i -e "foo$" *.pc') - self.assertExecs(set(["sed"])) - - def test_quotes_inside_arg_continuation(self): - self.setEmptyVars(["bindir", "D", "libdir"]) - self.parseExpression(""" -sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\ --e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\ -${D}${libdir}/pkgconfig/*.pc -""") - self.assertReferences(set(["bindir", "D", "libdir"])) - - def test_assign_subshell_expansion(self): - self.parseExpression("foo=$(echo bar)") - self.assertExecs(set(["echo"])) - - def test_shell_unexpanded(self): - self.setEmptyVars(["QT_BASE_NAME"]) - self.parseExpression('echo "${QT_BASE_NAME}"') - self.assertExecs(set(["echo"])) - self.assertReferences(set(["QT_BASE_NAME"])) - - def test_incomplete_varexp_single_quotes(self): - self.parseExpression("sed -i -e 's:IP{:I${:g' $pc") - self.assertExecs(set(["sed"])) - - - def test_until(self): - self.parseExpression("until false; do echo true; done") - self.assertExecs(set(["false", "echo"])) - self.assertReferences(set()) - - def test_case(self): - self.parseExpression(""" -case $foo in -*) -bar -;; -esac -""") - self.assertExecs(set(["bar"])) - self.assertReferences(set()) - - def test_assign_exec(self): - self.parseExpression("a=b c='foo bar' alpha 1 2 3") - self.assertExecs(set(["alpha"])) - - def test_redirect_to_file(self): - self.setEmptyVars(["foo"]) - self.parseExpression("echo foo >${foo}/bar") - self.assertExecs(set(["echo"])) - self.assertReferences(set(["foo"])) - - def test_heredoc(self): - self.setEmptyVars(["theta"]) - self.parseExpression(""" -cat <${B}/cachedpaths -shadow_cv_maildir=${SHADOW_MAILDIR} -shadow_cv_mailfile=${SHADOW_MAILFILE} -shadow_cv_utmpdir=${SHADOW_UTMPDIR} -shadow_cv_logdir=${SHADOW_LOGDIR} -shadow_cv_passwd_dir=${bindir} -END -""") - self.assertReferences(set(v)) - self.assertExecs(set(["cat"])) - -# def test_incomplete_command_expansion(self): -# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs, -# bbvalue.shparse("cp foo`", self.d), self.d) - -# def test_rogue_dollarsign(self): -# self.setValues({"D" : "/tmp"}) -# self.parseExpression("install -d ${D}$") -# self.assertReferences(set(["D"])) -# self.assertExecs(set(["install"])) - - -class PythonReferenceTest(ReferenceTest): - - def setUp(self): - self.d = bb.data.init() - if hasattr(bb.utils, "_context"): - self.context = bb.utils._context - else: - import __builtin__ - self.context = __builtin__.__dict__ - - def parseExpression(self, exp): - parsedvar = self.d.expandWithRefs(exp, None) - parser = bb.codeparser.PythonParser("ParserTest", logger) - parser.parse_python(parsedvar.value) - - self.references = parsedvar.references | parser.references - self.execs = parser.execs - - @staticmethod - def indent(value): - """Python Snippets have to be indented, python values don't have to -be. These unit tests are testing snippets.""" - return " " + value - - def test_getvar_reference(self): - self.parseExpression("bb.data.getVar('foo', d, True)") - self.assertReferences(set(["foo"])) - self.assertExecs(set()) - - def test_getvar_computed_reference(self): - self.parseExpression("bb.data.getVar('f' + 'o' + 'o', d, True)") - self.assertReferences(set()) - self.assertExecs(set()) - - def test_getvar_exec_reference(self): - self.parseExpression("eval('bb.data.getVar(\"foo\", d, True)')") - self.assertReferences(set()) - self.assertExecs(set(["eval"])) - - def test_var_reference(self): - self.context["foo"] = lambda x: x - self.setEmptyVars(["FOO"]) - self.parseExpression("foo('${FOO}')") - self.assertReferences(set(["FOO"])) - self.assertExecs(set(["foo"])) - del self.context["foo"] - - def test_var_exec(self): - for etype in ("func", "task"): - self.d.setVar("do_something", "echo 'hi mom! ${FOO}'") - self.d.setVarFlag("do_something", etype, True) - self.parseExpression("bb.build.exec_func('do_something', d)") - self.assertReferences(set(["do_something"])) - - def test_function_reference(self): - self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg) - self.d.setVar("FOO", "Hello, World!") - self.parseExpression("testfunc('${FOO}')") - self.assertReferences(set(["FOO"])) - self.assertExecs(set(["testfunc"])) - del self.context["testfunc"] - - def test_qualified_function_reference(self): - self.parseExpression("time.time()") - self.assertExecs(set(["time.time"])) - - def test_qualified_function_reference_2(self): - self.parseExpression("os.path.dirname('/foo/bar')") - self.assertExecs(set(["os.path.dirname"])) - - def test_qualified_function_reference_nested(self): - self.parseExpression("time.strftime('%Y%m%d',time.gmtime())") - self.assertExecs(set(["time.strftime", "time.gmtime"])) - - def test_function_reference_chained(self): - self.context["testget"] = lambda: "\tstrip me " - self.parseExpression("testget().strip()") - self.assertExecs(set(["testget"])) - del self.context["testget"] - - -class DependencyReferenceTest(ReferenceTest): - - pydata = """ -bb.data.getVar('somevar', d, True) -def test(d): - foo = 'bar %s' % 'foo' -def test2(d): - d.getVar(foo, True) - d.getVar('bar', False) - test2(d) - -def a(): - \"\"\"some - stuff - \"\"\" - return "heh" - -test(d) - -bb.data.expand(bb.data.getVar("something", False, d), d) -bb.data.expand("${inexpand} somethingelse", d) -bb.data.getVar(a(), d, False) -""" - - def test_python(self): - self.d.setVar("FOO", self.pydata) - self.setEmptyVars(["inexpand", "a", "test2", "test"]) - self.d.setVarFlags("FOO", {"func": True, "python": True}) - - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) - - self.assertEquals(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) - - - shelldata = """ -foo () { -bar -} -{ -echo baz -$(heh) -eval `moo` -} -a=b -c=d -( -true && false -test -f foo -testval=something -$testval -) || aiee -! inverted -echo ${somevar} - -case foo in -bar) -echo bar -;; -baz) -echo baz -;; -foo*) -echo foo -;; -esac -""" - - def test_shell(self): - execs = ["bar", "echo", "heh", "moo", "true", "aiee"] - self.d.setVar("somevar", "heh") - self.d.setVar("inverted", "echo inverted...") - self.d.setVarFlag("inverted", "func", True) - self.d.setVar("FOO", self.shelldata) - self.d.setVarFlags("FOO", {"func": True}) - self.setEmptyVars(execs) - - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) - - self.assertEquals(deps, set(["somevar", "inverted"] + execs)) - - - def test_vardeps(self): - self.d.setVar("oe_libinstall", "echo test") - self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") - self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") - - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) - - self.assertEquals(deps, set(["oe_libinstall"])) - - def test_vardeps_expand(self): - self.d.setVar("oe_libinstall", "echo test") - self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") - self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") - - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) - - self.assertEquals(deps, set(["oe_libinstall"])) - - #Currently no wildcard support - #def test_vardeps_wildcards(self): - # self.d.setVar("oe_libinstall", "echo test") - # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") - # self.d.setVarFlag("FOO", "vardeps", "oe_*") - # self.assertEquals(deps, set(["oe_libinstall"])) - - From 4e4a83c03585f5088c8c63cd3be258c91a44da79 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Wed, 13 Nov 2013 12:34:30 +0100 Subject: [PATCH 13/19] added codeparser --- lib/bb/tests/test_codeparser.py | 374 ++++++++++++++++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 lib/bb/tests/test_codeparser.py diff --git a/lib/bb/tests/test_codeparser.py b/lib/bb/tests/test_codeparser.py new file mode 100644 index 00000000000..938b04b2c60 --- /dev/null +++ b/lib/bb/tests/test_codeparser.py @@ -0,0 +1,374 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Test for codeparser.py +# +# Copyright (C) 2010 Chris Larson +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import logging +import bb + +logger = logging.getLogger('BitBake.TestCodeParser') + +# bb.data references bb.parse but can't directly import due to circular dependencies. +# Hack around it for now :( +import bb.parse +import bb.data + +class ReferenceTest(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + + def setEmptyVars(self, varlist): + for k in varlist: + self.d.setVar(k, "") + + def setValues(self, values): + for k, v in values.items(): + self.d.setVar(k, v) + + def assertReferences(self, refs): + self.assertEqual(self.references, refs) + + def assertExecs(self, execs): + self.assertEqual(self.execs, execs) + +class VariableReferenceTest(ReferenceTest): + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + self.references = parsedvar.references + + def test_simple_reference(self): + self.setEmptyVars(["FOO"]) + self.parseExpression("${FOO}") + self.assertReferences(set(["FOO"])) + + def test_nested_reference(self): + self.setEmptyVars(["BAR"]) + self.d.setVar("FOO", "BAR") + self.parseExpression("${${FOO}}") + self.assertReferences(set(["FOO", "BAR"])) + + def test_python_reference(self): + self.setEmptyVars(["BAR"]) + self.parseExpression("${@bb.data.getVar('BAR', d, True) + 'foo'}") + self.assertReferences(set(["BAR"])) + +class ShellReferenceTest(ReferenceTest): + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + parser = bb.codeparser.ShellParser("ParserTest", logger) + parser.parse_shell(parsedvar.value) + + self.references = parsedvar.references + self.execs = parser.execs + + def test_quotes_inside_assign(self): + self.parseExpression('foo=foo"bar"baz') + self.assertReferences(set([])) + + def test_quotes_inside_arg(self): + self.parseExpression('sed s#"bar baz"#"alpha beta"#g') + self.assertExecs(set(["sed"])) + + def test_arg_continuation(self): + self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc") + self.assertExecs(set(["sed"])) + + def test_dollar_in_quoted(self): + self.parseExpression('sed -i -e "foo$" *.pc') + self.assertExecs(set(["sed"])) + + def test_quotes_inside_arg_continuation(self): + self.setEmptyVars(["bindir", "D", "libdir"]) + self.parseExpression(""" +sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\ +-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\ +${D}${libdir}/pkgconfig/*.pc +""") + self.assertReferences(set(["bindir", "D", "libdir"])) + + def test_assign_subshell_expansion(self): + self.parseExpression("foo=$(echo bar)") + self.assertExecs(set(["echo"])) + + def test_shell_unexpanded(self): + self.setEmptyVars(["QT_BASE_NAME"]) + self.parseExpression('echo "${QT_BASE_NAME}"') + self.assertExecs(set(["echo"])) + self.assertReferences(set(["QT_BASE_NAME"])) + + def test_incomplete_varexp_single_quotes(self): + self.parseExpression("sed -i -e 's:IP{:I${:g' $pc") + self.assertExecs(set(["sed"])) + + + def test_until(self): + self.parseExpression("until false; do echo true; done") + self.assertExecs(set(["false", "echo"])) + self.assertReferences(set()) + + def test_case(self): + self.parseExpression(""" +case $foo in +*) +bar +;; +esac +""") + self.assertExecs(set(["bar"])) + self.assertReferences(set()) + + def test_assign_exec(self): + self.parseExpression("a=b c='foo bar' alpha 1 2 3") + self.assertExecs(set(["alpha"])) + + def test_redirect_to_file(self): + self.setEmptyVars(["foo"]) + self.parseExpression("echo foo >${foo}/bar") + self.assertExecs(set(["echo"])) + self.assertReferences(set(["foo"])) + + def test_heredoc(self): + self.setEmptyVars(["theta"]) + self.parseExpression(""" +cat <${B}/cachedpaths +shadow_cv_maildir=${SHADOW_MAILDIR} +shadow_cv_mailfile=${SHADOW_MAILFILE} +shadow_cv_utmpdir=${SHADOW_UTMPDIR} +shadow_cv_logdir=${SHADOW_LOGDIR} +shadow_cv_passwd_dir=${bindir} +END +""") + self.assertReferences(set(v)) + self.assertExecs(set(["cat"])) + +# def test_incomplete_command_expansion(self): +# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs, +# bbvalue.shparse("cp foo`", self.d), self.d) + +# def test_rogue_dollarsign(self): +# self.setValues({"D" : "/tmp"}) +# self.parseExpression("install -d ${D}$") +# self.assertReferences(set(["D"])) +# self.assertExecs(set(["install"])) + + +class PythonReferenceTest(ReferenceTest): + + def setUp(self): + self.d = bb.data.init() + if hasattr(bb.utils, "_context"): + self.context = bb.utils._context + else: + import __builtin__ + self.context = __builtin__.__dict__ + + def parseExpression(self, exp): + parsedvar = self.d.expandWithRefs(exp, None) + parser = bb.codeparser.PythonParser("ParserTest", logger) + parser.parse_python(parsedvar.value) + + self.references = parsedvar.references | parser.references + self.execs = parser.execs + + @staticmethod + def indent(value): + """Python Snippets have to be indented, python values don't have to +be. These unit tests are testing snippets.""" + return " " + value + + def test_getvar_reference(self): + self.parseExpression("bb.data.getVar('foo', d, True)") + self.assertReferences(set(["foo"])) + self.assertExecs(set()) + + def test_getvar_computed_reference(self): + self.parseExpression("bb.data.getVar('f' + 'o' + 'o', d, True)") + self.assertReferences(set()) + self.assertExecs(set()) + + def test_getvar_exec_reference(self): + self.parseExpression("eval('bb.data.getVar(\"foo\", d, True)')") + self.assertReferences(set()) + self.assertExecs(set(["eval"])) + + def test_var_reference(self): + self.context["foo"] = lambda x: x + self.setEmptyVars(["FOO"]) + self.parseExpression("foo('${FOO}')") + self.assertReferences(set(["FOO"])) + self.assertExecs(set(["foo"])) + del self.context["foo"] + + def test_var_exec(self): + for etype in ("func", "task"): + self.d.setVar("do_something", "echo 'hi mom! ${FOO}'") + self.d.setVarFlag("do_something", etype, True) + self.parseExpression("bb.build.exec_func('do_something', d)") + self.assertReferences(set(["do_something"])) + + def test_function_reference(self): + self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg) + self.d.setVar("FOO", "Hello, World!") + self.parseExpression("testfunc('${FOO}')") + self.assertReferences(set(["FOO"])) + self.assertExecs(set(["testfunc"])) + del self.context["testfunc"] + + def test_qualified_function_reference(self): + self.parseExpression("time.time()") + self.assertExecs(set(["time.time"])) + + def test_qualified_function_reference_2(self): + self.parseExpression("os.path.dirname('/foo/bar')") + self.assertExecs(set(["os.path.dirname"])) + + def test_qualified_function_reference_nested(self): + self.parseExpression("time.strftime('%Y%m%d',time.gmtime())") + self.assertExecs(set(["time.strftime", "time.gmtime"])) + + def test_function_reference_chained(self): + self.context["testget"] = lambda: "\tstrip me " + self.parseExpression("testget().strip()") + self.assertExecs(set(["testget"])) + del self.context["testget"] + + +class DependencyReferenceTest(ReferenceTest): + + pydata = """ +bb.data.getVar('somevar', d, True) +def test(d): + foo = 'bar %s' % 'foo' +def test2(d): + d.getVar(foo, True) + d.getVar('bar', False) + test2(d) + +def a(): + \"\"\"some + stuff + \"\"\" + return "heh" + +test(d) + +bb.data.expand(bb.data.getVar("something", False, d), d) +bb.data.expand("${inexpand} somethingelse", d) +bb.data.getVar(a(), d, False) +""" + + def test_python(self): + self.d.setVar("FOO", self.pydata) + self.setEmptyVars(["inexpand", "a", "test2", "test"]) + self.d.setVarFlags("FOO", {"func": True, "python": True}) + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) + + + shelldata = """ +foo () { +bar +} +{ +echo baz +$(heh) +eval `moo` +} +a=b +c=d +( +true && false +test -f foo +testval=something +$testval +) || aiee +! inverted +echo ${somevar} + +case foo in +bar) +echo bar +;; +baz) +echo baz +;; +foo*) +echo foo +;; +esac +""" + + def test_shell(self): + execs = ["bar", "echo", "heh", "moo", "true", "aiee"] + self.d.setVar("somevar", "heh") + self.d.setVar("inverted", "echo inverted...") + self.d.setVarFlag("inverted", "func", True) + self.d.setVar("FOO", self.shelldata) + self.d.setVarFlags("FOO", {"func": True}) + self.setEmptyVars(execs) + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["somevar", "inverted"] + execs)) + + + def test_vardeps(self): + self.d.setVar("oe_libinstall", "echo test") + self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["oe_libinstall"])) + + def test_vardeps_expand(self): + self.d.setVar("oe_libinstall", "echo test") + self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") + + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + + self.assertEquals(deps, set(["oe_libinstall"])) + + #Currently no wildcard support + #def test_vardeps_wildcards(self): + # self.d.setVar("oe_libinstall", "echo test") + # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") + # self.d.setVarFlag("FOO", "vardeps", "oe_*") + # self.assertEquals(deps, set(["oe_libinstall"])) + + From 896d590e989c1bfbc0dfd12a06258a8c2c7fc052 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Wed, 13 Nov 2013 16:32:54 +0100 Subject: [PATCH 14/19] fix issue with nose option --with-coverage --- lib/bb/data_smart.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py index fbcfd311f27..6b66cfc4413 100644 --- a/lib/bb/data_smart.py +++ b/lib/bb/data_smart.py @@ -117,7 +117,14 @@ def var_sub(self, match): def python_sub(self, match): code = match.group()[3:-1] - codeobj = compile(code.strip(), self.varname or "", "eval") + # compile builtin: compile(source, filename, mode[, flags[, dont_inherit]]) + # Compile the source into a code or AST object + # The filename argument is the file name from which the code was read; + # pass some recognizable value if it wasn't read from a file: '' + # The mode argument specifies what kind of code must be compiled + # ('eval' if it consists of a single expression) + # Use when to be able to run with nose option '--with-coverage' + codeobj = compile(code.strip(), "", "eval") parser = bb.codeparser.PythonParser(self.varname, logger) parser.parse_python(code) @@ -133,6 +140,9 @@ def python_sub(self, match): value = utils.better_eval(codeobj, DataContext(self.d)) return str(value) + def __str__(self): + return str(self.varname) + " : " + str(self.value) + class DataContext(dict): @@ -356,7 +366,9 @@ def expandWithRefs(self, s, varname): while s.find('${') != -1: olds = s try: + # Variable substitution: ${FOO} s = __expand_var_regexp__.sub(varparse.var_sub, s) + # Python expression substitution: ${@} s = __expand_python_regexp__.sub(varparse.python_sub, s) if s == olds: break From ae4f1f1364641e74bddece88cb86a3663437d059 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 14 Nov 2013 10:17:53 +0100 Subject: [PATCH 15/19] add test_utils --- lib/bb/tests/utils.py | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 lib/bb/tests/utils.py diff --git a/lib/bb/tests/utils.py b/lib/bb/tests/utils.py deleted file mode 100644 index 7c50b1d7867..00000000000 --- a/lib/bb/tests/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Tests for utils.py -# -# Copyright (C) 2012 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import unittest -import bb - -class VerCmpString(unittest.TestCase): - - def test_vercmpstring(self): - result = bb.utils.vercmp_string('1', '2') - self.assertTrue(result < 0) - result = bb.utils.vercmp_string('2', '1') - self.assertTrue(result > 0) - result = bb.utils.vercmp_string('1', '1.0') - self.assertTrue(result < 0) - result = bb.utils.vercmp_string('1', '1.1') - self.assertTrue(result < 0) - result = bb.utils.vercmp_string('1.1', '1_p2') - self.assertTrue(result < 0) - - def test_explode_dep_versions(self): - correctresult = {"foo" : ["= 1.10"]} - result = bb.utils.explode_dep_versions2("foo (= 1.10)") - self.assertEqual(result, correctresult) - result = bb.utils.explode_dep_versions2("foo (=1.10)") - self.assertEqual(result, correctresult) - result = bb.utils.explode_dep_versions2("foo ( = 1.10)") - self.assertEqual(result, correctresult) - result = bb.utils.explode_dep_versions2("foo ( =1.10)") - self.assertEqual(result, correctresult) - result = bb.utils.explode_dep_versions2("foo ( = 1.10 )") - self.assertEqual(result, correctresult) - result = bb.utils.explode_dep_versions2("foo ( =1.10 )") - self.assertEqual(result, correctresult) - From 98ed07e36725a731c0d81d1fb566dac1bc2b3f69 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 14 Nov 2013 10:17:59 +0100 Subject: [PATCH 16/19] add test_utils --- lib/bb/tests/test_utils.py | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 lib/bb/tests/test_utils.py diff --git a/lib/bb/tests/test_utils.py b/lib/bb/tests/test_utils.py new file mode 100644 index 00000000000..7c50b1d7867 --- /dev/null +++ b/lib/bb/tests/test_utils.py @@ -0,0 +1,53 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for utils.py +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import bb + +class VerCmpString(unittest.TestCase): + + def test_vercmpstring(self): + result = bb.utils.vercmp_string('1', '2') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('2', '1') + self.assertTrue(result > 0) + result = bb.utils.vercmp_string('1', '1.0') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1', '1.1') + self.assertTrue(result < 0) + result = bb.utils.vercmp_string('1.1', '1_p2') + self.assertTrue(result < 0) + + def test_explode_dep_versions(self): + correctresult = {"foo" : ["= 1.10"]} + result = bb.utils.explode_dep_versions2("foo (= 1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo (=1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( = 1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( =1.10)") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( = 1.10 )") + self.assertEqual(result, correctresult) + result = bb.utils.explode_dep_versions2("foo ( =1.10 )") + self.assertEqual(result, correctresult) + From 28474d2bc3550b398212b35770813ba3d5401bfa Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 14 Nov 2013 14:29:23 +0100 Subject: [PATCH 17/19] converted fetch test to nose --- lib/bb/tests/fetch.py | 424 ------------------------------------------ 1 file changed, 424 deletions(-) delete mode 100644 lib/bb/tests/fetch.py diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py deleted file mode 100644 index 4bcff543fc9..00000000000 --- a/lib/bb/tests/fetch.py +++ /dev/null @@ -1,424 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Tests for the Fetcher (fetch2/) -# -# Copyright (C) 2012 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import unittest -import tempfile -import subprocess -import os -from bb.fetch2 import URI -import bb - -class URITest(unittest.TestCase): - test_uris = { - "http://www.google.com/index.html" : { - 'uri': 'http://www.google.com/index.html', - 'scheme': 'http', - 'hostname': 'www.google.com', - 'port': None, - 'hostport': 'www.google.com', - 'path': '/index.html', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': False - }, - "http://www.google.com/index.html;param1=value1" : { - 'uri': 'http://www.google.com/index.html;param1=value1', - 'scheme': 'http', - 'hostname': 'www.google.com', - 'port': None, - 'hostport': 'www.google.com', - 'path': '/index.html', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': { - 'param1': 'value1' - }, - 'relative': False - }, - "http://www.example.com:8080/index.html" : { - 'uri': 'http://www.example.com:8080/index.html', - 'scheme': 'http', - 'hostname': 'www.example.com', - 'port': 8080, - 'hostport': 'www.example.com:8080', - 'path': '/index.html', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': False - }, - "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { - 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', - 'scheme': 'cvs', - 'hostname': 'cvs.handhelds.org', - 'port': None, - 'hostport': 'cvs.handhelds.org', - 'path': '/cvs', - 'userinfo': 'anoncvs', - 'username': 'anoncvs', - 'password': '', - 'params': { - 'module': 'familiar/dist/ipkg' - }, - 'relative': False - }, - "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { - 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', - 'scheme': 'cvs', - 'hostname': 'cvs.handhelds.org', - 'port': None, - 'hostport': 'cvs.handhelds.org', - 'path': '/cvs', - 'userinfo': 'anoncvs:anonymous', - 'username': 'anoncvs', - 'password': 'anonymous', - 'params': { - 'tag': 'V0-99-81', - 'module': 'familiar/dist/ipkg' - }, - 'relative': False - }, - "file://example.diff": { # NOTE: Not RFC compliant! - 'uri': 'file:example.diff', - 'scheme': 'file', - 'hostname': '', - 'port': None, - 'hostport': '', - 'path': 'example.diff', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': True - }, - "file:example.diff": { # NOTE: RFC compliant version of the former - 'uri': 'file:example.diff', - 'scheme': 'file', - 'hostname': '', - 'port': None, - 'hostport': '', - 'path': 'example.diff', - 'userinfo': '', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': True - }, - "file:///tmp/example.diff": { - 'uri': 'file:///tmp/example.diff', - 'scheme': 'file', - 'hostname': '', - 'port': None, - 'hostport': '', - 'path': '/tmp/example.diff', - 'userinfo': '', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': False - }, - "git:///path/example.git": { - 'uri': 'git:///path/example.git', - 'scheme': 'git', - 'hostname': '', - 'port': None, - 'hostport': '', - 'path': '/path/example.git', - 'userinfo': '', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': False - }, - "git:path/example.git": { - 'uri': 'git:path/example.git', - 'scheme': 'git', - 'hostname': '', - 'port': None, - 'hostport': '', - 'path': 'path/example.git', - 'userinfo': '', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': True - }, - "git://example.net/path/example.git": { - 'uri': 'git://example.net/path/example.git', - 'scheme': 'git', - 'hostname': 'example.net', - 'port': None, - 'hostport': 'example.net', - 'path': '/path/example.git', - 'userinfo': '', - 'userinfo': '', - 'username': '', - 'password': '', - 'params': {}, - 'relative': False - } - } - - def test_uri(self): - for test_uri, ref in self.test_uris.items(): - uri = URI(test_uri) - - self.assertEqual(str(uri), ref['uri']) - - # expected attributes - self.assertEqual(uri.scheme, ref['scheme']) - - self.assertEqual(uri.userinfo, ref['userinfo']) - self.assertEqual(uri.username, ref['username']) - self.assertEqual(uri.password, ref['password']) - - self.assertEqual(uri.hostname, ref['hostname']) - self.assertEqual(uri.port, ref['port']) - self.assertEqual(uri.hostport, ref['hostport']) - - self.assertEqual(uri.path, ref['path']) - self.assertEqual(uri.params, ref['params']) - - self.assertEqual(uri.relative, ref['relative']) - - def test_dict(self): - for test in self.test_uris.values(): - uri = URI() - - self.assertEqual(uri.scheme, '') - self.assertEqual(uri.userinfo, '') - self.assertEqual(uri.username, '') - self.assertEqual(uri.password, '') - self.assertEqual(uri.hostname, '') - self.assertEqual(uri.port, None) - self.assertEqual(uri.path, '') - self.assertEqual(uri.params, {}) - - - uri.scheme = test['scheme'] - self.assertEqual(uri.scheme, test['scheme']) - - uri.userinfo = test['userinfo'] - self.assertEqual(uri.userinfo, test['userinfo']) - self.assertEqual(uri.username, test['username']) - self.assertEqual(uri.password, test['password']) - - uri.hostname = test['hostname'] - self.assertEqual(uri.hostname, test['hostname']) - self.assertEqual(uri.hostport, test['hostname']) - - uri.port = test['port'] - self.assertEqual(uri.port, test['port']) - self.assertEqual(uri.hostport, test['hostport']) - - uri.path = test['path'] - self.assertEqual(uri.path, test['path']) - - uri.params = test['params'] - self.assertEqual(uri.params, test['params']) - - self.assertEqual(str(uri)+str(uri.relative), str(test['uri'])+str(test['relative'])) - - self.assertEqual(str(uri), test['uri']) - - uri.params = {} - self.assertEqual(uri.params, {}) - self.assertEqual(str(uri), (str(uri).split(";"))[0]) - -class FetcherTest(unittest.TestCase): - - def setUp(self): - self.d = bb.data.init() - self.tempdir = tempfile.mkdtemp() - self.dldir = os.path.join(self.tempdir, "download") - os.mkdir(self.dldir) - self.d.setVar("DL_DIR", self.dldir) - self.unpackdir = os.path.join(self.tempdir, "unpacked") - os.mkdir(self.unpackdir) - persistdir = os.path.join(self.tempdir, "persistdata") - self.d.setVar("PERSISTENT_DIR", persistdir) - - def tearDown(self): - bb.utils.prunedir(self.tempdir) - -class MirrorUriTest(FetcherTest): - - replaceuris = { - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") - : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") - : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") - : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", - ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") - : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", - ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") - : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", - ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") - : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", - ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") - : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", - ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") - : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", - ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") - : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") - : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - - #Renaming files doesn't work - #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" - #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", - } - - mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \ - "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \ - "https://.*/.* file:///someotherpath/downloads/ \n" \ - "http://.*/.* file:///someotherpath/downloads/ \n" - - def test_urireplace(self): - for k, v in self.replaceuris.items(): - ud = bb.fetch.FetchData(k[0], self.d) - ud.setup_localpath(self.d) - mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) - newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) - self.assertEqual([v], newuris) - - def test_urilist1(self): - fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) - mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) - uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) - self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) - - def test_urilist2(self): - # Catch https:// -> files:// bug - fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) - mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) - uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) - self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) - -class FetcherNetworkTest(FetcherTest): - - if os.environ.get("BB_SKIP_NETTESTS") == "yes": - print("Unset BB_SKIP_NETTESTS to run network tests") - else: - def test_fetch(self): - fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) - fetcher.download() - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) - self.d.setVar("BB_NO_NETWORK", "1") - fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) - fetcher.download() - fetcher.unpack(self.unpackdir) - self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) - self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) - - def test_fetch_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") - fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) - fetcher.download() - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) - - def test_fetch_premirror(self): - self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") - fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) - fetcher.download() - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) - - def gitfetcher(self, url1, url2): - def checkrevision(self, fetcher): - fetcher.unpack(self.unpackdir) - revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip() - self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") - - self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") - self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") - fetcher = bb.fetch.Fetch([url1], self.d) - fetcher.download() - checkrevision(self, fetcher) - # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works - bb.utils.prunedir(self.dldir + "/git2/") - bb.utils.prunedir(self.unpackdir) - self.d.setVar("BB_NO_NETWORK", "1") - fetcher = bb.fetch.Fetch([url2], self.d) - fetcher.download() - checkrevision(self, fetcher) - - def test_gitfetch(self): - url1 = url2 = "git://git.openembedded.org/bitbake" - self.gitfetcher(url1, url2) - - def test_gitfetch_premirror(self): - url1 = "git://git.openembedded.org/bitbake" - url2 = "git://someserver.org/bitbake" - self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") - self.gitfetcher(url1, url2) - - def test_gitfetch_premirror2(self): - url1 = url2 = "git://someserver.org/bitbake" - self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") - self.gitfetcher(url1, url2) - - def test_gitfetch_premirror3(self): - realurl = "git://git.openembedded.org/bitbake" - dummyurl = "git://someserver.org/bitbake" - self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") - os.chdir(self.tempdir) - bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) - self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir)) - self.gitfetcher(dummyurl, dummyurl) - -class URLHandle(unittest.TestCase): - - datatable = { - "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), - "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), - "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}) - } - - def test_decodeurl(self): - for k, v in self.datatable.items(): - result = bb.fetch.decodeurl(k) - self.assertEqual(result, v) - - def test_encodeurl(self): - for k, v in self.datatable.items(): - result = bb.fetch.encodeurl(v) - self.assertEqual(result, k) - - - From 40fabe8cae6633956be8dff63def5d996a37a644 Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 14 Nov 2013 14:29:30 +0100 Subject: [PATCH 18/19] converted fetch test to nose --- lib/bb/tests/test_fetch.py | 424 +++++++++++++++++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 lib/bb/tests/test_fetch.py diff --git a/lib/bb/tests/test_fetch.py b/lib/bb/tests/test_fetch.py new file mode 100644 index 00000000000..4bcff543fc9 --- /dev/null +++ b/lib/bb/tests/test_fetch.py @@ -0,0 +1,424 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Tests for the Fetcher (fetch2/) +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import unittest +import tempfile +import subprocess +import os +from bb.fetch2 import URI +import bb + +class URITest(unittest.TestCase): + test_uris = { + "http://www.google.com/index.html" : { + 'uri': 'http://www.google.com/index.html', + 'scheme': 'http', + 'hostname': 'www.google.com', + 'port': None, + 'hostport': 'www.google.com', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': False + }, + "http://www.google.com/index.html;param1=value1" : { + 'uri': 'http://www.google.com/index.html;param1=value1', + 'scheme': 'http', + 'hostname': 'www.google.com', + 'port': None, + 'hostport': 'www.google.com', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': { + 'param1': 'value1' + }, + 'relative': False + }, + "http://www.example.com:8080/index.html" : { + 'uri': 'http://www.example.com:8080/index.html', + 'scheme': 'http', + 'hostname': 'www.example.com', + 'port': 8080, + 'hostport': 'www.example.com:8080', + 'path': '/index.html', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': False + }, + "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { + 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', + 'scheme': 'cvs', + 'hostname': 'cvs.handhelds.org', + 'port': None, + 'hostport': 'cvs.handhelds.org', + 'path': '/cvs', + 'userinfo': 'anoncvs', + 'username': 'anoncvs', + 'password': '', + 'params': { + 'module': 'familiar/dist/ipkg' + }, + 'relative': False + }, + "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { + 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', + 'scheme': 'cvs', + 'hostname': 'cvs.handhelds.org', + 'port': None, + 'hostport': 'cvs.handhelds.org', + 'path': '/cvs', + 'userinfo': 'anoncvs:anonymous', + 'username': 'anoncvs', + 'password': 'anonymous', + 'params': { + 'tag': 'V0-99-81', + 'module': 'familiar/dist/ipkg' + }, + 'relative': False + }, + "file://example.diff": { # NOTE: Not RFC compliant! + 'uri': 'file:example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'example.diff', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': True + }, + "file:example.diff": { # NOTE: RFC compliant version of the former + 'uri': 'file:example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'example.diff', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': True + }, + "file:///tmp/example.diff": { + 'uri': 'file:///tmp/example.diff', + 'scheme': 'file', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': '/tmp/example.diff', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': False + }, + "git:///path/example.git": { + 'uri': 'git:///path/example.git', + 'scheme': 'git', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': '/path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': False + }, + "git:path/example.git": { + 'uri': 'git:path/example.git', + 'scheme': 'git', + 'hostname': '', + 'port': None, + 'hostport': '', + 'path': 'path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': True + }, + "git://example.net/path/example.git": { + 'uri': 'git://example.net/path/example.git', + 'scheme': 'git', + 'hostname': 'example.net', + 'port': None, + 'hostport': 'example.net', + 'path': '/path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'relative': False + } + } + + def test_uri(self): + for test_uri, ref in self.test_uris.items(): + uri = URI(test_uri) + + self.assertEqual(str(uri), ref['uri']) + + # expected attributes + self.assertEqual(uri.scheme, ref['scheme']) + + self.assertEqual(uri.userinfo, ref['userinfo']) + self.assertEqual(uri.username, ref['username']) + self.assertEqual(uri.password, ref['password']) + + self.assertEqual(uri.hostname, ref['hostname']) + self.assertEqual(uri.port, ref['port']) + self.assertEqual(uri.hostport, ref['hostport']) + + self.assertEqual(uri.path, ref['path']) + self.assertEqual(uri.params, ref['params']) + + self.assertEqual(uri.relative, ref['relative']) + + def test_dict(self): + for test in self.test_uris.values(): + uri = URI() + + self.assertEqual(uri.scheme, '') + self.assertEqual(uri.userinfo, '') + self.assertEqual(uri.username, '') + self.assertEqual(uri.password, '') + self.assertEqual(uri.hostname, '') + self.assertEqual(uri.port, None) + self.assertEqual(uri.path, '') + self.assertEqual(uri.params, {}) + + + uri.scheme = test['scheme'] + self.assertEqual(uri.scheme, test['scheme']) + + uri.userinfo = test['userinfo'] + self.assertEqual(uri.userinfo, test['userinfo']) + self.assertEqual(uri.username, test['username']) + self.assertEqual(uri.password, test['password']) + + uri.hostname = test['hostname'] + self.assertEqual(uri.hostname, test['hostname']) + self.assertEqual(uri.hostport, test['hostname']) + + uri.port = test['port'] + self.assertEqual(uri.port, test['port']) + self.assertEqual(uri.hostport, test['hostport']) + + uri.path = test['path'] + self.assertEqual(uri.path, test['path']) + + uri.params = test['params'] + self.assertEqual(uri.params, test['params']) + + self.assertEqual(str(uri)+str(uri.relative), str(test['uri'])+str(test['relative'])) + + self.assertEqual(str(uri), test['uri']) + + uri.params = {} + self.assertEqual(uri.params, {}) + self.assertEqual(str(uri), (str(uri).split(";"))[0]) + +class FetcherTest(unittest.TestCase): + + def setUp(self): + self.d = bb.data.init() + self.tempdir = tempfile.mkdtemp() + self.dldir = os.path.join(self.tempdir, "download") + os.mkdir(self.dldir) + self.d.setVar("DL_DIR", self.dldir) + self.unpackdir = os.path.join(self.tempdir, "unpacked") + os.mkdir(self.unpackdir) + persistdir = os.path.join(self.tempdir, "persistdata") + self.d.setVar("PERSISTENT_DIR", persistdir) + + def tearDown(self): + bb.utils.prunedir(self.tempdir) + +class MirrorUriTest(FetcherTest): + + replaceuris = { + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") + : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") + : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") + : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") + : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", + ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") + : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") + : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + + #Renaming files doesn't work + #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" + #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + } + + mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \ + "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \ + "https://.*/.* file:///someotherpath/downloads/ \n" \ + "http://.*/.* file:///someotherpath/downloads/ \n" + + def test_urireplace(self): + for k, v in self.replaceuris.items(): + ud = bb.fetch.FetchData(k[0], self.d) + ud.setup_localpath(self.d) + mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) + newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) + self.assertEqual([v], newuris) + + def test_urilist1(self): + fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) + + def test_urilist2(self): + # Catch https:// -> files:// bug + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) + +class FetcherNetworkTest(FetcherTest): + + if os.environ.get("BB_SKIP_NETTESTS") == "yes": + print("Unset BB_SKIP_NETTESTS to run network tests") + else: + def test_fetch(self): + fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) + self.d.setVar("BB_NO_NETWORK", "1") + fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) + self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) + + def test_fetch_mirror(self): + self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def test_fetch_premirror(self): + self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + def gitfetcher(self, url1, url2): + def checkrevision(self, fetcher): + fetcher.unpack(self.unpackdir) + revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip() + self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") + + self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") + self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") + fetcher = bb.fetch.Fetch([url1], self.d) + fetcher.download() + checkrevision(self, fetcher) + # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works + bb.utils.prunedir(self.dldir + "/git2/") + bb.utils.prunedir(self.unpackdir) + self.d.setVar("BB_NO_NETWORK", "1") + fetcher = bb.fetch.Fetch([url2], self.d) + fetcher.download() + checkrevision(self, fetcher) + + def test_gitfetch(self): + url1 = url2 = "git://git.openembedded.org/bitbake" + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror(self): + url1 = "git://git.openembedded.org/bitbake" + url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror2(self): + url1 = url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror3(self): + realurl = "git://git.openembedded.org/bitbake" + dummyurl = "git://someserver.org/bitbake" + self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") + os.chdir(self.tempdir) + bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) + self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir)) + self.gitfetcher(dummyurl, dummyurl) + +class URLHandle(unittest.TestCase): + + datatable = { + "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), + "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), + "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}) + } + + def test_decodeurl(self): + for k, v in self.datatable.items(): + result = bb.fetch.decodeurl(k) + self.assertEqual(result, v) + + def test_encodeurl(self): + for k, v in self.datatable.items(): + result = bb.fetch.encodeurl(v) + self.assertEqual(result, k) + + + From bba00d1dd4ced31098ac6c252c1035f24732285a Mon Sep 17 00:00:00 2001 From: Fabrice Coulon Date: Thu, 14 Nov 2013 14:52:30 +0100 Subject: [PATCH 19/19] use nose in bitbake-selftest --- bin/bitbake-nose-selftest | 63 --------------------------------------- bin/bitbake-selftest | 44 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 79 deletions(-) delete mode 100755 bin/bitbake-nose-selftest diff --git a/bin/bitbake-nose-selftest b/bin/bitbake-nose-selftest deleted file mode 100755 index c7d286b8861..00000000000 --- a/bin/bitbake-nose-selftest +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2012 Richard Purdie -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import sys -import nose -#sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) -sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + '/lib') - -if False: - import inspect - import __builtin__ - - print sys.path - cdir = os.path.abspath(os.path.curdir) - for d in sys.path: - if cdir in d: - print d - - savimp = __builtin__.__import__ - - def newimp(name, *x, **xx): - caller = inspect.currentframe().f_back - print "module: [%s]" % name, "caller: [%s]" % caller.f_globals.get('__name__') - return savimp(name, *x, **xx) - - __builtin__.__import__ = newimp - -#sys.exit(0) -#import unittest -#try: - #import bb -#except RuntimeError as exc: - #sys.exit(str(exc)) - -#tests = ["bb.tests.codeparser", - #"bb.tests.cow", - #"bb.tests.data", - #"bb.tests.fetch", - #"bb.tests.utils"] - -#for t in tests: - #__import__(t) - -#unittest.main(argv=["bitbake-selftest"] + tests) - - -if __name__ == '__main__': - nose.run(argv=sys.argv) diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest index 48a58fef677..6307613d3cd 100755 --- a/bin/bitbake-selftest +++ b/bin/bitbake-selftest @@ -15,24 +15,36 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os -import sys, logging -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) +""" + +Prerequirements: +% pip install nose +% pip install coverage + +* bitbake-selftest Usage: + + * Code coverage analysis -import unittest -try: - import bb -except RuntimeError as exc: - sys.exit(str(exc)) + Run all tests with coverage from bitbake top dir: + % ./bin/bitbake-selftest --with-coverage -tests = ["bb.tests.codeparser", - "bb.tests.cow", - "bb.tests.data", - "bb.tests.fetch", - "bb.tests.utils"] + Generate HTML coverage reports under htmlcov (default): + % coverage html && firefox htmlcov/index.html -for t in tests: - __import__(t) + * Has same options as nosetests -unittest.main(argv=["bitbake-selftest"] + tests) + Specify a specific unit testsuite: + % ./bin/bitbake-selftest -v lib/bb/tests/test_data.py + + Specify a specific testcase from a test suite: + % ./bin/bitbake-selftest -v lib/bb/tests/test_fetch.py:FetcherNetworkTest.test_gitfetch + +""" + +import os +import sys +import nose +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + '/lib') +if __name__ == '__main__': + nose.run(argv=sys.argv)