Initial commit

poky/meta/lib/oeqa/core/tests/cases/data.py

@@ -0,0 +1,23 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator import OETestTag
+from oeqa.core.decorator.data import OETestDataDepends
+
+class DataTest(OETestCase):
+    data_vars = ['IMAGE', 'ARCH']
+
+    @OETestDataDepends(['MACHINE',])
+    @OETestTag('dataTestOk')
+    def testDataOk(self):
+        self.assertEqual(self.td.get('IMAGE'), 'core-image-minimal')
+        self.assertEqual(self.td.get('ARCH'), 'x86')
+        self.assertEqual(self.td.get('MACHINE'), 'qemuarm')
+
+    @OETestTag('dataTestFail')
+    def testDataFail(self):
+        pass

poky/meta/lib/oeqa/core/tests/cases/depends.py

@@ -0,0 +1,41 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.depends import OETestDepends
+
+class DependsTest(OETestCase):
+
+    def testDependsFirst(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsFirst'])
+    def testDependsSecond(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsSecond'])
+    def testDependsThird(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsSecond'])
+    def testDependsFourth(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsThird', 'testDependsFourth'])
+    def testDependsFifth(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular3'])
+    def testDependsCircular1(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular1'])
+    def testDependsCircular2(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestDepends(['testDependsCircular2'])
+    def testDependsCircular3(self):
+        self.assertTrue(True, msg='How is this possible?')

poky/meta/lib/oeqa/core/tests/cases/loader/valid/another.py

@@ -0,0 +1,12 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+
+class AnotherTest(OETestCase):
+
+    def testAnother(self):
+        self.assertTrue(True, msg='How is this possible?')

poky/meta/lib/oeqa/core/tests/cases/oetag.py

@@ -0,0 +1,38 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator import OETestTag
+
+class TagTest(OETestCase):
+    @OETestTag('goodTag')
+    def testTagGood(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestTag('otherTag')
+    def testTagOther(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestTag('otherTag', 'multiTag')
+    def testTagOtherMulti(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testTagNone(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+@OETestTag('classTag')
+class TagClassTest(OETestCase):
+    @OETestTag('otherTag')
+    def testTagOther(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETestTag('otherTag', 'multiTag')
+    def testTagOtherMulti(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    def testTagNone(self):
+        self.assertTrue(True, msg='How is this possible?')
+

poky/meta/lib/oeqa/core/tests/cases/timeout.py

@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+from time import sleep
+
+from oeqa.core.case import OETestCase
+from oeqa.core.decorator.oetimeout import OETimeout
+from oeqa.core.decorator.depends import OETestDepends
+
+class TimeoutTest(OETestCase):
+
+    @OETimeout(1)
+    def testTimeoutPass(self):
+        self.assertTrue(True, msg='How is this possible?')
+
+    @OETimeout(1)
+    def testTimeoutFail(self):
+        sleep(2)
+        self.assertTrue(True, msg='How is this possible?')
+
+
+    def testTimeoutSkip(self):
+        self.skipTest("This test needs to be skipped, so that testTimeoutDepends()'s OETestDepends kicks in")
+
+    @OETestDepends(["timeout.TimeoutTest.testTimeoutSkip"])
+    @OETimeout(3)
+    def testTimeoutDepends(self):
+        self.assertTrue(False, msg='How is this possible?')
+
+    def testTimeoutUnrelated(self):
+        sleep(6)

poky/meta/lib/oeqa/core/tests/common.py

@@ -0,0 +1,38 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+import sys
+import os
+
+import unittest
+import logging
+import os
+
+logger = logging.getLogger("oeqa")
+logger.setLevel(logging.INFO)
+consoleHandler = logging.StreamHandler()
+formatter = logging.Formatter('OEQATest: %(message)s')
+consoleHandler.setFormatter(formatter)
+logger.addHandler(consoleHandler)
+
+def setup_sys_path():
+    directory = os.path.dirname(os.path.abspath(__file__))
+    oeqa_lib = os.path.realpath(os.path.join(directory, '../../../'))
+    if not oeqa_lib in sys.path:
+        sys.path.insert(0, oeqa_lib)
+
+class TestBase(unittest.TestCase):
+    def setUp(self):
+        self.logger = logger
+        directory = os.path.dirname(os.path.abspath(__file__))
+        self.cases_path = os.path.join(directory, 'cases')
+
+    def _testLoader(self, d={}, modules=[], tests=[], **kwargs):
+        from oeqa.core.context import OETestContext
+        tc = OETestContext(d, self.logger)
+        tc.loadTests(self.cases_path, modules=modules, tests=tests,
+                     **kwargs)
+        return tc

poky/meta/lib/oeqa/core/tests/test_data.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+import unittest
+import logging
+import os
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQAMissingVariable
+from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames
+
+class TestData(TestBase):
+    modules = ['data']
+
+    def test_data_fail_missing_variable(self):
+        expectedException = "oeqa.core.exception.OEQAMissingVariable"
+
+        tc = self._testLoader(modules=self.modules)
+        results = tc.runTests()
+        self.assertFalse(results.wasSuccessful())
+        for test, data in results.errors:
+            expect = False
+            if expectedException in data:
+                expect = True
+
+            self.assertTrue(expect)
+
+    def test_data_fail_wrong_variable(self):
+        expectedError = 'AssertionError'
+        d = {'IMAGE' : 'core-image-weston', 'ARCH' : 'arm'}
+
+        tc = self._testLoader(d=d, modules=self.modules)
+        results = tc.runTests()
+        self.assertFalse(results.wasSuccessful())
+        for test, data in results.failures:
+            expect = False
+            if expectedError in data:
+                expect = True
+
+            self.assertTrue(expect)
+
+    def test_data_ok(self):
+        d = {'IMAGE' : 'core-image-minimal', 'ARCH' : 'x86', 'MACHINE' : 'qemuarm'}
+
+        tc = self._testLoader(d=d, modules=self.modules)
+        self.assertEqual(True, tc.runTests().wasSuccessful())
+
+if __name__ == '__main__':
+    unittest.main()

poky/meta/lib/oeqa/core/tests/test_decorators.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+import signal
+import unittest
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQADependency
+from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs
+
+class TestTagDecorator(TestBase):
+    def _runTest(self, modules, filterfn, expect):
+        tc = self._testLoader(modules = modules, tags_filter = filterfn)
+        test_loaded = set(getSuiteCasesIDs(tc.suites))
+        self.assertEqual(expect, test_loaded)
+
+    def test_oetag(self):
+        # get all cases without any filtering
+        self._runTest(['oetag'], None, {
+                'oetag.TagTest.testTagGood',
+                'oetag.TagTest.testTagOther',
+                'oetag.TagTest.testTagOtherMulti',
+                'oetag.TagTest.testTagNone',
+                'oetag.TagClassTest.testTagOther',
+                'oetag.TagClassTest.testTagOtherMulti',
+                'oetag.TagClassTest.testTagNone',
+                })
+
+        # exclude any case with tags
+        self._runTest(['oetag'], lambda tags: tags, {
+                'oetag.TagTest.testTagNone',
+                })
+
+        # exclude any case with otherTag
+        self._runTest(['oetag'], lambda tags: "otherTag" in tags, {
+                'oetag.TagTest.testTagGood',
+                'oetag.TagTest.testTagNone',
+                'oetag.TagClassTest.testTagNone',
+                })
+
+        # exclude any case with classTag
+        self._runTest(['oetag'], lambda tags: "classTag" in tags, {
+                'oetag.TagTest.testTagGood',
+                'oetag.TagTest.testTagOther',
+                'oetag.TagTest.testTagOtherMulti',
+                'oetag.TagTest.testTagNone',
+                })
+
+        # include any case with classTag
+        self._runTest(['oetag'], lambda tags: "classTag" not in tags, {
+                'oetag.TagClassTest.testTagOther',
+                'oetag.TagClassTest.testTagOtherMulti',
+                'oetag.TagClassTest.testTagNone',
+                })
+
+        # include any case with classTag or no tags
+        self._runTest(['oetag'], lambda tags: tags and "classTag" not in tags, {
+                'oetag.TagTest.testTagNone',
+                'oetag.TagClassTest.testTagOther',
+                'oetag.TagClassTest.testTagOtherMulti',
+                'oetag.TagClassTest.testTagNone',
+                })
+
+class TestDependsDecorator(TestBase):
+    modules = ['depends']
+
+    def test_depends_order(self):
+        tests =  ['depends.DependsTest.testDependsFirst',
+                  'depends.DependsTest.testDependsSecond',
+                  'depends.DependsTest.testDependsThird',
+                  'depends.DependsTest.testDependsFourth',
+                  'depends.DependsTest.testDependsFifth']
+        tests2 = list(tests)
+        tests2[2], tests2[3] = tests[3], tests[2]
+        tc = self._testLoader(modules=self.modules, tests=tests)
+        test_loaded = getSuiteCasesIDs(tc.suites)
+        result = True if test_loaded == tests or test_loaded == tests2 else False
+        msg = 'Failed to order tests using OETestDepends decorator.\nTest order:'\
+              ' %s.\nExpected:   %s\nOr:         %s' % (test_loaded, tests, tests2)
+        self.assertTrue(result, msg=msg)
+
+    def test_depends_fail_missing_dependency(self):
+        expect = "TestCase depends.DependsTest.testDependsSecond depends on "\
+                 "depends.DependsTest.testDependsFirst and isn't available"
+        tests =  ['depends.DependsTest.testDependsSecond']
+        try:
+            # Must throw OEQADependency because missing 'testDependsFirst'
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            self.fail('Expected OEQADependency exception')
+        except OEQADependency as e:
+            result = True if expect in str(e) else False
+            msg = 'Expected OEQADependency exception missing testDependsFirst test'
+            self.assertTrue(result, msg=msg)
+
+    def test_depends_fail_circular_dependency(self):
+        expect = 'have a circular dependency'
+        tests =  ['depends.DependsTest.testDependsCircular1',
+                  'depends.DependsTest.testDependsCircular2',
+                  'depends.DependsTest.testDependsCircular3']
+        try:
+            # Must throw OEQADependency because circular dependency
+            tc = self._testLoader(modules=self.modules, tests=tests)
+            self.fail('Expected OEQADependency exception')
+        except OEQADependency as e:
+            result = True if expect in str(e) else False
+            msg = 'Expected OEQADependency exception having a circular dependency'
+            self.assertTrue(result, msg=msg)
+
+class TestTimeoutDecorator(TestBase):
+    modules = ['timeout']
+
+    def test_timeout(self):
+        tests = ['timeout.TimeoutTest.testTimeoutPass']
+        msg = 'Failed to run test using OETestTimeout'
+        alarm_signal = signal.getsignal(signal.SIGALRM)
+        tc = self._testLoader(modules=self.modules, tests=tests)
+        self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
+        msg = "OETestTimeout didn't restore SIGALRM"
+        self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
+
+    def test_timeout_fail(self):
+        tests = ['timeout.TimeoutTest.testTimeoutFail']
+        msg = "OETestTimeout test didn't timeout as expected"
+        alarm_signal = signal.getsignal(signal.SIGALRM)
+        tc = self._testLoader(modules=self.modules, tests=tests)
+        self.assertFalse(tc.runTests().wasSuccessful(), msg=msg)
+        msg = "OETestTimeout didn't restore SIGALRM"
+        self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
+
+    def test_timeout_cancel(self):
+        tests = ['timeout.TimeoutTest.testTimeoutSkip', 'timeout.TimeoutTest.testTimeoutDepends', 'timeout.TimeoutTest.testTimeoutUnrelated']
+        msg = 'Unrelated test failed to complete'
+        tc = self._testLoader(modules=self.modules, tests=tests)
+        self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
+
+if __name__ == '__main__':
+    unittest.main()

poky/meta/lib/oeqa/core/tests/test_loader.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+import os
+import unittest
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.exception import OEQADependency
+from oeqa.core.utils.test import getSuiteModules, getSuiteCasesIDs
+
+class TestLoader(TestBase):
+    @unittest.skip("invalid directory is missing oetag.py")
+    def test_fail_duplicated_module(self):
+        cases_path = self.cases_path
+        invalid_path = os.path.join(cases_path, 'loader', 'invalid')
+        self.cases_path = [self.cases_path, invalid_path]
+        expect = 'Duplicated oetag module found in'
+        msg = 'Expected ImportError exception for having duplicated module'
+        try:
+            # Must throw ImportEror because duplicated module
+            tc = self._testLoader()
+            self.fail(msg)
+        except ImportError as e:
+            result = True if expect in str(e) else False
+            self.assertTrue(result, msg=msg)
+        finally:
+            self.cases_path = cases_path
+
+    def test_filter_modules(self):
+        expected_modules = {'oetag'}
+        tc = self._testLoader(modules=expected_modules)
+        modules = getSuiteModules(tc.suites)
+        msg = 'Expected just %s modules' % ', '.join(expected_modules)
+        self.assertEqual(modules, expected_modules, msg=msg)
+
+    def test_filter_cases(self):
+        modules = ['oetag', 'data']
+        expected_cases = {'data.DataTest.testDataOk',
+                          'oetag.TagTest.testTagGood'}
+        tc = self._testLoader(modules=modules, tests=expected_cases)
+        cases = set(getSuiteCasesIDs(tc.suites))
+        msg = 'Expected just %s cases' % ', '.join(expected_cases)
+        self.assertEqual(cases, expected_cases, msg=msg)
+
+    def test_import_from_paths(self):
+        cases_path = self.cases_path
+        cases2_path = os.path.join(cases_path, 'loader', 'valid')
+        expected_modules = {'another'}
+        self.cases_path = [self.cases_path, cases2_path]
+        tc = self._testLoader(modules=expected_modules)
+        modules = getSuiteModules(tc.suites)
+        self.cases_path = cases_path
+        msg = 'Expected modules from two different paths'
+        self.assertEqual(modules, expected_modules, msg=msg)
+
+if __name__ == '__main__':
+    unittest.main()

poky/meta/lib/oeqa/core/tests/test_runner.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+import unittest
+import logging
+import tempfile
+
+from common import setup_sys_path, TestBase
+setup_sys_path()
+
+from oeqa.core.runner import OEStreamLogger
+
+class TestRunner(TestBase):
+    def test_stream_logger(self):
+        fp = tempfile.TemporaryFile(mode='w+')
+
+        logging.basicConfig(format='%(message)s', stream=fp)
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+
+        oeSL = OEStreamLogger(logger)
+
+        lines = ['init', 'bigline_' * 65535, 'morebigline_' * 65535 * 4, 'end']
+        for line in lines:
+            oeSL.write(line)
+
+        fp.seek(0)
+        fp_lines = fp.readlines()
+        for i, fp_line in enumerate(fp_lines):
+            fp_line = fp_line.strip()
+            self.assertEqual(lines[i], fp_line)
+
+        fp.close()
+
+if __name__ == '__main__':
+    unittest.main()