diff --git a/test/Makefile b/test/Makefile index 36fa4afd..7216bdcd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -113,7 +113,7 @@ test: @if [ -e tmp/test-failed ]; then echo '***' FAILED ; cat tmp/test-failed ; echo '***' ; exit 255; fi newtests: - ( if [ -e bin/exiv2 ]; then PATH=$$PWD/bin/build ; fi ; cd ../tests ; env EXIV2_PATH=$$(dirname $$(which exiv2)) python3 runner.py --verbose ) + ( cd ../tests ; python3 runner.py --verbose ) testv: @for t in /video ; do \ diff --git a/tests/bugfixes/github/test_CVE_2017_1000126.py b/tests/bugfixes/github/test_CVE_2017_1000126.py index c5f766e2..414da55e 100644 --- a/tests/bugfixes/github/test_CVE_2017_1000126.py +++ b/tests/bugfixes/github/test_CVE_2017_1000126.py @@ -10,7 +10,7 @@ class TestCvePoC(system_tests.Case): filename = "{data_path}/cve_2017_1000126_stack-oob-read.webp" commands = ["{exiv2} " + filename] stdout = [""] - stderr = ["""{exiv2_exception_msg} """ + filename + """: + stderr = ["""{exiv2_exception_message} """ + filename + """: {kerCorruptedMetadata} """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_1000127.py b/tests/bugfixes/github/test_CVE_2017_1000127.py index d3b12f2d..ed22da37 100644 --- a/tests/bugfixes/github/test_CVE_2017_1000127.py +++ b/tests/bugfixes/github/test_CVE_2017_1000127.py @@ -10,7 +10,7 @@ class TestPoC(system_tests.Case): filename = "{data_path}/heap-oob-write.tiff" commands = ["{exiv2} " + filename] stdout = [""] - stderr = ["""{exiv2_exception_msg} """ + filename + """: + stderr = ["""{exiv2_exception_message} """ + filename + """: {kerInvalidMalloc} """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11336.py b/tests/bugfixes/github/test_CVE_2017_11336.py new file mode 100644 index 00000000..95741a36 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11336.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/49" + + filename = "{data_path}/POC2" + commands = ["{exiv2} " + filename] + retval = [1] + stdout = [""] + stderr = [ + """{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] diff --git a/tests/bugfixes/github/test_CVE_2017_11337.py b/tests/bugfixes/github/test_CVE_2017_11337.py new file mode 100644 index 00000000..c2ebeac9 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11337.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/50" + + filename = "{data_path}/POC3" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11338.py b/tests/bugfixes/github/test_CVE_2017_11338.py new file mode 100644 index 00000000..74c852ff --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11338.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/51" + + filename = "{data_path}/POC4" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11339.py b/tests/bugfixes/github/test_CVE_2017_11339.py new file mode 100644 index 00000000..257619b0 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11339.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/52" + + filename = "{data_path}/POC5" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11340.py b/tests/bugfixes/github/test_CVE_2017_11340.py new file mode 100644 index 00000000..2b289c77 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11340.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/53" + + filename = "{data_path}/POC6" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11553.py b/tests/bugfixes/github/test_CVE_2017_11553.py new file mode 100644 index 00000000..f57d85e8 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11553.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/54" + + filename = "{data_path}/POC7" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11591.py b/tests/bugfixes/github/test_CVE_2017_11591.py new file mode 100644 index 00000000..772c5240 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11591.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/55" + + filename = "{data_path}/POC8" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11592.py b/tests/bugfixes/github/test_CVE_2017_11592.py new file mode 100644 index 00000000..233d38ea --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11592.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/56" + + filename = "{data_path}/POC9" + commands = ["{exiv2} " + filename] + stdout = [""""""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11683.py b/tests/bugfixes/github/test_CVE_2017_11683.py new file mode 100644 index 00000000..fac4a5ec --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_11683.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/57" + + filename = "{data_path}/POC" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{kerInvalidTypeValue}: 0 +{exiv2_exception_message} """ + filename + """: +{kerInvalidTypeValue} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12955.py b/tests/bugfixes/github/test_CVE_2017_12955.py new file mode 100644 index 00000000..5b40d148 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_12955.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/58" + + filename = "{data_path}/POC11" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12956.py b/tests/bugfixes/github/test_CVE_2017_12956.py new file mode 100644 index 00000000..74f73d65 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_12956.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/59" + + filename = "{data_path}/POC12" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12957.py b/tests/bugfixes/github/test_CVE_2017_12957.py new file mode 100644 index 00000000..acaaa07c --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_12957.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/60" + + filename = "{data_path}/POC13" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14857.py b/tests/bugfixes/github/test_CVE_2017_14857.py index 51eb8608..895a7e54 100644 --- a/tests/bugfixes/github/test_CVE_2017_14857.py +++ b/tests/bugfixes/github/test_CVE_2017_14857.py @@ -3,13 +3,15 @@ import system_tests -class CVE_2017_14857(system_tests.Case): +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/76" filename = "{data_path}/010_bad_free" commands = ["{exiv2} " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_msg} """ + filename + """: + """{exiv2_exception_message} """ + filename + """: {kerInvalidMalloc} """] diff --git a/tests/bugfixes/github/test_CVE_2017_14858.py b/tests/bugfixes/github/test_CVE_2017_14858.py new file mode 100644 index 00000000..bada8574 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14858.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/138" + + filename = "{data_path}/007-heap-buffer-over" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14859.py b/tests/bugfixes/github/test_CVE_2017_14859.py new file mode 100644 index 00000000..a156895a --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14859.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/74" + + filename = "{data_path}/005-invalid-mem" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerCorruptedMetadata} +"""] + retval = [1] + + def compare_stderr(self, i, command, got_stderr, expected_stderr): + """ Only check that an exception is thrown """ + self.assertIn(expected_stderr, got_stderr) diff --git a/tests/bugfixes/github/test_CVE_2017_14680.py b/tests/bugfixes/github/test_CVE_2017_14860.py similarity index 67% rename from tests/bugfixes/github/test_CVE_2017_14680.py rename to tests/bugfixes/github/test_CVE_2017_14860.py index b194b705..abae772a 100644 --- a/tests/bugfixes/github/test_CVE_2017_14680.py +++ b/tests/bugfixes/github/test_CVE_2017_14860.py @@ -3,15 +3,14 @@ import system_tests -class CVE_2017_14680(system_tests.Case): +class TestCvePoC(system_tests.Case): - bug_no = "73" url = "https://github.com/Exiv2/exiv2/issues/73" filename = "{data_path}/003-heap-buffer-over" commands = ["{exiv2} " + filename] stdout = [""] - stderr = ["""{exiv2_exception_msg} """ + filename + """: + stderr = ["""{exiv2_exception_message} """ + filename + """: {kerCorruptedMetadata} """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14861.py b/tests/bugfixes/github/test_CVE_2017_14861.py new file mode 100644 index 00000000..432a0a17 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14861.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = [ + "https://github.com/Exiv2/exiv2/issues/139", + "https://bugzilla.redhat.com/show_bug.cgi?id=1494787" + ] + + filename = "{data_path}/009-stack-over" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14862.py b/tests/bugfixes/github/test_CVE_2017_14862.py new file mode 100644 index 00000000..b915c348 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14862.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/75" + + filename = "{data_path}/008-invalid-mem" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerCorruptedMetadata} +"""] + retval = [1] + + def compare_stderr(self, i, command, got_stderr, expected_stderr): + """ + Only check that an exception is thrown for this file + ignore all the warnings on stderr on purpose. + """ + self.assertIn(expected_stderr, got_stderr) diff --git a/tests/bugfixes/github/test_CVE_2017_14863.py b/tests/bugfixes/github/test_CVE_2017_14863.py new file mode 100644 index 00000000..4cba3a92 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14863.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/132" + + filename = "{data_path}/01-Null-exiv2-poc" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14864.py b/tests/bugfixes/github/test_CVE_2017_14864.py new file mode 100644 index 00000000..26a46273 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14864.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/73" + + filename = "{data_path}/02-Invalid-mem-def" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerCorruptedMetadata} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14865.py b/tests/bugfixes/github/test_CVE_2017_14865.py new file mode 100644 index 00000000..e0f4da61 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14865.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/134" + + filename = "{data_path}/004-heap-buffer-over" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{kerInvalidTypeValue}: 250 +{exiv2_exception_message} """ + filename + """: +{kerInvalidTypeValue} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14866.py b/tests/bugfixes/github/test_CVE_2017_14866.py new file mode 100644 index 00000000..659aa617 --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_14866.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/140" + + filename = "{data_path}/006-heap-buffer-over" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_17669.py b/tests/bugfixes/github/test_CVE_2017_17669.py index d51e96c3..bf6a46b3 100644 --- a/tests/bugfixes/github/test_CVE_2017_17669.py +++ b/tests/bugfixes/github/test_CVE_2017_17669.py @@ -3,14 +3,16 @@ import system_tests -class RunPocFile(system_tests.Case): +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/187" filename = "{data_path}/issue_187" commands = ["{exiv2} " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_msg} """ + filename + """: + """{exiv2_exception_message} """ + filename + """: {kerFailedToReadImageData} """ ] diff --git a/tests/bugfixes/github/test_issue_208.py b/tests/bugfixes/github/test_CVE_2017_17722.py similarity index 59% rename from tests/bugfixes/github/test_issue_208.py rename to tests/bugfixes/github/test_CVE_2017_17722.py index 299b41f9..fd5391ce 100644 --- a/tests/bugfixes/github/test_issue_208.py +++ b/tests/bugfixes/github/test_CVE_2017_17722.py @@ -3,13 +3,15 @@ import system_tests -class CVE_2017_14857(system_tests.Case): +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/208" filename = "{data_path}/2018-01-09-exiv2-crash-001.tiff" commands = ["{exiv2} " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_msg} """ + filename + """: + """{exiv2_exception_message} """ + filename + """: {kerCorruptedMetadata} """] diff --git a/tests/bugfixes/github/test_CVE_2017_17725.py b/tests/bugfixes/github/test_CVE_2017_17725.py new file mode 100644 index 00000000..da8f48de --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_17725.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/188" + found_by = ["Wei You", "@youwei1988"] + + filename = "{data_path}/poc_2017-12-12_issue188" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_overflow_exception_message} """ + filename + """: +{addition_overflow_message} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_9953.py b/tests/bugfixes/github/test_CVE_2017_9953.py new file mode 100644 index 00000000..79070b6e --- /dev/null +++ b/tests/bugfixes/github/test_CVE_2017_9953.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestCvePoC(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/144" + + filename = "{data_path}/POC1" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerInvalidMalloc} +"""] + retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2018_4868.py b/tests/bugfixes/github/test_CVE_2018_4868.py index 3ddfd8d7..86fbd22c 100644 --- a/tests/bugfixes/github/test_CVE_2018_4868.py +++ b/tests/bugfixes/github/test_CVE_2018_4868.py @@ -12,7 +12,7 @@ class TestCvePoC(system_tests.Case): filename = "{data_path}/exiv2-memorymmap-error" commands = ["{exiv2} " + filename] stdout = [""] - stderr = ["""{exiv2_exception_msg} """ + filename + """: + stderr = ["""{exiv2_exception_message} """ + filename + """: {kerCorruptedMetadata} """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_159.py b/tests/bugfixes/github/test_issue_159.py new file mode 100644 index 00000000..be4bf998 --- /dev/null +++ b/tests/bugfixes/github/test_issue_159.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class TestFirstPoC(system_tests.Case): + """ + Regression test for the first bug described in: + https://github.com/Exiv2/exiv2/issues/159 + """ + url = "https://github.com/Exiv2/exiv2/issues/159" + + filename = "{data_path}/printStructure" + commands = ["{exiv2} " + filename] + stdout = [""] + stderr = ["""{exiv2_exception_message} """ + filename + """: +{kerCorruptedMetadata} +"""] + retval = [1] + + +# todo: +# class TestSecondPoC(system_tests.Case): diff --git a/tests/bugfixes/github/test_issue_170.py b/tests/bugfixes/github/test_issue_170.py index 8fd8d7af..a67e8a4b 100644 --- a/tests/bugfixes/github/test_issue_170.py +++ b/tests/bugfixes/github/test_issue_170.py @@ -3,7 +3,7 @@ import system_tests -class decodeIHDRChunkOutOfBoundsRead(system_tests.Case): +class DecodeIHDRChunkOutOfBoundsRead(system_tests.Case): url = "https://github.com/Exiv2/exiv2/issues/170" @@ -11,7 +11,7 @@ class decodeIHDRChunkOutOfBoundsRead(system_tests.Case): commands = ["{exiv2} " + filename] stdout = [""] - stderr = ["""{exiv2_exception_msg} """ + filename + """: + stderr = ["""{exiv2_exception_message} """ + filename + """: {kerFailedToReadImageData} """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_45.py b/tests/bugfixes/github/test_issue_45.py new file mode 100644 index 00000000..216a81b0 --- /dev/null +++ b/tests/bugfixes/github/test_issue_45.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +import system_tests + + +class Sigma24_105mmRecognization(system_tests.Case): + + url = "https://github.com/Exiv2/exiv2/issues/45" + + filename = "{data_path}/exiv2-g45.exv" + commands = ["{exiv2} -pa --grep lens/i " + filename] + stdout = ["""Exif.CanonCs.LensType Short 1 Sigma 24-105mm F4 DG OS HSM [Art 013] +Exif.CanonCs.Lens Short 3 24.0 - 105.0 mm +Exif.CanonCf.LensAFStopButton Short 1 0 +Exif.Canon.LensModel Ascii 74 24-105mm F4 DG OS HSM | Art 013 +"""] + stderr = [""] + retval = [0] diff --git a/tests/doc.md b/tests/doc.md index ad31bd57..fff7ba5f 100644 --- a/tests/doc.md +++ b/tests/doc.md @@ -344,6 +344,31 @@ customize how the return value is checked. This is indented, as the return value is often used by the OS to indicate segfaults and ignoring it (in combination with flawed checks of the output) could lead to crashes not being noticed. +A drop-in replacement for `compare_stderr` is provided by the `system_tests` +module itself: `check_no_ASAN_UBSAN_errors`. This function only checks that +errors from AddressSanitizer and undefined behavior sanitizer are not present in +the obtained output to standard error **and nothing else**. This is useful for +test cases where stderr is filled with warnings that are not worth being tracked +by the test suite. It can be used in the following way: +``` python +# -*- coding: utf-8 -*- + +import system_tests + + +class AnInformativeName(system_tests.Case): + + filename = "invalid_input_file" + commands = ["{binary} -c {import_file} -i {filename}"] + retval = ["{abort_exit_value}"] + stdout = ["Reading {filename}"] + stderr = ["""A huge amount of error messages would be here that we absolutely do not care about. Actually everything in this string gets ignored, so we can just leave it empty. +""" + ] + + compare_stderr = system_tests.check_no_ASAN_UBSAN_errors +``` + ### Manually expanding variables in strings diff --git a/tests/suite.conf b/tests/suite.conf index e31685a8..a1795c00 100644 --- a/tests/suite.conf +++ b/tests/suite.conf @@ -18,4 +18,7 @@ tiff-test: ${ENV:exiv2_path}/tiff-test${ENV:binary_extension} kerFailedToReadImageData: Failed to read image data kerCorruptedMetadata: corrupted image metadata kerInvalidMalloc: invalid memory allocation request -exiv2_exception_msg: Exiv2 exception in print action for file +kerInvalidTypeValue: invalid type value detected in Image::printIFDStructure +addition_overflow_message: Overflow in addition +exiv2_exception_message: Exiv2 exception in print action for file +exiv2_overflow_exception_message: std::overflow_error exception in print action for file diff --git a/tests/system_tests.py b/tests/system_tests.py index 8e75b0de..c494681b 100644 --- a/tests/system_tests.py +++ b/tests/system_tests.py @@ -156,83 +156,254 @@ def configure_suite(config_file): _parameters[key] = abs_path -def _setUp_factory(old_setUp, *files): +class FileDecoratorBase(object): """ - Factory function that returns a setUp function suitable to replace the - existing setUp of a unittest.TestCase. The returned setUp calls at first - old_setUp(self) and then creates a copy of all files in *files with the - name: fname.ext -> fname_copy.ext + Base class for decorators that manipulate files for test cases. - All file names in *files are at first expanded using self.expand_variables() - and the path to the copy is saved in self._file_copies + The decorator expects to be provided with at least one file path + on construction. When called, it replaces the setUp() and + tearDown() functions of the type it is called on with custom ones. + + The new setUp() function performs the following steps: + - call the old setUp() + - create a list _files in the decorated class + - iterate over all files, performing: + - expand the file's path via expand_variables (member function + of the decorated class) + - call self.setUp_file_action(expanded file name) + - append the result to _files in the decorated class + + The function self.setUp_file_action is provided by this class and + is intended to be overridden by child classes to provide some + functionality, like file copies. + + + The new tearDown() function performs the following steps: + - iterate over all files in _files (from the decorated class): + - call self.tearDown_file_action(filename) + - call the old tearDown() function + + The function self.tearDown_file_action can be overridden by child + classes. The default version provided by this class simply deletes + all files that are passed to it. + + + Example + ------- + + We'll inherit from FileDecoratorBase and override the member + functions setUp_file_action and tearDown_file_action: + + >>> class TestDecorator(FileDecoratorBase): + ... def setUp_file_action(self, f): + ... print("setUp_file_action with", f) + ... return f.capitalize() + ... + ... def tearDown_file_action(self, f): + ... print("tearDown_file_action with", f) + + Then, we use that decorator to wrap a class mocking + system_tests.Case: + + >>> @TestDecorator("one", "two", "three") + ... class MockCase(object): + ... def setUp(self): + ... print("calling MockCase.setUp()") + ... + ... def tearDown(self): + ... print("calling MockCase.tearDown()") + ... + ... def expand_variables(self, var): + ... return var + "_file" + + >>> M = MockCase() + + setUp has been replaced by a the new version, but the old one is + still called. The new setUp iterates over all parameters passed to + the constructor of the decorator, passes them to expand_variables + and then to setUp_file_action: + >>> M.setUp() + calling MockCase.setUp() + setUp_file_action with one_file + setUp_file_action with two_file + setUp_file_action with three_file + + The tearDown() function works accordingly: + >>> M.tearDown() + tearDown_file_action with One_file + tearDown_file_action with Two_file + tearDown_file_action with Three_file + calling MockCase.tearDown() + + Please note the capitalized "file" names (this is due to + setUp_file_action returning f.capitalized()) and that the old + tearDown is called after the new one runs. """ - def setUp(self): - old_setUp(self) - self._file_copies = [] - for f in files: - expanded_fname = self.expand_variables(f) - fname, ext = os.path.splitext(expanded_fname) - new_name = fname + '_copy' + ext - self._file_copies.append( - shutil.copyfile(expanded_fname, new_name) - ) - return setUp + + def __init__(self, *files): + """ + Constructor of FileDecoratorBase. + + To prevent accidental wrong usage, it raises an exception if + it is not called as a decorator with parameters. + + Only the following syntax works for this decorator: + >>> @FileDecoratorBase("test") + ... class Test(unittest.TestCase): + ... pass + + Calling it without parameters or without parenthesis raises an + exception: + >>> @FileDecoratorBase() + ... class Test(unittest.TestCase): + ... pass + Traceback (most recent call last): + .. + ValueError: No files supplied. + + >>> @FileDecoratorBase + ... class Test(unittest.TestCase): + ... pass + Traceback (most recent call last): + .. + UserWarning: Decorator used wrongly, must be called with filenames in parenthesis + """ + if len(files) == 0: + raise ValueError("No files supplied.") + elif len(files) == 1: + if isinstance(files[0], type): + raise UserWarning( + "Decorator used wrongly, must be called with " + "filenames in parenthesis" + ) + + self._files = files + + def new_setUp(self, old_setUp): + """ + Returns a new setUp() function that can be used as a class + member function (i.e. invoked via self.setUp()). + + It's functionality is described in this classes' docstring. + """ + + def setUp(other): + old_setUp(other) + other._files = [] + for f in self._files: + expanded_fname = other.expand_variables(f) + other._files.append( + self.setUp_file_action(expanded_fname) + ) + return setUp + + def setUp_file_action(self, expanded_file_name): + """ + This function is called on each file that is passed to the + constructor during the call of the decorated class' setUp(). + + Parameters: + - expanded_file_name: the file's path expanded via + expand_variables from system_tests.Case + + Returns: + This function should return a path that will be stored in the + decorated class' list _files. The custom tearDown() function + (that is returned by self.new_tearDown()) iterates over this + list and invokes self.tearDown_file_action on each element in + that list. + E.g. if a child class creates file copies, that should be + deleted after the test ran, then one would have to return the + path of the copy, so that tearDown() can delete the copies. + + The default implementation does nothing. + """ + pass + + def new_tearDown(self, old_tearDown): + """ + Returns a new tearDown() function that can be used as a class + member function. + + It's functionality is described in this classes' docstring. + """ + + def tearDown(other): + for f in other._files: + self.tearDown_file_action(f) + old_tearDown(other) + + return tearDown + + def tearDown_file_action(self, f): + """ + This function is called on each file in the decorated class' + list _files (that list is populated during setUp()). + + It can be used to perform cleanup operations after a test run. + + Parameters: + - f: An element of _files + + Returns: + The return value is ignored + + The default implementation removes f. + """ + os.remove(f) + + def __call__(self, cls): + """ + Call operator for the usage as a decorator. It is + automatically used by Python when this class is used as a + decorator. + + Parameters: + - cls: The decorated type. Must be a type + + Returns: + - cls where the setUp and tearDown functions have been + replaced by the functions that are returned by + self.new_setUp() and self.new_tearDown() + """ + if not isinstance(cls, type): + raise ValueError("The decorator must be called on a type") + old_setUp = cls.setUp + cls.setUp = self.new_setUp(old_setUp) + + old_tearDown = cls.tearDown + cls.tearDown = self.new_tearDown(old_tearDown) + + return cls -def _tearDown_factory(old_tearDown): - """ - Factory function that returns a new tearDown method to replace an existing - tearDown method. It at first deletes all files in self._file_copies and then - calls old_tearDown(self). - This factory is intended to be used in conjunction with _setUp_factory - """ - def tearDown(self): - for f in self._file_copies: - os.remove(f) - old_tearDown(self) - return tearDown - - -def CopyFiles(*files): +class CopyFiles(FileDecoratorBase): """ Decorator for subclasses of system_test.Case that automatically creates a - copy of the files specified as the parameters to the decorator. + copy of the files specified as the parameters passed to the decorator. Example: >>> @CopyFiles("{some_var}/file.txt", "{another_var}/other_file.png") - class Foo(Case): - pass + ... class Foo(Case): + ... pass The decorator will inject new setUp method that at first calls the already defined setUp(), then expands all supplied file names using Case.expand_variables and then creates copies by appending '_copy' before - the file extension. The paths to the copies are stored in self._file_copies. + the file extension. The paths to the copies are stored in self._files. The decorator also injects a new tearDown method that deletes all files in - self._file_copies and then calls the original tearDown method. + self._files and then calls the original tearDown method. This function will also complain if it is called without arguments or without paranthesis, which is valid decorator syntax but is obviously a bug - in this case. + in this case as it can result in tests not being run without a warning. """ - if len(files) == 0: - raise ValueError("No files to copy supplied.") - elif len(files) == 1: - if isinstance(files[0], type): - raise UserWarning( - "Decorator used wrongly, must be called with filenames in paranthesis" - ) - def wrapper(cls): - old_setUp = cls.setUp - cls.setUp = _setUp_factory(old_setUp, *files) - - old_tearDown = cls.tearDown - cls.tearDown = _tearDown_factory(old_tearDown) - - return cls - - return wrapper + def setUp_file_action(self, expanded_file_name): + fname, ext = os.path.splitext(expanded_file_name) + new_name = fname + '_copy' + ext + return shutil.copyfile(expanded_file_name, new_name) class Case(unittest.TestCase): @@ -354,3 +525,49 @@ class Case(unittest.TestCase): _process_output_post(got_stderr.decode('utf-8')), stderr ) self.assertEqual(retval, proc.returncode) + + +def check_no_ASAN_UBSAN_errors(self, i, command, got_stderr, expected_stderr): + """ + Drop-in replacement for the default Case.compare_stderr() function that + **only** checks for any signs of ASAN (address sanitizer) and UBSAN + (undefined behavior sanitizer). + + Parameters: + - i, command, expected_stderr: ignored + - got_stderr: checked for signs of ASAN und UBSAN error messages + + This function ignores the expected output to stderr! It is intended for + test cases where standard error is filled with useless debugging + messages/warnings that are not really relevant for the test and not worth + storing in the test suite. This function can be used to still be able to + catch ASAN & UBSAN error messages. + + Example usage + ------------- + + Override the default compare_stderr function in your subclass of Case with + this function: + >>> class TestNoAsan(Case): + ... compare_stderr = check_no_ASAN_UBSAN_errors + + >>> T = TestNoAsan() + + The new compare_stderr will only complain if there are strings inside the + obtained stderr which could be an error reported by ASAN/UBSAN: + >>> T.compare_stderr(0, "", "runtime error: load of value 190", "some output") + Traceback (most recent call last): + .. + AssertionError: 'runtime error' unexpectedly found in 'runtime error: load of value 190' + + >>> T.compare_stderr(0, "", "SUMMARY: AddressSanitizer: heap-buffer-overflow", "") + Traceback (most recent call last): + .. + AssertionError: 'AddressSanitizer' unexpectedly found in 'SUMMARY: AddressSanitizer: heap-buffer-overflow' + + It will not complain in all other cases, especially when expected_stderr + and got_stderr do not match: + >>> T.compare_stderr(0, "", "some output", "other output") + """ + self.assertNotIn("runtime error", got_stderr) + self.assertNotIn("AddressSanitizer", got_stderr)