From c40c90141fb23dae2294c94de627cb9fca364b8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= <dan.cermak@cgc-instruments.com>
Date: Sat, 21 Apr 2018 00:53:15 +0200
Subject: [PATCH] [testsuite] Add debug mode & search directory to runner

---
 tests/doc.md          |  8 ++++++++
 tests/runner.py       | 20 +++++++++++++++++++-
 tests/system_tests.py | 30 ++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/tests/doc.md b/tests/doc.md
index 1432d008..4c351497 100644
--- a/tests/doc.md
+++ b/tests/doc.md
@@ -430,7 +430,15 @@ Then navigate to the `tests/` subdirectory and run:
 python3 runner.py
 ```
 
+One can supply the script with a directory where the suite should look for the
+tests (it will search the directory recursively). If omitted, the runner will
+look in the directory where the configuration file is located.
+
 The runner script also supports the optional arguments `--config_file` which
 allows to provide a different test suite configuration file than the default
 `suite.conf`. It also forwards the verbosity setting via the `-v`/`--verbose`
 flags to Python's unittest module.
+
+Optionally one can provide the `--debug` flag which will instruct test suite to
+print all command invocations and all expected and obtained outputs to the
+standard output.
diff --git a/tests/runner.py b/tests/runner.py
index 745dcebd..63b82810 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -15,16 +15,34 @@ if __name__ == '__main__':
         "--config_file",
         type=str,
         nargs=1,
+        help="Path to the suite's configuration file",
         default=['suite.conf']
     )
     parser.add_argument(
         "--verbose", "-v",
         action='count',
+        help="verbosity level",
         default=1
     )
+    parser.add_argument(
+        "--debug",
+        help="enable debugging output",
+        action='store_true'
+    )
+
+    parser.add_argument(
+        "dir",
+        help="directory where the test are searched for (defaults to the config"
+        "file's location)",
+        default=None,
+        type=str,
+        nargs='?'
+    )
+
     args = parser.parse_args()
     conf_file = args.config_file[0]
-    discovery_root = os.path.dirname(conf_file)
+    discovery_root = os.path.dirname(conf_file if args.dir is None else args.dir)
+    system_tests.set_debug_mode(args.debug)
 
     system_tests.configure_suite(conf_file)
 
diff --git a/tests/system_tests.py b/tests/system_tests.py
index 5d968e4e..20492fd6 100644
--- a/tests/system_tests.py
+++ b/tests/system_tests.py
@@ -80,6 +80,20 @@ class CasePreservingConfigParser(configparser.ConfigParser):
 _parameters = {}
 
 
+#: setting whether debug mode is enabled or not
+_debug_mode = False
+
+
+def set_debug_mode(debug):
+    """ Enable or disable debug mode
+
+    In debug mode the test suite will print out all commands that it runs, the
+    expected output and the actually obtained output
+    """
+    global _debug_mode
+    _debug_mode = debug
+
+
 def configure_suite(config_file):
     """
     Populates a global datastructure with the parameters from the suite's
@@ -432,6 +446,14 @@ def test_run(self):
         retval = int(retval)
         timeout = {"flag": False}
 
+        if _debug_mode:
+            print(
+                '', "="*80, "will run: " + command, "expected stdout:", stdout,
+                "expected stderr:", stderr,
+                "expected return value: {:d}".format(retval),
+                sep='\n'
+            )
+
         proc = subprocess.Popen(
             _cmd_splitter(command),
             stdout=subprocess.PIPE,
@@ -453,6 +475,14 @@ def test_run(self):
         processed_stdout = _process_output_post(got_stdout.decode('utf-8'))
         processed_stderr = _process_output_post(got_stderr.decode('utf-8'))
 
+        if _debug_mode:
+            print(
+                "got stdout:", processed_stdout, "got stderr:",
+                processed_stderr, "got return value: {:d}"
+                .format(proc.returncode),
+                sep='\n'
+            )
+
         self.assertFalse(timeout["flag"] and "Timeout reached")
         self.compare_stdout(i, command, processed_stdout, stdout)
         self.compare_stderr(i, command, processed_stderr, stderr)