[testsuite] Add debug mode & search directory to runner

tests/doc.md

@@ -430,7 +430,15 @@ Then navigate to the `tests/` subdirectory and run:
 python3 runner.py
 ```
 
+One can supply the script with a directory where the suite should look for the
+tests (it will search the directory recursively). If omitted, the runner will
+look in the directory where the configuration file is located.
+
 The runner script also supports the optional arguments `--config_file` which
 allows to provide a different test suite configuration file than the default
 `suite.conf`. It also forwards the verbosity setting via the `-v`/`--verbose`
 flags to Python's unittest module.
+
+Optionally one can provide the `--debug` flag which will instruct test suite to
+print all command invocations and all expected and obtained outputs to the
+standard output.

tests/runner.py

@@ -15,16 +15,34 @@ if __name__ == '__main__':
         "--config_file",
         type=str,
         nargs=1,
+        help="Path to the suite's configuration file",
         default=['suite.conf']
     )
     parser.add_argument(
         "--verbose", "-v",
         action='count',
+        help="verbosity level",
         default=1
     )
+    parser.add_argument(
+        "--debug",
+        help="enable debugging output",
+        action='store_true'
+    )
+
+    parser.add_argument(
+        "dir",
+        help="directory where the test are searched for (defaults to the config"
+        "file's location)",
+        default=None,
+        type=str,
+        nargs='?'
+    )
+
     args = parser.parse_args()
     conf_file = args.config_file[0]
-    discovery_root = os.path.dirname(conf_file)
+    discovery_root = os.path.dirname(conf_file if args.dir is None else args.dir)
+    system_tests.set_debug_mode(args.debug)
 
     system_tests.configure_suite(conf_file)
 

tests/system_tests.py

@@ -80,6 +80,20 @@ class CasePreservingConfigParser(configparser.ConfigParser):
 _parameters = {}
 
 
+#: setting whether debug mode is enabled or not
+_debug_mode = False
+
+
+def set_debug_mode(debug):
+    """ Enable or disable debug mode
+
+    In debug mode the test suite will print out all commands that it runs, the
+    expected output and the actually obtained output
+    """
+    global _debug_mode
+    _debug_mode = debug
+
+
 def configure_suite(config_file):
     """
     Populates a global datastructure with the parameters from the suite's
@@ -432,6 +446,14 @@ def test_run(self):
         retval = int(retval)
         timeout = {"flag": False}
 
+        if _debug_mode:
+            print(
+                '', "="*80, "will run: " + command, "expected stdout:", stdout,
+                "expected stderr:", stderr,
+                "expected return value: {:d}".format(retval),
+                sep='\n'
+            )
+
         proc = subprocess.Popen(
             _cmd_splitter(command),
             stdout=subprocess.PIPE,
@@ -453,6 +475,14 @@ def test_run(self):
         processed_stdout = _process_output_post(got_stdout.decode('utf-8'))
         processed_stderr = _process_output_post(got_stderr.decode('utf-8'))
 
+        if _debug_mode:
+            print(
+                "got stdout:", processed_stdout, "got stderr:",
+                processed_stderr, "got return value: {:d}"
+                .format(proc.returncode),
+                sep='\n'
+            )
+
         self.assertFalse(timeout["flag"] and "Timeout reached")
         self.compare_stdout(i, command, processed_stdout, stdout)
         self.compare_stderr(i, command, processed_stderr, stderr)