Introduce integration tests

Adds an initial version of integration tests. The tests are located in
the top level of the 'tests' directory. Integration tests are executed via
the hermit_test_runner.py script, which can use either uhyve or qemu to execute
the tests.
In this initial version, using the test_harness is deactivated.

.cargo/config

+#[target.x86_64-unknown-hermit-kernel]
+#runner = "uhyve -v"
+
 [target.x86_64-unknown-hermit-kernel]
-runner = "uhyve -v"
+runner = "tests/hermit_test_runner.py"

Cargo.toml

@@ -25,6 +25,14 @@ travis-ci = { repository = "hermitcore/libhermit-rs" }
 crate-type = ["staticlib"]
 name = "hermit"
 
+[[test]]
+name = "basic_print"
+harness = false
+
+[[test]]
+name = "basic_math"
+harness = false
+
 [features]
 default = ["pci", "acpi"]
 vga = []

src/lib.rs

@@ -28,6 +28,7 @@
 #![feature(allocator_api)]
 #![feature(const_btree_new)]
 #![feature(const_fn)]
+#![feature(custom_test_frameworks)]
 #![feature(global_asm)]
 #![feature(lang_items)]
 #![feature(linkage)]
@@ -39,6 +40,9 @@
 #![feature(core_intrinsics)]
 #![feature(alloc_error_handler)]
 #![allow(unused_macros)]
+#![test_runner(crate::test_runner)]
+#![reexport_test_harness_main = "test_main"]
+#![cfg_attr(test, no_main)]
 #![no_std]
 
 // EXTERNAL CRATES
@@ -62,6 +66,7 @@ extern crate x86;
 
 use alloc::alloc::Layout;
 use core::alloc::GlobalAlloc;
+use core::panic::PanicInfo;
 use core::sync::atomic::{spin_loop_hint, AtomicU32, Ordering};
 
 use arch::percore::*;
@@ -95,6 +100,20 @@ mod synch;
 mod syscalls;
 mod util;
 
+#[doc(hidden)]
+pub fn _print(args: ::core::fmt::Arguments) {
+	use core::fmt::Write;
+	crate::console::CONSOLE.lock().write_fmt(args).unwrap();
+}
+
+pub fn test_runner(tests: &[&dyn Fn()]) {
+	println!("Running {} tests", tests.len());
+	for test in tests {
+		test();
+	}
+	sys_exit(0);
+}
+
 #[cfg(target_os = "hermit")]
 #[global_allocator]
 static ALLOCATOR: LockedHeap = LockedHeap::empty();

src/macros.rs

@@ -22,14 +22,15 @@ macro_rules! align_up {
 ///
 /// From http://blog.phil-opp.com/rust-os/printing-to-screen.html, but tweaked
 /// for HermitCore.
+#[macro_export]
 macro_rules! print {
 	($($arg:tt)+) => ({
-		use core::fmt::Write;
-		$crate::console::CONSOLE.lock().write_fmt(format_args!($($arg)+)).unwrap();
+        $crate::_print(format_args!($($arg)*));
 	});
 }
 
 /// Print formatted text to our console, followed by a newline.
+#[macro_export]
 macro_rules! println {
     () => (print!("\n"));
 	($($arg:tt)+) => (print!("{}\n", format_args!($($arg)+)));

tests/basic_math.rs

+#![no_std]
+#![no_main]
+
+extern crate hermit;
+use hermit::{print, println};
+
+// Workaround since the "real" runtime_entry function (defined in libstd) is not available,
+// since the target-os is hermit-kernel and not hermit
+#[no_mangle]
+extern "C"
+fn runtime_entry(argc: i32, argv: *const *const u8, _env: *const *const u8) -> ! {
+    let res = main(argc as isize, argv);
+    match res {
+        Ok(_) => hermit::sys_exit(0),
+        Err(_) => hermit::sys_exit(1),  //ToDo: sys_exit exitcode doesn't seem to get passed to qemu
+        // sys_exit argument doesn't actually get used, gets silently dropped!
+        // Maybe this is not possible on QEMU?
+        // https://os.phil-opp.com/testing/#exiting-qemu device needed?
+    }
+}
+
+
+
+
+/*
+/// assert_eq but returns Result<(),&str> instead of panicking
+/// no error message possible
+/// adapted from libcore assert_eq macro
+macro_rules! equals {
+    ($left:expr, $right:expr) => ({
+        match (&$left, &$right) {
+            (left_val, right_val) => {
+                if !(*left_val == *right_val) {
+                    return Err(r#"assertion failed: `(left == right)`
+  left: `{:?}`,
+ right: `{:?}`"# &*left_val, &*right_val);
+                }
+                else { return Ok(()); }
+            }
+        }
+    });
+    ($left:expr, $right:expr,) => ({
+        $crate::assert_eq!($left, $right)
+    });
+}
+
+macro_rules! n_equals {
+    ($left:expr, $right:expr) => ({
+        match (&$left, &$right) {
+            (left_val, right_val) => {
+                if *left_val == *right_val {
+                    // The reborrows below are intentional. Without them, the stack slot for the
+                    // borrow is initialized even before the values are compared, leading to a
+                    // noticeable slow down.
+                    return Err(r#"assertion failed: `(left == right)`
+  left: `{:?}`,
+ right: `{:?}`"#, &*left_val, &*right_val);
+                }
+                else return Ok(());
+            }
+        }
+    });
+    ($left:expr, $right:expr,) => {
+        $crate::assert_ne!($left, $right)
+    };
+}
+*/
+
+//ToDo - add a testrunner so we can group multiple similar tests
+
+//ToDo - Idea: pass some values into main - compute and print result to stdout
+//ToDo - add some kind of assert like macro that returns a result instead of panicking, Err contains line number etc to pinpoint the issue
+pub fn main(_argc: isize, _argv: *const *const u8) -> Result<(), ()>{
+    let x = 25;
+    let y = 310;
+    let z = x * y;
+    println!("25 * 310 = {}", z);
+    assert_eq!(z, 7750);
+    Ok(())
+}

tests/basic_print.rs

+#![no_std]
+#![no_main]
+//#![test_runner(hermit::test_runner)]
+//#![feature(custom_test_frameworks)]
+//#![reexport_test_harness_main = "test_main"]
+
+//use core::panic::PanicInfo;
+extern crate hermit;
+use hermit::{print, println};
+
+//ToDo: Define exit code enum in hermit!!!
+
+// Workaround since the "real" runtime_entry function (defined in libstd) is not available,
+// since the target-os is hermit-kernel and not hermit
+#[no_mangle]
+extern "C" fn runtime_entry(argc: i32, argv: *const *const u8, env: *const *const u8) -> ! {
+	main(argc as isize, argv);
+	hermit::sys_exit(-1);
+}
+
+//#[test_case]
+pub fn main(argc: isize, argv: *const *const u8) {
+	println!("hey we made it to the test function :O");
+}

tests/hermit_test_runner.py

+#!/usr/bin/env python3
+
+import time
+import argparse
+import subprocess
+from subprocess import Popen, PIPE, STDOUT
+import os, os.path
+
+SMP_CORES = 1  # Number of cores
+MEMORY_MB = 64  # amount of memory
+# Path if libhermit-rs was checked out via rusty-hermit repository
+BOOTLOADER_PATH = '../loader/target/x86_64-unknown-hermit-loader/debug/rusty-loader'
+
+
+# ToDo add test dependent section for custom kernel arguments / application arguments
+# Idea: Use TOML format to specify things like should_panic, expected output
+# Parse test executable name and check tests directory for corresponding toml file
+# If it doesn't exist just assure that the return code is not a failure
+
+# ToDo Think about always being verbose, or hiding the output
+def run_test(process_args):
+    print(os.getcwd())
+    abs_bootloader_path = os.path.abspath(BOOTLOADER_PATH)
+    print("Abspath: ", abs_bootloader_path)
+    p = Popen(process_args, stdout=PIPE, stderr=STDOUT, text=True)
+    output: str = ""
+    for line in p.stdout:
+        dec_line = line
+        output += dec_line
+        #print(line, end='')  # stdout will already contain line break
+    rc = p.wait()
+    # ToDo: add some timeout
+    return rc, output
+
+
+def validate_test(returncode, output, test_exe_path):
+    print("returncode ", returncode)
+    # ToDo handle expected failures
+    if returncode != 0:
+        return False
+    # ToDo parse output for panic
+    return True
+
+
+def clean_test_name(name: str):
+    if name.endswith('.exe'):
+        name = name.replace('.exe', '')
+    # Remove the hash from the name
+    parts = name.split('-')
+    if len(parts) > 1:
+        try:
+            _hex = int(parts[-1], base=16)  # Test if last element is hex hash
+            clean_name = "-".join(parts[:-1])  # Rejoin with '-' as seperator in case test has it in filename
+        except ValueError as e:
+            print(e)
+            clean_name = name  # In this case name doesn't contain a hash, so don't modify it any further
+    return clean_name
+
+
+print("Test runner called")
+parser = argparse.ArgumentParser(description='See documentation of cargo test runner for custom test framework')
+parser.add_argument('runner_args', type=str, nargs='*')
+args = parser.parse_args()
+print("Arguments: {}".format(args.runner_args))
+
+qemu_base_arguments = ['qemu-system-x86_64',
+                       '-display', 'none',
+                       '-smp', str(SMP_CORES),
+                       '-m', str(MEMORY_MB) + 'M',
+                       '-serial', 'stdio',
+                       '-kernel', BOOTLOADER_PATH,
+                       # skip initrd - it depends on test executable
+                       '-cpu', 'qemu64,apic,fsgsbase,rdtscp,xsave,fxsr'
+                       ]
+ok_tests: int = 0
+failed_tests: int = 0
+# This is assuming test_runner only passes executable files as parameters
+for arg in args.runner_args:
+    assert isinstance(arg, str)
+    curr_qemu_arguments = qemu_base_arguments.copy()
+    # ToDo: assert that arg is a path to an executable before calling qemu
+    # ToDo: Add addional test based arguments for qemu / uhyve
+    curr_qemu_arguments.extend(['-initrd', arg])
+    rc, output = run_test(curr_qemu_arguments)
+    test_ok = validate_test(rc, output, arg)
+    test_name = os.path.basename(arg)
+    test_name = clean_test_name(test_name)
+    if test_ok:
+        print("Test Ok: {}".format(test_name))
+        ok_tests += 1
+    else:
+        print("Test failed: {}".format(test_name))
+        failed_tests += 1
+    #Todo: improve information about the test
+
+print("{} from {} tests successful".format(ok_tests, ok_tests + failed_tests))
+# todo print something ala x/y tests failed etc.
+#  maybe look at existing standards (TAP?)
+#  - TAP: could use tappy to convert to python style unit test output (benefit??)
+
+if failed_tests == 0:
+    exit(0)
+else:
+    exit(1)
+
+
+
+
+