Add a variety of automated tests

test/src/tr_test/test.py

@@ -28,9 +28,10 @@ import pylink
 import time
 import os
 import sys
+import re
 
 PYTEST_DIR = os.path.dirname(os.path.abspath(__file__))
-DEFAULT_FW_DIR =  os.path.abspath(PYTEST_DIR + "../../../../firmware/")
+DEFAULT_FW_DIR = os.path.abspath(PYTEST_DIR + "../../../../firmware/")
 SAMPLE_TIME_MS = 2500
 SAMPLES_PER_SEC = 12000000
 DRIVER = "fx2lafw"
@@ -39,18 +40,20 @@ TEST_START_TEXT = b"TEST_BEGIN\r\n"
 TEST_PASS_TEXT = b"TEST_PASS\r\n"
 TEST_FAIL_TEXT = b"TEST_FAIL\r\n"
 
+
 @pytest.fixture
 def logger():
     logging.basicConfig()
     return logging.getLogger(__name__)
 
+
 @pytest.fixture
 def context_factory():
 
     def create_context(fw_rel_path: str,
-                       mcu: str="STM32L412RB",
-                       addr: int=0x8000000,
-                       leave_halted: bool=False):
+                       mcu: str = "STM32L412RB",
+                       addr: int = 0x8000000,
+                       leave_halted: bool = False):
 
         proc = subprocess.run(["make", "BOARD=devboard", fw_rel_path],
                               cwd=DEFAULT_FW_DIR,
@@ -72,7 +75,6 @@ def context_factory():
         jlink.disable_dialog_boxes()
         jlink.set_tif(pylink.enums.JLinkInterfaces.SWD)
         jlink.connect(mcu)
-        #jlink.set_reset_strategy(pylink.enums.JLinkResetStrategyCortexM3.HALT_AFTER_BTL)
         fw = DEFAULT_FW_DIR + "/" + fw_rel_path
         logging.info("Flashing {}...".format(fw))
         jlink.flash_file(fw, addr)
@@ -80,7 +82,9 @@ def context_factory():
         assert jlink.halted()
         jlink.reset(halt=True)
 
-        serial_dev = serial.Serial(port=ports[0].device, baudrate=115200, timeout=1)
+        serial_dev = serial.Serial(port=ports[0].device,
+                                   baudrate=115200,
+                                   timeout=1)
         if leave_halted:
             return serial_dev, jlink
 
@@ -89,7 +93,9 @@ def context_factory():
         while True:
             try:
                 logging.info("Waiting for firmware to start...")
-                assert serial_dev.read_until(TEST_START_TEXT).endswith(TEST_START_TEXT), \
+                assert serial_dev \
+                    .read_until(TEST_START_TEXT) \
+                    .endswith(TEST_START_TEXT), \
                     "Timed out starting test firmware application"
                 logging.debug("Test execution started")
             except serial.serialutil.SerialException:
@@ -99,20 +105,43 @@ def context_factory():
     return create_context
 
 
-def test_basic(context_factory, logger):
-    serial_dev, jlink = context_factory("Test/basic.bin")
+def test_meta_pass(context_factory, logger):
+    serial_dev, jlink = context_factory("Test/pass.bin")
     text = serial_dev.read_until(TEST_PASS_TEXT)
     print("Got serial output: {}".format(text))
     assert text.endswith(TEST_PASS_TEXT)
 
 
+def test_meta_fail(context_factory, logger):
+    serial_dev, jlink = context_factory("Test/fail.bin")
+    text = serial_dev.read_until(TEST_PASS_TEXT)
+    print("Got serial output: {}".format(text))
+    assert not text.endswith(TEST_PASS_TEXT)
+
+
+def test_meta_timeout(context_factory, logger):
+    serial_dev, jlink = context_factory("Test/timeout.bin")
+    text = serial_dev.read_until(TEST_PASS_TEXT)
+    assert not text.endswith(TEST_PASS_TEXT)
+
+
+def test_meta_nostart(context_factory, logger):
+    with pytest.raises(AssertionError):
+        serial_dev, jlink = context_factory("Test/no_start.bin")
+
+
 def test_watch(context_factory, logger):
-    serial_dev, jlink = context_factory("Application/main.bin", leave_halted=True)
+    serial_dev, jlink = context_factory("Application/main.bin",
+                                        leave_halted=True)
     jlink.reset(halt=False)
 
+
 def test_clock(context_factory, logger):
     serial_dev, jlink = context_factory("Test/clock.bin")
-    serial_dev.timeout = 1.5
+    EXPECTED_RUNTIME = 10
+    TOLERANCE = .1
+
+    serial_dev.timeout = EXPECTED_RUNTIME * 1.2
 
     START_MARKER = b"GO\r\n"
     END_MARKER = b"STOP\r\n"
@@ -133,10 +162,37 @@ def test_clock(context_factory, logger):
     # accurate. Add support via sigrok.
     assert start_text.endswith(START_MARKER)
     assert end_text.endswith(END_MARKER)
-    assert delta < 1.1 and delta > .9
+    assert (delta - EXPECTED_RUNTIME) < TOLERANCE
+
+
+def test_stop(context_factory, logger):
+    serial_dev, jlink = context_factory("Test/stop.bin")
+    serial_dev.timeout = 65
+
+    pattern = re.compile("Requested=(\\d*) Actual=(\\d*)")
+    while True:
+        line = serial_dev.readline()
+        if line == TEST_PASS_TEXT:
+            break
+
+        line = line.decode("ascii", errors="ignore")
+        print(line.strip())
+        match = pattern.match(line)
+        assert match
+        req = int(match.group(1))
+        actual = int(match.group(2))
+        delta = req - actual
+
+        if req < 32000:
+            assert abs(delta < req * (2.0 / 100.0)) or (delta <= 1)
+        else:
+            # Delays > 32sec have reduced resolution (1 sec)
+            assert abs(delta) < 1000
+
 
 def main():
     pytest.main(sys.argv)
 
+
 if __name__ == "__main__":
     main()