Add unit test infrastructure

Jenkinsfile

@@ -27,7 +27,7 @@ pipeline {
             xcoreLibraryChecks("${REPO}")
           }
         }
-        stage('Tests') {
+        stage('XS2 Tests') {
           failFast true
           parallel {
             stage('Legacy tests') {
@@ -35,6 +35,57 @@ pipeline {
                 runXmostest("${REPO}", 'legacy_tests')
               }
             }
+            stage('Unit tests') {
+              steps {
+                dir("${REPO}") {
+                  dir('tests') {
+                    dir('xua_unit_tests') {
+                      viewEnv() {
+                        withVenv {
+                          runWaf('.', "configure clean build --target=xcore200")
+                          runWaf('.', "configure clean build --target=xcoreai")
+                          stash name: 'xua_unit_tests', includes: 'bin/*xcoreai.xe, '
+                          runPython("TARGET=XCORE200 pytest -n 1")
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('xcore.ai Verification') {
+          agent {
+            label 'xcore.ai-explorer'
+          }
+          stages{
+            stage('Get View') {
+              steps {
+                xcorePrepareSandbox("${VIEW}", "${REPO}")
+              }
+            }
+            stage('Unit tests') {
+              steps {
+                dir("${REPO}") {
+                  dir('tests') {
+                    dir('xua_unit_tests') {
+                      viewEnv() {
+                        withVenv {
+                          unstash 'xua_unit_tests'
+                          runPython("TARGET=XCOREAI pytest -n 1 --junitxml pytest_result.xml")
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              cleanWs()
+            }
           }
         }
         stage('xCORE builds') {

tests/xua_unit_tests/config.xscope

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- ======================================================= -->
+<!-- The 'ioMode' attribute on the xSCOPEconfig              -->
+<!-- element can take the following values:                  -->
+<!--   "none", "basic", "timed"                              -->
+<!--                                                         -->
+<!-- The 'type' attribute on Probe                           -->
+<!-- elements can take the following values:                 -->
+<!--   "STARTSTOP", "CONTINUOUS", "DISCRETE", "STATEMACHINE" -->
+<!--                                                         -->
+<!-- The 'datatype' attribute on Probe                       -->
+<!-- elements can take the following values:                 -->
+<!--   "NONE", "UINT", "INT", "FLOAT"                        -->
+<!-- ======================================================= -->
+
+<xSCOPEconfig ioMode="none" enabled="false">
+
+    <!-- For example: -->
+    <!-- <Probe name="Probe Name" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/> -->
+    <!-- From the target code, call: xscope_int(PROBE_NAME, value); -->
+
+</xSCOPEconfig>

tests/xua_unit_tests/conftest.py

@@ -0,0 +1,127 @@
+# Copyright 2021 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+import os.path
+import pytest
+import subprocess
+
+target = os.environ.get('TARGET', 'all_possible')
+print("target = ", target)
+def pytest_collect_file(parent, path):
+    if(path.ext == ".xe"):
+        if(target == 'all_possible'):
+            return UnityTestSource.from_parent(parent, fspath=path)
+        if(target == 'XCOREAI' and ('xcoreai' in path.basename)):
+            return UnityTestSource.from_parent(parent, fspath=path)
+        if(target == 'XCORE200' and ('xcore200' in path.basename)):
+            return UnityTestSource.from_parent(parent, fspath=path)
+
+
+class UnityTestSource(pytest.File):
+    def collect(self):
+        # Find the binary built from the runner for this test file
+        #
+        # Assume the following directory layout:
+        # unit_tests/       <- Test root directory
+        # |-- bin/          <- Compiled binaries of the test runners
+        # |-- conftest.py   <- This file
+        # |-- runners/      <- Auto-generated buildable source of test binaries
+        # |-- src/          <- Unity test functions
+        # `-- wscript       <- Build system file used to generate/build runners
+        print("self.name = ",self.name)
+        #xe_name = ((os.path.basename(self.name)).split("."))[0] + ".xe"
+        #test_bin_path = os.path.join('bin', xe_name)
+        #
+        #test_root_dir_name = os.path.basename(os.path.dirname(__file__))
+        #test_src_path = os.path.basename(str(self.fspath))
+        #test_src_name = os.path.splitext(test_src_path)[0]
+
+        #test_bin_name_si = os.path.join(
+        #    test_src_name + '_single_issue.xe')
+        #test_bin_path_si = os.path.join('bin',
+        #                                test_bin_name_si)
+        #yield UnityTestExecutable.from_parent(self, name=test_bin_path_si)
+
+        #test_bin_name_di = os.path.join(
+        #    test_src_name + '_dual_issue.xe')
+        #test_bin_path_di = os.path.join('bin',
+        #                                test_bin_name_di)
+        yield UnityTestExecutable.from_parent(self, name=self.name)
+
+
+class UnityTestExecutable(pytest.Item):
+    def __init__(self, name, parent):
+        super(UnityTestExecutable, self).__init__(name, parent)
+        self._nodeid = self.name  # Override the naming to suit C better
+
+    def runtest(self):
+        # Run the binary in the simulator
+        simulator_fail = False
+        test_output = None
+        try:
+            if('xcore200' in self.name):
+                print("run axe for executable ", self.name)
+                test_output = subprocess.check_output(['axe', self.name], text=True)
+            else:
+                print("run xrun for executable ", self.name)
+                test_output = subprocess.check_output(['xrun', '--io', '--id', '0', self.name], text=True, stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            # Unity exits non-zero if an assertion fails
+            simulator_fail = True
+            test_output = e.output
+
+        # Parse the Unity output
+        unity_pass = False
+        test_output = test_output.split('\n')
+        for line in test_output:
+            if 'test' in line:
+                test_report = line.split(':')
+                # Unity output is as follows:
+                #   <test_source>:<line_number>:<test_case>:PASS
+                #   <test_source>:<line_number>:<test_case>:FAIL:<failure_reason>
+                test_source = test_report[0]
+                line_number = test_report[1]
+                test_case = test_report[2]
+                result = test_report[3]
+                failure_reason = None
+                print('\n {}()'.format(test_case)),
+                if result == 'PASS':
+                    unity_pass = True
+                    continue
+                if result == 'FAIL':
+                    failure_reason = test_report[4]
+                    print('')  # Insert line break after test_case print
+                    raise UnityTestException(self, {'test_source': test_source,
+                                                    'line_number': line_number,
+                                                    'test_case': test_case,
+                                                    'failure_reason':
+                                                        failure_reason})
+
+        if simulator_fail:
+            raise Exception(self, "Simulation failed.")
+        if not unity_pass:
+            raise Exception(self, "Unity test output not found.")
+        print('')  # Insert line break after final test_case which passed
+
+    def repr_failure(self, excinfo):
+        if isinstance(excinfo.value, UnityTestException):
+            return '\n'.join([str(self.parent).strip('<>'),
+                              '{}:{}:{}()'.format(
+                                    excinfo.value[1]['test_source'],
+                                    excinfo.value[1]['line_number'],
+                                    excinfo.value[1]['test_case']),
+                              'Failure reason:',
+                              excinfo.value[1]['failure_reason']])
+        else:
+            return str(excinfo.value)
+
+    def reportinfo(self):
+        # It's not possible to give sensible line number info for an executable
+        # so we return it as 0.
+        #
+        # The source line number will instead be recovered from the Unity print
+        # statements.
+        return self.fspath, 0, self.name
+
+
+class UnityTestException(Exception):
+    pass

tests/xua_unit_tests/src/test_hid/test_hid.xc

@@ -0,0 +1,7 @@
+// Copyright 2021 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#include "xua_unit_tests.h"
+
+void test_null(){
+    TEST_ASSERT_MESSAGE(1, "Success!");
+}

tests/xua_unit_tests/src/xua_unit_tests.h

@@ -0,0 +1,25 @@
+// Copyright 2018-2021 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#ifndef VTB_UNIT_TESTS_H_
+#define VTB_UNIT_TESTS_H_
+
+#include "unity.h"
+
+#ifdef __XC__
+
+#include <xs1.h>
+#include <string.h>
+#include <math.h>
+
+#include <xclib.h>
+
+#include "audio_test_tools.h"
+#include "voice_toolbox.h"
+#include "voice_toolbox_fp.h"
+#include "vtb_references.h"
+
+#define TEST_ASM 1
+
+#endif // __XC__
+
+#endif /* VTB_UNIT_TESTS_H_ */

tests/xua_unit_tests/wscript

@@ -0,0 +1,274 @@
+from __future__ import print_function
+import glob
+import os.path
+import subprocess
+import sys
+from waflib import Options, Errors
+from waflib.Build import BuildContext, CleanContext
+
+TARGETS = ['xcore200', 'xcoreai']
+
+def get_ruby():
+    """
+    Check ruby is avaliable and return the command to invoke it.
+    """
+    interpreter_name = 'ruby'
+    try:
+        dev_null = open(os.devnull, 'w')
+        # Call the version command to check the interpreter can be run
+        subprocess.check_call([interpreter_name, '--version'],
+                              stdout=dev_null,
+                              close_fds=True)
+    except OSError as e:
+        print("Failed to run Ruby interpreter: {}".format(e), file=sys.stderr)
+        exit(1)  # TODO: Check this is the correct way to kill xwaf on error
+
+    return interpreter_name
+
+
+def get_unity_runner_generator(project_root_path):
+    """
+    Check the Unity generate_test_runner script is avaliable, and return the
+    path to it.
+    """
+    unity_runner_generator = os.path.join(
+        project_root_path, 'Unity', 'auto', 'generate_test_runner.rb')
+    if not os.path.exists(unity_runner_generator):
+        print("Unity repo not found in workspace", file=sys.stderr)
+        exit(1)  # TODO: Check this is the correct way to kill xwaf on error
+    return unity_runner_generator
+
+
+def get_test_name(test_path):
+    """
+    Return the test name by removing the extension from the filename.
+    """
+    return os.path.splitext(os.path.basename(test_path))[0]
+
+
+def get_file_type(filename):
+    """
+    Return the extension from the filename.
+    """
+    return filename.rsplit('.')[-1:][0]
+
+
+def generate_unity_runner(project_root_path, unity_test_path, unity_runner_dir,
+                          unity_runner_suffix):
+    """
+    Invoke the Unity runner generation script for the given test file, and
+    return the path to the generated file. The output directory will be created
+    if it does not already exist.
+    """
+    runner_path = os.path.join(os.path.join(unity_runner_dir, get_test_name(unity_test_path)))
+    if not os.path.exists(runner_path):
+        os.makedirs(runner_path)
+
+    unity_runner_path = os.path.join(
+        runner_path, get_test_name(unity_test_path) + unity_runner_suffix
+        + '.' + 'c')
+    try:
+        subprocess.check_call([get_ruby(),
+                               get_unity_runner_generator(project_root_path),
+                               unity_test_path,
+                               unity_runner_path])
+    except OSError as e:
+        print("Ruby generator failed for {}\n\t{}".format(unity_test_path, e),
+              file=sys.stderr)
+        exit(1)  # TODO: Check this is the correct way to kill xwaf on error
+
+
+def set_common_build_config(waf_conf, project_root_path, unity_test_path,
+                            unity_runner_build_flags):
+    """
+    Set the common xwaf config variables.
+    """
+    waf_conf.load('xwaf.compiler_xcc')
+    waf_conf.env.XCC_FLAGS = unity_runner_build_flags
+    waf_conf.env.PROJECT_ROOT = project_root_path
+    # TODO: can the xwaf boilerplate help here?
+
+
+def add_single_issue_unity_runner_build_config(waf_conf, project_root_path,
+                                               unity_test_path,
+                                               unity_runner_build_flags, target):
+    """
+    Add a single issue config to xwaf to build each Unity test runner into an
+    xCORE executable.
+    """
+    waf_conf.setenv(get_test_name(unity_test_path) + '_single_issue' + '_' + target)
+    set_common_build_config(waf_conf, project_root_path, unity_test_path,
+                            unity_runner_build_flags + '-mno-dual-issue')
+
+
+def add_dual_issue_unity_runner_build_config(waf_conf, project_root_path,
+                                             unity_test_path,
+                                             unity_runner_build_flags, target):
+    """
+    Add a dual issue config to xwaf to build each Unity test runner into an
+    xCORE executable.
+    """
+    waf_conf.setenv(get_test_name(unity_test_path) + '_dual_issue' + '_' + target)
+    set_common_build_config(waf_conf, project_root_path, unity_test_path,
+                            unity_runner_build_flags + '-mdual-issue')
+
+
+def prepare_unity_test_for_build(waf_conf, project_root_path, unity_test_path,
+                                 unity_runner_dir, unity_runner_suffix, target):
+    generate_unity_runner(project_root_path, unity_test_path,
+                          unity_runner_dir, unity_runner_suffix)
+    runner_build_flags = ''  # Could extract flags from the test name
+    add_single_issue_unity_runner_build_config(waf_conf, project_root_path,
+                                               unity_test_path,
+                                               runner_build_flags, target)
+    add_dual_issue_unity_runner_build_config(waf_conf, project_root_path,
+                                             unity_test_path,
+                                             runner_build_flags, target)
+
+
+
+def find_unity_test_paths(unity_test_dir, unity_test_prefix):
+    """
+    Return a list of all file paths with the unity_test_prefix found in the
+    unity_test_dir.
+    """
+    file_list = []
+    for root, dirs, files in os.walk(unity_test_dir):
+        for f in files:
+            if f.startswith(unity_test_prefix):
+                file_list.append(os.path.join(root,f))
+    return file_list
+
+def find_unity_tests(unity_test_dir, unity_test_prefix):
+    """
+    Return a dictionary of all {test names, test language} pairs with the
+    unity_test_prefix found in the unity_test_dir.
+    """
+    unity_test_paths = find_unity_test_paths(unity_test_dir, unity_test_prefix)
+    return {get_test_name(path): get_file_type(path)
+            for path in unity_test_paths}
+
+
+def generate_all_unity_runners(waf_conf, project_root_path,
+                               unity_test_dir, unity_test_prefix,
+                               unity_runner_dir, unity_runner_suffix):
+    """
+    Generate a runner and a build config for each test file in the
+    unity_test_dir.
+    """
+    # FIXME: pass unity_tests in?
+    unity_test_paths = find_unity_test_paths(unity_test_dir, unity_test_prefix)
+    for trgt in TARGETS:
+        for unity_test_path in unity_test_paths:
+            prepare_unity_test_for_build(waf_conf, project_root_path,
+                                         unity_test_path,
+                                         unity_runner_dir, unity_runner_suffix, trgt)
+
+
+# TODO: can the xwaf boilerplate help here?
+def create_waf_contexts(configs):
+    for trgt in TARGETS:
+        for test_name, test_language in configs.items():
+            # Single issue test configurations
+            for ctx in (BuildContext, CleanContext):
+                raw_context = ctx.__name__.replace('Context', '').lower()
+
+                class si_tmp(ctx):
+                    cmd = raw_context + '_' + test_name + '_single_issue' + '_' + trgt
+                    variant = test_name + '_single_issue' + '_' + trgt
+                    source = test_name
+                    language = test_language
+                    target = trgt
+                    runner = test_name
+
+            # Dual issue test configurations
+            for ctx in (BuildContext, CleanContext):
+                raw_context = ctx.__name__.replace('Context', '').lower()
+
+                class di_tmp(ctx):
+                    cmd = raw_context + '_' + test_name + '_dual_issue' + '_' + trgt
+                    variant = test_name + '_dual_issue' + '_' + trgt
+                    source = test_name
+                    language = test_language
+                    target = trgt
+                    runner = test_name
+
+
+UNITY_TEST_DIR = 'src'
+UNITY_TEST_PREFIX = 'test_'
+UNITY_RUNNER_DIR = 'runners'
+UNITY_RUNNER_SUFFIX = '_Runner'
+UNITY_TESTS = find_unity_tests(UNITY_TEST_DIR, UNITY_TEST_PREFIX)
+
+create_waf_contexts(UNITY_TESTS)
+
+def options(opt):
+    opt.load('xwaf.xcommon')
+    opt.add_option('--target', action='store', default='xcore200')
+
+def configure(conf):
+    # TODO: move the call to generate_all_unity_runners() to build()
+    project_root = os.path.join('..', '..', '..')
+    generate_all_unity_runners(conf, project_root,
+                               UNITY_TEST_DIR, UNITY_TEST_PREFIX,
+                               UNITY_RUNNER_DIR, UNITY_RUNNER_SUFFIX)
+    conf.load('xwaf.xcommon')
+
+
+def build(bld):
+    if not bld.variant:
+        trgt = [
+            c for c in TARGETS if c == bld.options.target
+        ]
+
+        if len(trgt) == 0:
+            bld.fatal('specify a target with --target.\nAvailable targets: {}'.format(', '.join(TARGETS)))
+            return
+        print('Adding test runners to build queue')
+        for name in UNITY_TESTS:
+            Options.commands.insert(0, 'build_' + name + '_single_issue' + '_' + trgt[0])
+            Options.commands.insert(0, 'build_' + name + '_dual_issue' + '_' + trgt[0])
+        print('Build queue {}'.format(Options.commands))
+    else:
+        print('Building runner {}'.format(bld.runner))
+        if(bld.target == 'xcoreai'):
+            bld.env.TARGET_ARCH = 'XCORE-AI-EXPLORER'
+        else:
+            bld.env.TARGET_ARCH = 'XCORE-200-EXPLORER'
+
+        bld.env.XSCOPE = bld.path.find_resource('config.xscope')
+        # The issue mode for each build is set during the configure step,
+        # as the string bld.env.XCC_FLAGS. We append this to the list last to
+        # ensure it takes precedence over other flags set here.
+        bld.env.XCC_FLAGS = ['-O2', '-g', '-Wall', '-DUNITY_SUPPORT_64',
+                             '-DUNITY_INCLUDE_DOUBLE', bld.env.XCC_FLAGS]
+
+        depends_on = ['lib_xua', 'Unity']
+        include = ['.']
+        source = [
+            os.path.join(UNITY_TEST_DIR,
+                         '{}.{}'.format(bld.source, bld.language)),
+            os.path.join(UNITY_RUNNER_DIR,
+                         '{}{}.{}'.format(bld.source, UNITY_RUNNER_SUFFIX,
+                                          'c'))]
+        makefile_opts = {}
+        makefile_opts['SOURCE_DIRS'] = [os.path.join('src',bld.runner), os.path.join('runners',bld.runner)]
+        if(bld.target == 'xcoreai'):
+            print('TARGET XCOREAI')
+            makefile_opts['TARGET'] = ['XCORE-AI-EXPLORER']
+        else:
+            print('TARGET XCORE200')
+            makefile_opts['TARGET'] = ['XCORE-200-EXPLORER']
+
+        makefile_opts['INCLUDE_DIRS'] = ['src']
+        makefile_opts['XCC_FLAGS'] = ['-O2', '-g', '-Wall', '-DUNITY_SUPPORT_64', '-DUNITY_INCLUDE_DOUBLE']
+        makefile_opts['APP_NAME'] = [bld.variant]
+        makefile_opts['USED_MODULES'] = depends_on
+        makefile_opts['XCOMMON_MAKEFILE'] = ['Makefile.common']
+        bld.do_xcommon(makefile_opts)
+
+def dist(ctx):
+    ctx.load('xwaf.xcommon')
+
+def distcheck(ctx):
+    ctx.load('xwaf.xcommon')