Add RISCOF test suite (doesn't function yet)

.gitignore

@@ -19,4 +19,6 @@ Testing
 result
 build
 example/example
-example/example.raw
+example/example.raw
+__pycache__
+*.pyc

rve.nix

@@ -22,6 +22,10 @@ pkgs.gcc13Stdenv.mkDerivation rec {
     pkgsCross.riscv32-embedded.buildPackages.gcc
     pkgsCross.riscv32-embedded.buildPackages.binutils
     pkgsCross.riscv32-embedded.buildPackages.gdb
+    sail-riscv-rv32
+    (python3.withPackages (ps: with ps; [
+      riscof
+    ]))
   ];
 
   hardeningDisable = [ "all" ];

test/config.ini

@@ -0,0 +1,14 @@
+[RISCOF]
+ReferencePlugin=sail_cSim
+ReferencePluginPath=./sail_cSim
+DUTPlugin=rve
+DUTPluginPath=./rve
+
+[rve]
+pluginpath=./rve
+ispec=./rve/rve_isa.yaml
+pspec=./rve/rve_platform.yaml
+target_run=1
+
+[sail_cSim]
+pluginpath=./sail_cSim

test/rve/env/link.ld

@@ -0,0 +1,18 @@
+OUTPUT_ARCH( "riscv" )
+ENTRY(rvtest_entry_point)
+
+SECTIONS
+{
+  . = 0x80000000;
+  .text.init : { *(.text.init) }
+  . = ALIGN(0x1000);
+  .tohost : { *(.tohost) }
+  . = ALIGN(0x1000);
+  .text : { *(.text) }
+  . = ALIGN(0x1000);
+  .data : { *(.data) }
+  .data.string : { *(.data.string)}
+  .bss : { *(.bss) }
+  _end = .;
+}
+

test/rve/env/model_test.h

@@ -0,0 +1,60 @@
+#ifndef _COMPLIANCE_MODEL_H
+#define _COMPLIANCE_MODEL_H
+#define RVMODEL_DATA_SECTION \
+        .pushsection .tohost,"aw",@progbits;                            \
+        .align 8; .global tohost; tohost: .dword 0;                     \
+        .align 8; .global fromhost; fromhost: .dword 0;                 \
+        .popsection;                                                    \
+        .align 8; .global begin_regstate; begin_regstate:               \
+        .word 128;                                                      \
+        .align 8; .global end_regstate; end_regstate:                   \
+        .word 4;
+
+//RV_COMPLIANCE_HALT
+#define RVMODEL_HALT                                              \
+  li x1, 1;                                                                   \
+  write_tohost:                                                               \
+    sw x1, tohost, t5;                                                        \
+    j write_tohost;
+
+#define RVMODEL_BOOT
+
+//RV_COMPLIANCE_DATA_BEGIN
+#define RVMODEL_DATA_BEGIN                                              \
+  RVMODEL_DATA_SECTION                                                        \
+  .align 4;\
+  .global begin_signature; begin_signature:
+
+//RV_COMPLIANCE_DATA_END
+#define RVMODEL_DATA_END                                                      \
+  .align 4;\
+  .global end_signature; end_signature:  
+
+//RVTEST_IO_INIT
+#define RVMODEL_IO_INIT
+//RVTEST_IO_WRITE_STR
+#define RVMODEL_IO_WRITE_STR(_R, _STR)
+//RVTEST_IO_CHECK
+#define RVMODEL_IO_CHECK()
+//RVTEST_IO_ASSERT_GPR_EQ
+#define RVMODEL_IO_ASSERT_GPR_EQ(_S, _R, _I)
+//RVTEST_IO_ASSERT_SFPR_EQ
+#define RVMODEL_IO_ASSERT_SFPR_EQ(_F, _R, _I)
+//RVTEST_IO_ASSERT_DFPR_EQ
+#define RVMODEL_IO_ASSERT_DFPR_EQ(_D, _R, _I)
+
+#define RVMODEL_SET_MSW_INT       \
+ li t1, 1;                         \
+ li t2, 0x2000000;                 \
+ sw t1, 0(t2);
+
+#define RVMODEL_CLEAR_MSW_INT     \
+ li t2, 0x2000000;                 \
+ sw x0, 0(t2);
+
+#define RVMODEL_CLEAR_MTIMER_INT
+
+#define RVMODEL_CLEAR_MEXT_INT
+
+
+#endif // _COMPLIANCE_MODEL_H

test/rve/riscof_rve.py

@@ -0,0 +1,240 @@
+import os
+import re
+import shutil
+import subprocess
+import shlex
+import logging
+import random
+import string
+from string import Template
+import sys
+
+import riscof.utils as utils
+import riscof.constants as constants
+from riscof.pluginTemplate import pluginTemplate
+
+logger = logging.getLogger()
+
+class rve(pluginTemplate):
+    __model__ = "rve"
+
+    #TODO: please update the below to indicate family, version, etc of your DUT.
+    __version__ = "XXX"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        config = kwargs.get('config')
+
+        # If the config node for this DUT is missing or empty. Raise an error. At minimum we need
+        # the paths to the ispec and pspec files
+        if config is None:
+            print("Please enter input file paths in configuration.")
+            raise SystemExit(1)
+
+        self.dut_exe = os.path.join(config['PATH'] if 'PATH' in config else "","rve")
+
+        self.num_jobs = str(config['jobs'] if 'jobs' in config else 1)
+
+        self.pluginpath=os.path.abspath(config['pluginpath'])
+
+        self.isa_spec = os.path.abspath(config['ispec'])
+        self.platform_spec = os.path.abspath(config['pspec'])
+
+        #We capture if the user would like the run the tests on the target or
+        #not. If you are interested in just compiling the tests and not running
+        #them on the target, then following variable should be set to False
+        if 'target_run' in config and config['target_run']=='0':
+            self.target_run = False
+        else:
+            self.target_run = True
+
+    def initialise(self, suite, work_dir, archtest_env):
+
+       # capture the working directory. Any artifacts that the DUT creates should be placed in this
+       # directory. Other artifacts from the framework and the Reference plugin will also be placed
+       # here itself.
+       self.work_dir = work_dir
+
+       # capture the architectural test-suite directory.
+       self.suite_dir = suite
+
+       # Note the march is not hardwired here, because it will change for each
+       # test. Similarly the output elf name and compile macros will be assigned later in the
+       # runTests function
+       self.compile_cmd = 'riscv{1}-unknown-elf-gcc -march={0} \
+         -static -mcmodel=medany -fvisibility=hidden -nostdlib -nostartfiles -g\
+         -T '+self.pluginpath+'/env/link.ld\
+         -I '+self.pluginpath+'/env/\
+         -I ' + archtest_env + ' {2} -o {3} {4}'
+
+       # add more utility snippets here
+
+    def build(self, isa_yaml, platform_yaml):
+
+      # load the isa yaml as a dictionary in python.
+      ispec = utils.load_yaml(isa_yaml)['hart0']
+
+      # capture the XLEN value by picking the max value in 'supported_xlen' field of isa yaml. This
+      # will be useful in setting integer value in the compiler string (if not already hardcoded);
+      self.xlen = ('64' if 64 in ispec['supported_xlen'] else '32')
+
+      # for sail start building the '--isa' argument. the self.isa is dutnmae specific and may not be
+      # useful for all DUTs
+      self.isa = 'rv' + self.xlen
+      if "I" in ispec["ISA"]:
+          self.isa += 'i'
+      if "M" in ispec["ISA"]:
+          self.isa += 'm'
+      if "F" in ispec["ISA"]:
+          self.isa += 'f'
+      if "D" in ispec["ISA"]:
+          self.isa += 'd'
+      if "C" in ispec["ISA"]:
+          self.isa += 'c'
+
+      #TODO: The following assumes you are using the riscv-gcc toolchain. If
+      #      not please change appropriately
+      self.compile_cmd = self.compile_cmd+' -mabi='+('lp64 ' if 64 in ispec['supported_xlen'] else 'ilp32 ')
+
+    def runTests(self, testList):
+
+      # Delete Makefile if it already exists.
+      if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
+      # create an instance the makeUtil class that we will use to create targets.
+      make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
+
+      # set the make command that will be used. The num_jobs parameter was set in the __init__
+      # function earlier
+      make.makeCommand = 'make -k -j' + self.num_jobs
+
+      # we will iterate over each entry in the testList. Each entry node will be refered to by the
+      # variable testname.
+      for testname in testList:
+
+          # for each testname we get all its fields (as described by the testList format)
+          testentry = testList[testname]
+
+          # we capture the path to the assembly file of this test
+          test = testentry['test_path']
+
+          # capture the directory where the artifacts of this test will be dumped/created. RISCOF is
+          # going to look into this directory for the signature files
+          test_dir = testentry['work_dir']
+
+          # name of the elf file after compilation of the test
+          elf = 'my.elf'
+
+          # name of the signature file as per requirement of RISCOF. RISCOF expects the signature to
+          # be named as DUT-<dut-name>.signature. The below variable creates an absolute path of
+          # signature file.
+          sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
+
+          # for each test there are specific compile macros that need to be enabled. The macros in
+          # the testList node only contain the macros/values. For the gcc toolchain we need to
+          # prefix with "-D". The following does precisely that.
+          compile_macros= ' -D' + " -D".join(testentry['macros'])
+
+          # substitute all variables in the compile command that we created in the initialize
+          # function
+          cmd = self.compile_cmd.format(testentry['isa'].lower(), self.xlen, test, elf, compile_macros)
+
+          # if the user wants to disable running the tests and only compile the tests, then
+          # the "else" clause is executed below assigning the sim command to simple no action
+          # echo statement.
+          if self.target_run:
+            # set up the simulation command. Template is for spike. Please change.
+            simcmd = self.dut_exe + ' {0}'.format(elf)
+          else:
+            simcmd = 'echo "NO RUN"'
+
+          # concatenate all commands that need to be executed within a make-target.
+          execute = '@cd {0}; {1}; {2};'.format(testentry['work_dir'], cmd, simcmd)
+
+          # create a target. The makeutil will create a target with the name "TARGET<num>" where num
+          # starts from 0 and increments automatically for each new target that is added
+          make.add_target(execute)
+
+      # if you would like to exit the framework once the makefile generation is complete uncomment the
+      # following line. Note this will prevent any signature checking or report generation.
+      #raise SystemExit
+
+      # once the make-targets are done and the makefile has been created, run all the targets in
+      # parallel using the make command set above.
+      make.execute_all(self.work_dir)
+
+      # if target runs are not required then we simply exit as this point after running all
+      # the makefile targets.
+      if not self.target_run:
+          raise SystemExit(0)
+
+#The following is an alternate template that can be used instead of the above.
+#The following template only uses shell commands to compile and run the tests.
+
+#    def runTests(self, testList):
+#
+#      # we will iterate over each entry in the testList. Each entry node will be referred to by the
+#      # variable testname.
+#      for testname in testList:
+#
+#          logger.debug('Running Test: {0} on DUT'.format(testname))
+#          # for each testname we get all its fields (as described by the testList format)
+#          testentry = testList[testname]
+#
+#          # we capture the path to the assembly file of this test
+#          test = testentry['test_path']
+#
+#          # capture the directory where the artifacts of this test will be dumped/created.
+#          test_dir = testentry['work_dir']
+#
+#          # name of the elf file after compilation of the test
+#          elf = 'my.elf'
+#
+#          # name of the signature file as per requirement of RISCOF. RISCOF expects the signature to
+#          # be named as DUT-<dut-name>.signature. The below variable creates an absolute path of
+#          # signature file.
+#          sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
+#
+#          # for each test there are specific compile macros that need to be enabled. The macros in
+#          # the testList node only contain the macros/values. For the gcc toolchain we need to
+#          # prefix with "-D". The following does precisely that.
+#          compile_macros= ' -D' + " -D".join(testentry['macros'])
+#
+#          # collect the march string required for the compiler
+#          marchstr = testentry['isa'].lower()
+#
+#          # substitute all variables in the compile command that we created in the initialize
+#          # function
+#          cmd = self.compile_cmd.format(marchstr, self.xlen, test, elf, compile_macros)
+#
+#          # just a simple logger statement that shows up on the terminal
+#          logger.debug('Compiling test: ' + test)
+#
+#          # the following command spawns a process to run the compile command. Note here, we are
+#          # changing the directory for this command to that pointed by test_dir. If you would like
+#          # the artifacts to be dumped else where change the test_dir variable to the path of your
+#          # choice.
+#          utils.shellCommand(cmd).run(cwd=test_dir)
+#
+#          # for debug purposes if you would like stop the DUT plugin after compilation, you can
+#          # comment out the lines below and raise a SystemExit
+#
+#          if self.target_run:
+#            # build the command for running the elf on the DUT. In this case we use spike and indicate
+#            # the isa arg that we parsed in the build stage, elf filename and signature filename.
+#            # Template is for spike. Please change for your DUT
+#            execute = self.dut_exe + ' --isa={0} +signature={1} +signature-granularity=4 {2}'.format(self.isa, sig_file, elf)
+#            logger.debug('Executing on Spike ' + execute)
+#
+#          # launch the execute command. Change the test_dir if required.
+#          utils.shellCommand(execute).run(cwd=test_dir)
+#
+#          # post-processing steps can be added here in the template below
+#          #postprocess = 'mv {0} temp.sig'.format(sig_file)'
+#          #utils.shellCommand(postprocess).run(cwd=test_dir)
+#
+#      # if target runs are not required then we simply exit as this point after running all
+#      # the makefile targets.
+#      if not self.target_run:
+#          raise SystemExit

test/rve/rve_isa.yaml

@@ -0,0 +1,28 @@
+hart_ids: [0]
+hart0:
+  ISA: RV32IM
+  physical_addr_sz: 32
+  User_Spec_Version: '2.3'
+  supported_xlen: [32]
+  misa:
+   reset-val: 0x40001104
+   rv32:
+     accessible: true
+     mxl:
+       implemented: true
+       type:
+           warl:
+              dependency_fields: []
+              legal:
+                - mxl[1:0] in [0x1]
+              wr_illegal:
+                - Unchanged
+     extensions:
+       implemented: true
+       type:
+           warl:
+              dependency_fields: []
+              legal:
+                - extensions[25:0] bitmask [0x0001104, 0x0000000]
+              wr_illegal:
+                - Unchanged

test/rve/rve_platform.yaml

@@ -0,0 +1,10 @@
+mtime:
+  implemented: true
+  address: 0xbff8
+mtimecmp:
+  implemented: true
+  address: 0x4000
+nmi:
+  label: nmi_vector
+reset:
+  label: reset_vector

test/sail_cSim/__init__.py

@@ -0,0 +1,2 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

test/sail_cSim/env/link.ld

@@ -0,0 +1,18 @@
+OUTPUT_ARCH( "riscv" )
+ENTRY(rvtest_entry_point)
+
+SECTIONS
+{
+  . = 0x80000000;
+  .text.init : { *(.text.init) }
+  . = ALIGN(0x1000);
+  .tohost : { *(.tohost) }
+  . = ALIGN(0x1000);
+  .text : { *(.text) }
+  . = ALIGN(0x1000);
+  .data : { *(.data) }
+  .data.string : { *(.data.string)}
+  .bss : { *(.bss) }
+  _end = .;
+}
+

test/sail_cSim/env/model_test.h

@@ -0,0 +1,55 @@
+#ifndef _COMPLIANCE_MODEL_H
+#define _COMPLIANCE_MODEL_H
+
+#define RVMODEL_DATA_SECTION \
+        .pushsection .tohost,"aw",@progbits;                            \
+        .align 8; .global tohost; tohost: .dword 0;                     \
+        .align 8; .global fromhost; fromhost: .dword 0;                 \
+        .popsection;                                                    \
+        .align 8; .global begin_regstate; begin_regstate:               \
+        .word 128;                                                      \
+        .align 8; .global end_regstate; end_regstate:                   \
+        .word 4;
+
+//RV_COMPLIANCE_HALT
+#define RVMODEL_HALT                                              \
+  li x1, 1;                                                                   \
+  write_tohost:                                                               \
+    sw x1, tohost, t5;                                                        \
+    j write_tohost;
+
+#define RVMODEL_BOOT
+
+//RV_COMPLIANCE_DATA_BEGIN
+#define RVMODEL_DATA_BEGIN                                              \
+  RVMODEL_DATA_SECTION                                                        \
+  .align 4;\
+  .global begin_signature; begin_signature:
+
+//RV_COMPLIANCE_DATA_END
+#define RVMODEL_DATA_END                                                      \
+  .align 4; .global end_signature; end_signature:  
+
+//RVTEST_IO_INIT
+#define RVMODEL_IO_INIT
+//RVTEST_IO_WRITE_STR
+#define RVMODEL_IO_WRITE_STR(_R, _STR)
+//RVTEST_IO_CHECK
+#define RVMODEL_IO_CHECK()
+//RVTEST_IO_ASSERT_GPR_EQ
+#define RVMODEL_IO_ASSERT_GPR_EQ(_S, _R, _I)
+//RVTEST_IO_ASSERT_SFPR_EQ
+#define RVMODEL_IO_ASSERT_SFPR_EQ(_F, _R, _I)
+//RVTEST_IO_ASSERT_DFPR_EQ
+#define RVMODEL_IO_ASSERT_DFPR_EQ(_D, _R, _I)
+
+#define RVMODEL_SET_MSW_INT
+
+#define RVMODEL_CLEAR_MSW_INT
+
+#define RVMODEL_CLEAR_MTIMER_INT
+
+#define RVMODEL_CLEAR_MEXT_INT
+
+
+#endif // _COMPLIANCE_MODEL_H

test/sail_cSim/riscof_sail_cSim.py

@@ -0,0 +1,124 @@
+import os
+import re
+import shutil
+import subprocess
+import shlex
+import logging
+import random
+import string
+from string import Template
+
+import riscof.utils as utils
+from riscof.pluginTemplate import pluginTemplate
+import riscof.constants as constants
+from riscv_isac.isac import isac
+
+logger = logging.getLogger()
+
+class sail_cSim(pluginTemplate):
+    __model__ = "sail_c_simulator"
+    __version__ = "0.5.0"
+
+    def __init__(self, *args, **kwargs):
+        sclass = super().__init__(*args, **kwargs)
+
+        config = kwargs.get('config')
+        if config is None:
+            logger.error("Config node for sail_cSim missing.")
+            raise SystemExit(1)
+        self.num_jobs = str(config['jobs'] if 'jobs' in config else 1)
+        self.pluginpath = os.path.abspath(config['pluginpath'])
+        self.sail_exe = { '32' : os.path.join(config['PATH'] if 'PATH' in config else "","riscv_sim_RV32"),
+                '64' : os.path.join(config['PATH'] if 'PATH' in config else "","riscv_sim_RV64")}
+        self.isa_spec = os.path.abspath(config['ispec']) if 'ispec' in config else ''
+        self.platform_spec = os.path.abspath(config['pspec']) if 'ispec' in config else ''
+        self.make = config['make'] if 'make' in config else 'make'
+        logger.debug("SAIL CSim plugin initialised using the following configuration.")
+        for entry in config:
+            logger.debug(entry+' : '+config[entry])
+        return sclass
+
+    def initialise(self, suite, work_dir, archtest_env):
+        self.suite = suite
+        self.work_dir = work_dir
+        self.objdump_cmd = 'riscv{1}-unknown-elf-objdump -D {0} > {2};'
+        self.compile_cmd = 'riscv{1}-unknown-elf-gcc -march={0} \
+         -static -mcmodel=medany -fvisibility=hidden -nostdlib -nostartfiles\
+         -T '+self.pluginpath+'/env/link.ld\
+         -I '+self.pluginpath+'/env/\
+         -I ' + archtest_env
+
+    def build(self, isa_yaml, platform_yaml):
+        ispec = utils.load_yaml(isa_yaml)['hart0']
+        self.xlen = ('64' if 64 in ispec['supported_xlen'] else '32')
+        self.isa = 'rv' + self.xlen
+        self.compile_cmd = self.compile_cmd+' -mabi='+('lp64 ' if 64 in ispec['supported_xlen'] else 'ilp32 ')
+        if "I" in ispec["ISA"]:
+            self.isa += 'i'
+        if "M" in ispec["ISA"]:
+            self.isa += 'm'
+        if "C" in ispec["ISA"]:
+            self.isa += 'c'
+        if "F" in ispec["ISA"]:
+            self.isa += 'f'
+        if "D" in ispec["ISA"]:
+            self.isa += 'd'
+        objdump = "riscv{0}-unknown-elf-objdump".format(self.xlen)
+        if shutil.which(objdump) is None:
+            logger.error(objdump+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        compiler = "riscv{0}-unknown-elf-gcc".format(self.xlen)
+        if shutil.which(compiler) is None:
+            logger.error(compiler+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        if shutil.which(self.sail_exe[self.xlen]) is None:
+            logger.error(self.sail_exe[self.xlen]+ ": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+        if shutil.which(self.make) is None:
+            logger.error(self.make+": executable not found. Please check environment setup.")
+            raise SystemExit(1)
+
+
+    def runTests(self, testList, cgf_file=None):
+        if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
+        make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
+        make.makeCommand = self.make + ' -j' + self.num_jobs
+        for file in testList:
+            testentry = testList[file]
+            test = testentry['test_path']
+            test_dir = testentry['work_dir']
+            test_name = test.rsplit('/',1)[1][:-2]
+
+            elf = 'ref.elf'
+
+            execute = "@cd "+testentry['work_dir']+";"
+
+            cmd = self.compile_cmd.format(testentry['isa'].lower(), self.xlen) + ' ' + test + ' -o ' + elf
+            compile_cmd = cmd + ' -D' + " -D".join(testentry['macros'])
+            execute+=compile_cmd+";"
+
+            execute += self.objdump_cmd.format(elf, self.xlen, 'ref.disass')
+            sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
+
+            execute += self.sail_exe[self.xlen] + ' --test-signature={0} {1} > {2}.log 2>&1;'.format(sig_file, elf, test_name)
+
+            cov_str = ' '
+            for label in testentry['coverage_labels']:
+                cov_str+=' -l '+label
+
+            if cgf_file is not None:
+                coverage_cmd = 'riscv_isac --verbose info coverage -d \
+                        -t {0}.log --parser-name c_sail -o coverage.rpt  \
+                        --sig-label begin_signature  end_signature \
+                        --test-label rvtest_code_begin rvtest_code_end \
+                        -e ref.elf -c {1} -x{2} {3};'.format(\
+                        test_name, ' -c '.join(cgf_file), self.xlen, cov_str)
+            else:
+                coverage_cmd = ''
+
+
+            execute+=coverage_cmd
+
+            make.add_target(execute)
+        make.execute_all(self.work_dir)