From ce78849e3d0d0fd2458c6e69ae8e9568b005462c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Wed, 2 Sep 2026 12:04:41 +0200 Subject: [PATCH] Report a failed verification as two .mat files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed comparison used to produce a CSV and a dygraph HTML page per differing variable, plus an index page: `diffSimulationResults` wrote the CSVs itself, testmodel.py the HTML, and every double became text on the way. It now writes two files, the result and the reference reduced to the differing variables with `filterSimulationResults`, and the report links them to the OMPlot page of the OpenModelica playground, which runs the same tube comparison in the browser and plots every variable on demand: https://playground.openmodelica.org/latest/omplot/?result=…&reference=… The page needs absolute URLs, so library.html.tpl resolves the two relative paths at load time; without JavaScript the link is the result file itself. `diffSimulationResults` is called with an empty prefix, which (since the matching omc change) writes no per-variable CSV. Filtering a CSV reference needs an omc with that change as well; an older one logs the failure to the .err and the report has no reference file for that model. The files server has to allow cross-origin reads (Access-Control- Allow-Origin) for the page to fetch them. The docker images were updated to pull in the nightly omc since it is needed for the small API changes to `filterSimulationResults`. Assisted-by: Claude Fable 5.1 --- .CI/build-dep/Dockerfile | 13 +++++++ .CI/testing/Dockerfile | 6 ++-- library.html.tpl | 10 ++++++ test.py | 13 +++---- testmodel.py | 75 ++++++++++------------------------------ 5 files changed, 47 insertions(+), 70 deletions(-) diff --git a/.CI/build-dep/Dockerfile b/.CI/build-dep/Dockerfile index 5f329b7..b04c872 100644 --- a/.CI/build-dep/Dockerfile +++ b/.CI/build-dep/Dockerfile @@ -5,3 +5,16 @@ FROM docker.openmodelica.org/build-deps:ubuntu-22.04 # and python3-pip because the line below needs it and the base may not carry it. RUN apt-get update && apt-get install -y libxml2 libxslt1.1 libxml2-dev libxslt1-dev python3-pip RUN pip3 install matplotlib FMPy psycopg2-binary + +# The omc for --ompython_omhome (the tested one is built in the workspace). +ARG OMC_VERSION= +RUN export DEBIAN_FRONTEND=noninteractive \ + && . /etc/os-release \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openmodelica-keyring.gpg]" \ + "https://build.openmodelica.org/apt ${VERSION_CODENAME} nightly" \ + > /etc/apt/sources.list.d/openmodelica.list \ + && apt-get update \ + && apt-get install -qy omc${OMC_VERSION:+=${OMC_VERSION}} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && omc --version diff --git a/.CI/testing/Dockerfile b/.CI/testing/Dockerfile index 35bc5a7..33b9401 100644 --- a/.CI/testing/Dockerfile +++ b/.CI/testing/Dockerfile @@ -40,14 +40,12 @@ RUN python3 -m venv /opt/libtest-venv \ && rm /tmp/requirements.txt ENV PATH=/opt/libtest-venv/bin:$PATH -# The omc for --ompython_omhome (the tested one is built in the workspace). The -# base image points apt at the nightly component; use stable instead. -# OMC_VERSION pins a version, empty takes the newest stable one. +# The omc for --ompython_omhome (the tested one is built in the workspace). ARG OMC_VERSION= RUN export DEBIAN_FRONTEND=noninteractive \ && . /etc/os-release \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openmodelica-keyring.gpg]" \ - "https://build.openmodelica.org/apt ${VERSION_CODENAME} stable" \ + "https://build.openmodelica.org/apt ${VERSION_CODENAME} nightly" \ > /etc/apt/sources.list.d/openmodelica.list \ && apt-get update \ && apt-get install -qy omc${OMC_VERSION:+=${OMC_VERSION}} \ diff --git a/library.html.tpl b/library.html.tpl index 6d17218..9813ba9 100644 --- a/library.html.tpl +++ b/library.html.tpl @@ -55,5 +55,15 @@ Config: 
#config#
ModelVerifiedSimulateTotal buildModelParsingFrontendBackendSimCodeTemplatesCompileTotal Execution #testsHTML# + diff --git a/test.py b/test.py index fb1445c..2a07437 100755 --- a/test.py +++ b/test.py @@ -1456,19 +1456,14 @@ def publishLibrary(libname): filename_prefix = "files/%s_%s" % (s[2],s[1]) stagePublished(stageRoot, workspace_prefix, filename_prefix, suffix) stageShared(stageRoot, "files/%s_%s" % (s[2],s[1]), suffix) - filesList.write("/%s*diff*csv\n" % filename_prefix) - filesList.write("/%s*diff*html\n" % filename_prefix) if is_non_zero_file(workspace_prefix+".sim"): filesList.write("/%s.sim\n" % filename_prefix) errPrefix = "files/%s_%s" % (s[2],s[1]) if is_non_zero_file(errPrefix+".err"): filesList.write("/%s.err\n" % errPrefix) - variables = (s[3].get("diff") or {}).get("vars") or [] - if len(variables)>0: - filesList.write("/%s.diff.html\n" % filename_prefix) - for v in variables: - filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) - filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) + if (s[3].get("diff") or {}).get("vars"): + filesList.write("/%s.diff.mat\n" % filename_prefix) + filesList.write("/%s.diff.ref.mat\n" % filename_prefix) filesList.close() testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % (lambda filename_prefix, errPrefix, diff: @@ -1479,7 +1474,7 @@ def publishLibrary(libname): # Nothing was compared, whatever phase the model reports. " " if diff is None else (("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else - ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), + ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), checkPhase(s[3]["phase"], 6), timeSeconds(s[3].get("sim") or 0), checkPhase(s[3]["phase"], 5), diff --git a/testmodel.py b/testmodel.py index e976bf9..5bf3990 100755 --- a/testmodel.py +++ b/testmodel.py @@ -1078,8 +1078,9 @@ def verifyAgainstReference(resFile, prefix, stat): fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) start=monotonic() try: - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % - (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) + # An empty diffPrefix: no per-variable CSV, the two filtered files below are the report. + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % + (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) except TimeoutError as e: with open(errFile, 'a+') as fp: fp.write("Timeout error for diffSimulationResults") @@ -1098,63 +1099,23 @@ def verifyAgainstReference(resFile, prefix, stat): resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) fp.write("\nVariables in the result:" ) fp.write(",".join(resVars)+"\n") - diffFiles = [prefix + "." + var for var in diffVars] stat["diff"]["vars"]=diffVars + # Both files reduced to the differing variables: what the report's OMPlot + # link compares, instead of a CSV and an HTML page per variable. + filterDiffVars(resFile, prefix + ".mat", diffVars) + filterDiffVars(referenceFile, prefix + ".ref.mat", diffVars) - # Create a file containing only the calibrated variables, for easy display - lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) - with open(prefix+".html", 'w') as fp: - fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) - for var in diffVars: - if "/" in var: - continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below - with open(prefix+"."+var+".html", 'w') as fp: - fp.write(""" - - - - - -
    -

    - - - - - - - - - - - - Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    - - - """ % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) - +def filterDiffVars(inFile, outFile, diffVars): + vars = ", ".join('"%s"' % v.replace('\\', '\\\\').replace('"', '\\"') for v in diffVars) + try: + ok = sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "%s", vars={%s}, removeDescription=false)' % (inFile, outFile, vars), conf["ulimitOmc"]) + except TimeoutError as e: + ok = False + if not ok: + with open(errFile, 'a+') as fp: + fp.write("Failed to write the differing variables of %s to %s:\n%s" % (inFile, outFile, omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False))) + if os.path.exists(outFile): + os.remove(outFile) # The first simulator's results are the ones every non-FMI code path expects. # There is nothing to compare when it never produced any.