# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1645273177 -3600
# Node ID 81021f0793870627ad1fa04b76439f6c93f0da29
# Parent  1382d6a6e782e16fda49b59e491172ee0612afac
GitHub-CI: Analyze test results for OS X and Windows runners.

* .github/workflows/make.yaml (macos, windows): Add step to analyze results of
running the test suite. Don't make it an error (for now) because some tests are
consistently failing on those runners currently.

diff -r 1382d6a6e782 -r 81021f079387 .github/workflows/make.yaml
--- a/.github/workflows/make.yaml	Fri Feb 18 22:20:41 2022 -0500
+++ b/.github/workflows/make.yaml	Sat Feb 19 13:19:37 2022 +0100
@@ -278,14 +278,13 @@
         # if the test suite crashes.  But if we don't continue, we'll loose the
         # ccache and other jobs will be cancelled, too.
         # It would be nice if we could mark the job as "yellow" (and continue)
-        # in that case.
-        # It would also be nice if we could analyze the test suite results
-        # automatically.
+        # in that case.  The second best thing is to display a warning in the
+        # job summary (see below).
         continue-on-error: true
         timeout-minutes: 60
         run: |
           echo 'makeinfo_program ("/usr/local/opt/texinfo/bin/makeinfo");' >> ./scripts/startup/site-rcfile
-          make -C ./.build check
+          make -C ./.build check | tee ./test-suite.log
 
       - name: display test log
         continue-on-error: true
@@ -293,6 +292,13 @@
         timeout-minutes: 5
         run: cat ./.build/test/fntests.log
 
+      - name: analyze test suite results
+        # Make sure the test summary lists 0 "FAIL"s and no "REGRESSION"
+        run:  |
+          [ -n "$(grep -e "FAIL\s*0" ./test-suite.log)" ] || echo "::warning::At least one test failed"
+          [ -z "$(grep -e "REGRESSION" ./test-suite.log)" ] || echo "::warning::At least one regression in test suite"
+          echo Finished analyzing test suite results.
+
 
   windows:
     runs-on: ${{ matrix.os }}
@@ -484,9 +490,8 @@
         # if the test suite crashes.  But if we don't continue, we'll loose the
         # ccache and other jobs will be cancelled, too.
         # It would be nice if we could mark the job as "yellow" (and continue)
-        # in that case.
-        # It would also be nice if we could analyze the test suite results
-        # automatically.
+        # in that case.  The second best thing is to display a warning in the
+        # job summary (see below).
         continue-on-error: true
         timeout-minutes: 60
 
@@ -496,10 +501,17 @@
         run: |
           echo 'makeinfo_program (sprintf ("%s && cd %s/../usr/bin && perl makeinfo", OCTAVE_HOME ()(1:2), OCTAVE_HOME ()));' \
             >> "${MINGW_PREFIX}/share/octave/site/m/startup/octaverc"
-          octave --eval "__run_test_suite__"
+          octave --eval "diary ('test-suite.log'); __run_test_suite__ ();"
 
       - name: display test log
         continue-on-error: true
         # Displaying the log shouldn't take long. Cancel the step if it does.
         timeout-minutes: 5
         run: cat fntests.log
+
+      - name: analyze test suite results
+        # Make sure the test summary lists 0 "FAIL"s and no "REGRESSION"
+        run:  |
+          [ -n "$(grep -e "FAIL\s*0" ./test-suite.log)" ] || echo "::warning::At least one test failed"
+          [ -z "$(grep -e "REGRESSION" ./test-suite.log)" ] || echo "::warning::At least one regression in test suite"
+          echo Finished analyzing test suite results.