Restore integtests and add to CI as warning comment on failure (#7404)

* Add coverage for both targets

* Use var for test bins

* Disable cmake-format comment fiddling

* Refactor GUI source config

* Use bin var for tests

* Remove unused member

* Add integration tests to CI

* Use modern cmake args

* Use max threads for build

* Use better var name for CPU core count

* Split build and configure steps

* Combine tests into action and add a PR comment

* Fixed yaml indentation in action

* Pass GITHUB_TOKEN

* Update coverage paths for SonarCloud

* Don't ignore return codes

* Add shell

* Run Valgrind on integ tests

* Use header for tests

* Save test results in table

* Move setup step outside of action

* Change logic of creating PR comment

* Remove header formatting

* Use emojis for simplicity

* Run build wrapper in build dir

* Use default make target

* Pass secrets

* Fixed SonarScanner warnings

* Don't allow unit tests to fail

* Fixed typo

* Update sonar scanner paths

* Fixed line endings

* Use step output

* Improve exclusion glob

* Exclude files from coverage

* Restore simpler pattern

* Set temp file path

* Coverage tests

* Re-create comment at start of job

* Append table header

* Add setup action

* Checkout before action

* Re-add projectBaseDir

* Restore original sonar scanner

* Use bash syntax for if

* Remove unused `shell`

* Add missing shell for valgrind action

* Restore new sonar scanner config

* Add missing shell

* Run only MainWindowTests

* Test with big change

* More changes

* Move to correct dir

* Remove test code

* Disable broken integ tests

* Switch coverage to front of filename

* Remove filter

* Refactor status step

* Disable segfault test

* Fixed: No status showing

* Add link to workflows

* Add test code for coverage

* Revert "Add test code for coverage"

This reverts commit c42309349b64f7828f2ca89149b30c5b0f93478a.

* Get workflow run URL

* Add missing shell

* Use dynamic URL in valgrind comment

* Revert "Revert "Add test code for coverage""

This reverts commit 9cff58b7ea5c581681ae6d6660c073bd76ba99aa.

* Test with commented out code

* Reintroduce 6 lines for coverage

* Test code to pass scanner

* Test code to pass scanner (take 2)

* Simplify to 2 new lines

* Add another line

* Trim changes to only 3 lines

* Add task for all tests

* Surface warning on failure

* Simplify build-wrapper step and move settings from web UI to CI

* Add missing line delims

* Also run tests action on Windows and macOS

* Add names to action steps

* Add timeout for test steps

* Add failure warning for integ tests

* Remove space

* Disable failing test on macOS

* Disable problem matcher

* Simplify names

* Disable freezing test on Windows

* Disable failing integ test on Windows

* Add build-kill task

* Ignore kill result on Windows

* Delete test code

* Update ChangeLog

* Move timeout to workflow step

.github/workflows/ci.yml

@@ -33,6 +33,18 @@ env:
   UPLOAD_TO_GDRIVE: ${{ github.event_name != 'pull_request' }}
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: ./.github/actions/run-tests-setup
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   windows:
     name: ${{ matrix.target.name }}
     runs-on: ${{ matrix.target.runs-on }}
@@ -75,8 +87,12 @@ jobs:
       - name: Build
         run: cmake --build build -j8
 
-      - name: Test
-        run: ./build/bin/unittests
+      - name: Tests
+        uses: ./.github/actions/run-tests
+        timeout-minutes: 2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          job: ${{ matrix.target.name }}
 
       - name: Package
         if: ${{ vars.SYNERGY_ENABLE_PACKAGING && env.PACKAGE_BUILD == 'true' }}
@@ -142,8 +158,12 @@ jobs:
       - name: Build
         run: cmake --build build -j8
 
-      - name: Test
-        run: ./build/bin/unittests
+      - name: Tests
+        uses: ./.github/actions/run-tests
+        timeout-minutes: 2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          job: ${{ matrix.target.name }}
 
       - name: Package
         if: ${{ vars.SYNERGY_ENABLE_PACKAGING && env.PACKAGE_BUILD == 'true' }}
@@ -243,10 +263,12 @@ jobs:
       - name: Build
         run: cmake --build build -j8
 
-      - name: Test
-        env:
-          QT_QPA_PLATFORM: offscreen
-        run: ./build/bin/unittests
+      - name: Tests
+        uses: ./.github/actions/run-tests
+        timeout-minutes: 2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          job: linux-${{ matrix.distro.name }}
 
       - name: Package
         if: ${{ vars.SYNERGY_ENABLE_PACKAGING && env.PACKAGE_BUILD == 'true' }}

.github/workflows/sonarcloud-analysis.yml

@@ -27,7 +27,7 @@ jobs:
       SONAR_SCANNER_VERSION: 6.1.0.4477
       SONAR_SCANNER_OPTS: -server
       SONAR_SCANNER_URL_BASE: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli
-      SONAR_SCANNER_THREADS: 32
+      CPU_CORE_COUNT: 32
 
     steps:
       - name: Checkout
@@ -35,6 +35,9 @@ jobs:
         with:
           submodules: "recursive"
 
+          # Fetch all history for SonarScanner blame data
+          fetch-depth: 0
+
       - name: Config Git safe dir
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
@@ -53,38 +56,64 @@ jobs:
 
       - name: Install build-wrapper
         run: |
-          curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip
-          unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/
+          file="build-wrapper-linux-x86.zip"
+          url="https://sonarcloud.io/static/cpp/$file"
+          curl --create-dirs -sSLo $HOME/.sonar/$file $url
+          unzip -o $HOME/.sonar/$file -d $HOME/.sonar/
+
+      - name: Configure
+        run: cmake -B build --preset=linux-debug -DENABLE_COVERAGE=ON
 
       - name: Build
         run: |
           export PATH=$HOME/.sonar/build-wrapper-linux-x86:$PATH
-          mkdir -p build
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON ..
-          build-wrapper-linux-x86-64 --out-dir bw-output make -j
+          build-wrapper-linux-x86-64 --out-dir bw-output cmake --build build -j${CPU_CORE_COUNT}
 
-      - name: Make coverage
+      - name: Unit tests coverage
+        env:
+          QT_QPA_PLATFORM: offscreen
+        run: cmake --build build --target coverage-unittests
+
+      - name: Integration tests coverage
         env:
           QT_QPA_PLATFORM: offscreen
         run: |
-          cd build
-          make coverage
+          cmake --build build --target coverage-integtests
+          result=$?
+
+          if [ $result -ne 0 ]; then
+            echo "::warning ::Integration tests failed with code: $result"
+          fi
+        continue-on-error: true
+
+      - name: Get coverage report paths
+        id: coverage-paths
+        run: |
+          unittests=$(find build -name coverage-unittests.xml)
+          integtests=$(find build -name coverage-integtests.xml)
+          paths="${unittests}${integtests:+,$integtests}"
+          if [ -z "$paths" ]; then
+            echo "Error: No coverage files found"
+            exit 1
+          fi
+          echo "csv=$paths" >> $GITHUB_OUTPUT
 
       - name: Run SonarScanner
         run: |
           export PATH=$HOME/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux-x64/bin:$PATH
-          cd build
           sonar-scanner \
             -Dsonar.organization=symless \
             -Dsonar.projectKey=symless_synergy-core \
-            -Dsonar.sources=. \
-            -Dsonar.projectBaseDir=../ \
+            -Dsonar.sources=scripts,src/cmd,src/gui,src/lib \
+            -Dsonar.tests=src/test \
             -Dsonar.exclusions=ext/**,build/** \
-            -Dsonar.cfamily.build-wrapper-output=bw-output \
+            -Dsonar.coverage.exclusions=ext/**,scripts/**,src/test/** \
+            -Dsonar.cpd.exclusions=**/*Test*.cpp \
             -Dsonar.host.url=https://sonarcloud.io \
-            -Dsonar.coverageReportPaths=build/coverage.xml \
-            -Dsonar.cfamily.threads=${{ env.SONAR_SCANNER_THREADS }}
+            -Dsonar.coverageReportPaths=${{ steps.coverage-paths.outputs.csv }} \
+            -Dsonar.cfamily.compile-commands=build/compile_commands.json \
+            -Dsonar.cfamily.threads=${{ env.CPU_CORE_COUNT }} \
+            -Dsonar.python.version=3.10
         env:
           SONAR_TOKEN: ${{secrets.SONAR_TOKEN}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/valgrind-analysis.yml

@@ -39,37 +39,41 @@ jobs:
       - name: Build
         run: cmake --build build -j8
 
-      - name: Run Valgrind on unit tests
-        env:
-          QT_QPA_PLATFORM: offscreen
-        run: |
-          valgrind \
-          --leak-check=full \
-          --show-leak-kinds=all \
-          --track-origins=yes \
-          --verbose \
-          ./build/bin/unittests \
-          2>&1 | tee valgrind.log
+      - name: Valgrind unit tests
+        id: unittests
+        uses: ./.github/actions/run-valgrind
+        with:
+          executable: ./build/bin/unittests
 
-      - name: Parse summary
-        id: parse
+      - name: Valgrind integration tests
+        id: integtests
+        uses: ./.github/actions/run-valgrind
+        with:
+          executable: ./build/bin/integtests
+
+      - name: Get workflow URL
+        id: workflow-url
         run: |
-          echo "summary<<EOF" >> $GITHUB_OUTPUT
-          echo "$(grep -A 2 "HEAP SUMMARY:" valgrind.log)" >> $GITHUB_OUTPUT
-          echo >> $GITHUB_OUTPUT
-          echo "$(awk '/LEAK SUMMARY/,/ERROR SUMMARY/' valgrind.log)" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+          repo_url="${{ github.server_url }}/${{ github.repository }}"
+          echo "url=$repo_url/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
+        shell: bash
 
       - name: Append to PR comment
         uses: marocchino/sticky-pull-request-comment@v2
-        env:
-          URL: https://github.com/symless/synergy-core/actions/workflows/valgrind-analysis.yml
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           recreate: true
+          header: valgrind
           message: |
             ## Valgrind summary
-            See [workflow output](${{ env.URL }}) for full `valgrind` output.
+            See [workflow run](${{ steps.workflow-url.outputs.url }}) for full `valgrind` output.
+
+            ### Unit tests
             ```
-            ${{ steps.parse.outputs.summary }}
+            ${{ steps.unittests.outputs.summary }}
+            ```
+
+            ### Integration tests
+            ```
+            ${{ steps.integtests.outputs.summary }}
             ```