Regression finder

.github/workflows/regression.yml

@@ -0,0 +1,67 @@
+name: Regression Testing
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install things
+        run: |
+          sudo apt install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib
+          sudo pip3 install meson
+
+      - name: Compile Thorvg PR
+        run: |
+          meson . build -Dtools=svg2png -Db_sanitize=address,undefined
+          sudo ninja -C build install
+
+      - name: Compile Thorvg Develop
+        run: |
+          git clone https://github.com/thorvg/thorvg.git thorvg_develop
+          cd thorvg_develop
+          meson . build -Dtools=svg2png -Db_sanitize=address,undefined
+          sudo ninja -C build install
+          cd ..
+
+      - name: Download SVG Regression finder
+        run: |
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/svg_tester
+          chmod +x ./svg_tester
+
+      - name: Prepare files to test
+        run: |
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/SVG1000.zip
+          unzip SVG1000.zip
+          mv test/regression/settings.toml settings.toml
+          mkdir BrokenSVG
+
+      - name: Run tests
+        run: |
+          ./svg_tester 2>&1 | tee regression_logs.txt
+
+      - name: Store Broken Images
+        uses: actions/upload-artifact@v3
+        with:
+          name: broken-images
+          path: BrokenSVG
+          if-no-files-found: ignore
+
+      - name: Store Problematic Images
+        uses: actions/upload-artifact@v3
+        with:
+          name: problematic-images
+          path: ProblematicSVG
+          if-no-files-found: ignore
+
+      - name: Add comment to PR
+        run: |
+          export PATH=$PATH:~/.local/bin/
+          chmod +x "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
+          "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/regression_check.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [[ -z "$GITHUB_TOKEN" ]]; then
+    echo "The GITHUB_TOKEN is required."
+    exit 1
+fi
+
+if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+    pwd
+
+    PAYLOAD_REGRESSION=`cat regression_logs.txt | grep "Regression results:"`
+    COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url)
+
+    echo $COMMENTS_URL
+    echo $PAYLOAD_REGRESSION
+
+    if [[ $PAYLOAD_REGRESSION == *"Regression results:"* ]]; then
+        echo "Found broken files"
+        OUTPUT+=$'\nFound difference in generated example png files, look into CI artifacts for list of files that was changed.'
+        OUTPUT+="$PAYLOAD_REGRESSION"
+        OUTPUT+=$'\n'
+    fi
+
+    PAYLOAD=$(echo '{}' | jq --arg body "$OUTPUT" '.body = $body')
+
+    curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "$PAYLOAD" "$COMMENTS_URL"
+fi
+

.gitignore

@@ -4,3 +4,4 @@ builddir
 *.swp
 doc
 capi_test.tvg
+.idea

test/regression/settings.toml

@@ -0,0 +1,27 @@
+[general]
+folder_with_files_to_check = "SVG1000"
+problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
+output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
+timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds
+px_size_of_generated_file = 400
+ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
+ignore_similarity_checking_step = false # Useful to finding problems with generating files
+ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
+similarity = 10 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
+limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
+remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
+debug_show_always_output = false # Allows to find broken files
+test_version = false # Tests `app --version`, this allows to earlier find but with problematic files instead in runtime, when checking files
+return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1
+
+[first_tool]
+name = "thorvg_pr"
+path = "./build/src/bin/svg2png/svg2png"
+png_name_ending = "_thorvg_pr.png"
+arguments = "{FILE} -r {SIZE}x{SIZE}"
+
+[other_tool]
+name = "thorvg_develop"
+path = "./thorvg_develop/build/src/bin/svg2png/svg2png"
+png_name_ending = "_thorvg_pr.png"
+arguments = "{FILE} -r {SIZE}x{SIZE}"