From e8b1c654d4b329b303091956f89f5c6c7cc5b7b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rafa=C5=82=20Mikrut?= <mikrutrafal@protonmail.com>
Date: Fri, 21 Apr 2023 22:51:15 +0200
Subject: [PATCH] Regression finder

---
 .github/workflows/regression.yml      | 67 +++++++++++++++++++++++++++
 .github/workflows/regression_check.sh | 28 +++++++++++
 .gitignore                            |  1 +
 test/regression/settings.toml         | 27 +++++++++++
 4 files changed, 123 insertions(+)
 create mode 100644 .github/workflows/regression.yml
 create mode 100644 .github/workflows/regression_check.sh
 create mode 100644 test/regression/settings.toml

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
new file mode 100644
index 00000000..19e4e2cb
--- /dev/null
+++ b/.github/workflows/regression.yml
@@ -0,0 +1,67 @@
+name: Regression Testing
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install things
+        run: |
+          sudo apt install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib
+          sudo pip3 install meson
+
+      - name: Compile Thorvg PR
+        run: |
+          meson . build -Dtools=svg2png -Db_sanitize=address,undefined
+          sudo ninja -C build install
+
+      - name: Compile Thorvg Develop
+        run: |
+          git clone https://github.com/thorvg/thorvg.git thorvg_develop
+          cd thorvg_develop
+          meson . build -Dtools=svg2png -Db_sanitize=address,undefined
+          sudo ninja -C build install
+          cd ..
+
+      - name: Download SVG Regression finder
+        run: |
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/svg_tester
+          chmod +x ./svg_tester
+
+      - name: Prepare files to test
+        run: |
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/SVG1000.zip
+          unzip SVG1000.zip
+          mv test/regression/settings.toml settings.toml
+          mkdir BrokenSVG
+
+      - name: Run tests
+        run: |
+          ./svg_tester 2>&1 | tee regression_logs.txt
+
+      - name: Store Broken Images
+        uses: actions/upload-artifact@v3
+        with:
+          name: broken-images
+          path: BrokenSVG
+          if-no-files-found: ignore
+
+      - name: Store Problematic Images
+        uses: actions/upload-artifact@v3
+        with:
+          name: problematic-images
+          path: ProblematicSVG
+          if-no-files-found: ignore
+
+      - name: Add comment to PR
+        run: |
+          export PATH=$PATH:~/.local/bin/
+          chmod +x "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
+          "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/regression_check.sh b/.github/workflows/regression_check.sh
new file mode 100644
index 00000000..748a897e
--- /dev/null
+++ b/.github/workflows/regression_check.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [[ -z "$GITHUB_TOKEN" ]]; then
+    echo "The GITHUB_TOKEN is required."
+    exit 1
+fi
+
+if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+    pwd
+
+    PAYLOAD_REGRESSION=`cat regression_logs.txt | grep "Regression results:"`
+    COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url)
+
+    echo $COMMENTS_URL
+    echo $PAYLOAD_REGRESSION
+
+    if [[ $PAYLOAD_REGRESSION == *"Regression results:"* ]]; then
+        echo "Found broken files"
+        OUTPUT+=$'\nFound difference in generated example png files, look into CI artifacts for list of files that was changed.'
+        OUTPUT+="$PAYLOAD_REGRESSION"
+        OUTPUT+=$'\n'
+    fi
+
+    PAYLOAD=$(echo '{}' | jq --arg body "$OUTPUT" '.body = $body')
+
+    curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "$PAYLOAD" "$COMMENTS_URL"
+fi
+
diff --git a/.gitignore b/.gitignore
index ad5a7608..256316f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ builddir
 *.swp
 doc
 capi_test.tvg
+.idea
\ No newline at end of file
diff --git a/test/regression/settings.toml b/test/regression/settings.toml
new file mode 100644
index 00000000..90846ab3
--- /dev/null
+++ b/test/regression/settings.toml
@@ -0,0 +1,27 @@
+[general]
+folder_with_files_to_check = "SVG1000"
+problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
+output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
+timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds
+px_size_of_generated_file = 400
+ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
+ignore_similarity_checking_step = false # Useful to finding problems with generating files
+ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
+similarity = 10 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
+limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
+remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
+debug_show_always_output = false # Allows to find broken files
+test_version = false # Tests `app --version`, this allows to earlier find but with problematic files instead in runtime, when checking files
+return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1
+
+[first_tool]
+name = "thorvg_pr"
+path = "./build/src/bin/svg2png/svg2png"
+png_name_ending = "_thorvg_pr.png"
+arguments = "{FILE} -r {SIZE}x{SIZE}"
+
+[other_tool]
+name = "thorvg_develop"
+path = "./thorvg_develop/build/src/bin/svg2png/svg2png"
+png_name_ending = "_thorvg_pr.png"
+arguments = "{FILE} -r {SIZE}x{SIZE}"