infra: Improve regression finder

.github/workflows/regression.yml

@@ -28,36 +28,83 @@ jobs:
           sudo ninja -C build install
           cd ..
 
-      - name: Download SVG Regression finder
+      - name: Download SVG Regression finder and setup settings
         run: |
-          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/svg_tester
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/svg_tester
           chmod +x ./svg_tester
-
-      - name: Prepare files to test
-        run: |
-          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/SVG1000.zip
-          unzip SVG1000.zip
           mv test/regression/settings.toml settings.toml
-          mkdir BrokenSVG
 
-      - name: Run tests
+      # Test valid files
+      - name: Prepare valid files to test
         run: |
-          ./svg_tester 2>&1 | tee regression_logs.txt
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip
+          unzip ThorvgValidFiles.zip
+          mv ThorvgValidFiles FilesToTest
 
-      - name: Store Broken Images
+      - name: Run regression finder tests
+        run: |
+          ./svg_tester 2>&1 | tee result_valid_files.txt
+
+      - name: Store Broken Images for valid inputs
         uses: actions/upload-artifact@v3
         with:
-          name: broken-images
+          name: differences-in-valid-files
           path: BrokenSVG
           if-no-files-found: ignore
 
-      - name: Store Problematic Images
+      - name: Store Problematic Images for valid inputs
         uses: actions/upload-artifact@v3
         with:
-          name: problematic-images
+          name: problematic-images-in-valid-files
           path: ProblematicSVG
           if-no-files-found: ignore
 
+      - name: Clean Data
+        run: |
+          rm -rf BrokenSVG || true
+          rm -rf FilesToTest || true
+          rm -rf ProblematicSVG || true
+          rm -rf IgnoredSVG || true
+
+      # Test files that may not work currently good with Thorvg, errors from this are not critical
+      - name: Prepare not valid to test
+        run: |
+          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip
+          unzip ThorvgNotValidFiles.zip
+          mv ThorvgNotValidFiles FilesToTest
+
+      - name: Run invalid files tests
+        run: |
+          ./svg_tester 2>&1 | tee result_not_valid_files.txt
+
+      - name: Store Broken Images for not valid inputs
+        uses: actions/upload-artifact@v3
+        with:
+          name: differences-in-not-valid-files
+          path: BrokenSVG
+          if-no-files-found: ignore
+
+      - name: Store Problematic Images for not valid inputs
+        uses: actions/upload-artifact@v3
+        with:
+          name: problematic-images-in-not-valid-files
+          path: ProblematicSVG
+          if-no-files-found: ignore
+
+      - name: Clean Data
+        run: |
+          rm -rf BrokenSVG || true
+          rm -rf FilesToTest || true
+          rm -rf ProblematicSVG || true
+          rm -rf IgnoredSVG || true
+
+      - name: Test png reproducibility
+        run: |
+          wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip
+          unzip AA_5.svg.zip
+          cp test/regression/check_same_image_size.py check_same_image_size.py
+          taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/bin/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt
+
       - name: Add comment to PR
         run: |
           export PATH=$PATH:~/.local/bin/
@@ -65,3 +112,8 @@ jobs:
           "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Failing should happen after creating comment PR and passing all tests
+      - name: Fail if needed
+        run: |
+          if [ -f "EXIT_REQUESTED" ]; then false; fi

.github/workflows/regression_check.sh

@@ -8,21 +8,57 @@ fi
 if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
   pwd
 
-    PAYLOAD_REGRESSION=`cat regression_logs.txt | grep "Regression results:"`
+  CREATE_COMMENT=0
+  FAIL_CI=0
+
   COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url)
+  echo "$COMMENTS_URL"
+  POSSIBLE_PROBLEM_SUBSTRING="POSSIBLE_PROBLEM - "
 
-    echo $COMMENTS_URL
-    echo $PAYLOAD_REGRESSION
+  VALID_FILES=$(cat result_valid_files.txt | grep "POSSIBLE_PROBLEM")
+  echo "$VALID_FILES"
 
-    if [[ $PAYLOAD_REGRESSION == *"Regression results:"* ]]; then
-        echo "Found broken files"
-        OUTPUT+=$'\nFound difference in generated example png files, look into CI artifacts for list of files that was changed.'
-        OUTPUT+="$PAYLOAD_REGRESSION"
+  if [[ $VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then
+    echo "Found changed valid files"
+    OUTPUT+=$'\nFound regression in converting images that properly converted in develop branch.\n'
+    OUTPUT+="${VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}"
     OUTPUT+=$'\n'
+    CREATE_COMMENT=1
+    FAIL_CI=1
   fi
 
+  NOT_VALID_FILES=$(cat result_not_valid_files.txt | grep "POSSIBLE_PROBLEM")
+  echo "$NOT_VALID_FILES"
+
+  if [[ $NOT_VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then
+    echo "Found changed non valid files"
+    OUTPUT+=$'\nFound differences in converting images that were not properly converted in develop branch.\n'
+    OUTPUT+="${NOT_VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}"
+    OUTPUT+=$'\n'
+    CREATE_COMMENT=1
+  fi
+
+  IMAGE_SIZE=$(cat result_image_size.txt | grep "POSSIBLE_PROBLEM")
+  echo "$IMAGE_SIZE"
+
+  if [[ $IMAGE_SIZE == *"POSSIBLE_PROBLEM"* ]]; then
+    echo "Found difference in size generated image"
+    OUTPUT+=$'\nGenerated png have different size in each run.\n'
+    OUTPUT+="$IMAGE_SIZE"
+    OUTPUT+="${IMAGE_SIZE#$POSSIBLE_PROBLEM_SUBSTRING}"
+    OUTPUT+=$'\n'
+    CREATE_COMMENT=1
+    FAIL_CI=1
+  fi
+
+
+  if [ "$FAIL_CI" -eq 1 ]; then
+    touch "EXIT_REQUESTED"
+  fi
+
+  if [ "$CREATE_COMMENT" -eq 1 ]; then
     PAYLOAD=$(echo '{}' | jq --arg body "$OUTPUT" '.body = $body')
-
     curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "$PAYLOAD" "$COMMENTS_URL"
-fi
+  fi
 
+fi

test/regression/check_same_image_size.py

@@ -0,0 +1,34 @@
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 5 or not sys.argv[1].endswith(".svg") or not os.path.isfile(sys.argv[1]) or not os.path.isfile(
+        sys.argv[2]):
+    print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"')
+    print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"')
+    raise ValueError(f'Missing or invalid input file or missing path to svg2png')
+
+try_number = int(sys.argv[4])
+image_size = sys.argv[3]
+svg2png_path = sys.argv[2]
+image_input = sys.argv[1]
+image_output = image_input.replace(".svg", ".png")
+
+args = [svg2png_path, image_input, "-r", f"{image_size}x{image_size}"]
+sizes = {}
+for i in range(try_number + 1):
+    # if i % 100 == 0:
+    #    print(f"{i + 1}/{try_number + 1}")
+    subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+    new_size = os.path.getsize(image_output)
+    if new_size in sizes:
+        sizes[new_size] += 1
+    else:
+        sizes[new_size] = 1
+
+sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True))
+
+if len(sizes) == 1:
+    print(f'Image size {str(sizes)}')
+else:
+    print(f'POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}')

test/regression/settings.toml

@@ -1,18 +1,23 @@
 [general]
-folder_with_files_to_check = "SVG1000"
+folder_with_files_to_check = "FilesToTest"
 problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
 output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
+ignored_files_path = "IgnoredSVG" # Place where to save ignored files
 timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds
+limit_threads = 0 # 0 will use all available threads
 px_size_of_generated_file = 400
 ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
 ignore_similarity_checking_step = false # Useful to finding problems with generating files
 ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
-similarity = 10 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
+similarity = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
 limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
 remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
 debug_show_always_output = false # Allows to find broken files
-test_version = false # Tests `app --version`, this allows to earlier find but with problematic files instead in runtime, when checking files
 return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1
+remove_problematic_files_after_copying = false # Remove from output folder problematic svg files
+remove_broken_files_after_copying = false # Remove from output folder broken svg files
+remove_ignored_files_after_copying = false # Removes not supported folders after copyting
+remove_generated_png_files_at_end = false # Remove all png from output folder at end
 
 [first_tool]
 name = "thorvg_pr"