Finding crashes and adding comments

.github/workflows/regression.yml

@@ -14,12 +14,12 @@ jobs:
       - name: Install things
         run: |
           sudo apt-get update
-          sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib
+          sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib earlyoom
           sudo pip3 install meson
 
       - name: Compile Thorvg PR
         run: |
-          meson . build -Dtools=svg2png
+          meson . build -Dtools=svg2png,lottie2gif -Dsavers=gif -Db_sanitize=address,undefined
           sudo ninja -C build install
 
       - name: Compile Thorvg Develop
@@ -32,14 +32,52 @@ jobs:
 
       - name: Download SVG Regression finder and setup settings
         run: |
-          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/svg_tester
+          wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.4.0/svg_tester
           chmod +x ./svg_tester
-          mv test/regression/settings.toml settings.toml
+
+      - name: Prepare valid files to test
+        run: |
+          wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
+          unzip -q files.zip
+          rm files.zip
+          wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
+          unzip -q files.zip
+          rm files.zip
+          mv ThorvgValidFiles FilesToTest
+          find ThorvgNotValidFiles -type f -exec mv {} FilesToTest \;
+          rmdir ThorvgNotValidFiles
+
+      - name: Setup settings for crash/leak/timeout tests
+        run: |
+          mv test/regression/settings_crash_leak.toml settings.toml
+
+      - name: Run regression finder tests
+        run: |
+          ./svg_tester 2>&1 | tee result_crashes.txt
+
+      - name: Setup settings for comparison tests
+        run: |
+          mv test/regression/settings_comparison.toml settings.toml
+
+      - name: Store Crashing/Leaking/Timeouting Images
+        uses: actions/upload-artifact@v4
+        with:
+          name: crashing-leaking-timeouting-images
+          path: BrokenFILES
+          if-no-files-found: ignore
+
+      - name: Clean Data
+        run: |
+          rm -rf BrokenSVG || true
+          rm -rf FilesToTest || true
+          rm -rf ProblematicSVG || true
+          rm -rf IgnoredSVG || true
+          rm -rf BrokenFILES || true
 
       # Test valid files
       - name: Prepare valid files to test
         run: |
-          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
+          wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
           unzip -q files.zip
           rm files.zip
           mv ThorvgValidFiles FilesToTest
@@ -68,11 +106,13 @@ jobs:
           rm -rf FilesToTest || true
           rm -rf ProblematicSVG || true
           rm -rf IgnoredSVG || true
+          rm -rf BrokenFILES || true
 
       # Test files that may not work currently good with Thorvg, errors from this are not critical
+      # it is possible that this will find also some improvements
       - name: Prepare not valid to test
         run: |
-          wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
+          wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
           unzip -q files.zip
           rm files.zip
           mv ThorvgNotValidFiles FilesToTest
@@ -101,19 +141,42 @@ jobs:
           rm -rf FilesToTest || true
           rm -rf ProblematicSVG || true
           rm -rf IgnoredSVG || true
+          rm -rf BrokenFILES || true
 
       - name: Test png reproducibility
         run: |
-          wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip
+          wget -q https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip
           unzip -q files.zip
           rm files.zip
           cp test/regression/check_same_image_size.py check_same_image_size.py
           # Forces to run tasks on different threads if possible, which should help find problem with data races
           taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/tools/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt
 
-      - name: Check output and send comment
+      - name: Check results
         run: |
           export PATH=$PATH:~/.local/bin/
           python3 "${GITHUB_WORKSPACE}/.github/workflows/regression_check.py"
-        env:
+
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Regression report
+
+# TODO - not works
+#      - name: Create or update comment
+#        uses: peter-evans/create-or-update-comment@v4
+#        with:
+#          comment-id: ${{ steps.fc.outputs.comment-id }}
+#          issue-number: ${{ github.event.pull_request.number }}
+#          body-path: 'comment.txt'
+#          edit-mode: replace
+
+      - name: Fail CI if regression found
+        run: |
+          if [[ -f "fail_ci.txt" ]]; then
+            echo "Check Check results section for more details which tests failed"
+            exit 1
+          fi

.github/workflows/regression_check.py

@@ -1,140 +1,67 @@
-import os
+from typing import Optional
-import json
-import subprocess
 
-GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
-GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME")
-GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH") or "Missing"
-
-RUST_PANIC = "panicked"
 POSSIBLE_PROBLEM = "POSSIBLE_PROBLEM - "
 
 
-def check_github_token() -> None:
+def check_file(file_name: str) -> Optional[str]:
-    if not GITHUB_TOKEN:
-        print("The GITHUB_TOKEN is required.")
-        exit(1)
-
-
-def check_file(file_name: str) -> tuple[list[str], list[str]]:
-    panics = []
-    possible_problems = []
     with open(file_name, "r") as file:
         print(f"Checking file {file_name}")
         for line in file:
             line = line.strip()
-            if RUST_PANIC in line:
+            if line.startswith(POSSIBLE_PROBLEM):
-                panics.append(line)
+                return line[len(POSSIBLE_PROBLEM) :]
-            elif POSSIBLE_PROBLEM in line:
+    return None
-                possible_problems.append(line)
-
-    print(f"Panics in {file_name}", panics)
-    print(f"Possible problems in {file_name}", possible_problems)
-    return panics, possible_problems
-
-
-def check_changes_in_valid_files() -> tuple[bool, str]:
-    panics, possible_problems = check_file("result_valid_files.txt")
-
-    if len(panics) != 0 or len(possible_problems) != 0:
-        print(">>>>> Found changed in previously valid files (see above)")
-
-        github_comment = "\nFound regression in converting images that properly converted in develop branch.\n"
-        for line in panics:
-            github_comment += (
-                line.replace(RUST_PANIC, "") + " - panicked when checking\n"
-            )
-        for line in possible_problems:
-            github_comment += (
-                line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
-            )
-        github_comment += "\n"
-
-        return True, github_comment
-    return False, ""
-
-
-def check_changes_in_invalid_files() -> tuple[bool, str]:
-    panics, possible_problems = check_file("result_not_valid_files.txt")
-
-    if len(panics) != 0 or len(possible_problems) != 0:
-        print(">>>>> Found changed non valid files (see above)")
-        for line in panics + possible_problems:
-            print(line)
-
-        github_comment = "\nFound differences in converting images that were not properly converted in develop branch.\n"
-        for line in panics:
-            github_comment += (
-                line.replace(RUST_PANIC, "") + " - panicked when checking\n"
-            )
-        for line in possible_problems:
-            github_comment += (
-                line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
-            )
-        github_comment += "\n"
-
-        return False, github_comment
-    return False, ""
-
-
-def check_changes_in_image_size() -> tuple[bool, str]:
-    panics, possible_problems = check_file("result_image_size.txt")
-
-    if len(panics) != 0 or len(possible_problems) != 0:
-        print(">>>>> Found difference in size generated image(see above)")
-        for line in panics + possible_problems:
-            print(line)
-
-        github_comment = "\nGenerated png have different size in each run.\n"
-        for line in panics:
-            github_comment += (
-                line.replace(RUST_PANIC, "") + " - panicked when checking\n"
-            )
-        for line in possible_problems:
-            github_comment += (
-                line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
-            )
-        github_comment += "\n"
-
-        return True, github_comment
-    return False, ""
-
-
-def send_comment(github_comment: str) -> None:
-    with open(GITHUB_EVENT_PATH, "r") as file:
-        data = json.load(file)
-        comments_url = data["pull_request"]["comments_url"]
-
-    print(f"Sending comment to {comments_url}")
-
-    payload = json.dumps({"body": github_comment})
-    curl_command = f'curl -s -S -H "Authorization: token {GITHUB_TOKEN}" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "{payload}" "{comments_url}"'
-    # Print output from curl
-    subprocess.run(curl_command, shell=True)
 
 
 if __name__ == "__main__":
-    if GITHUB_EVENT_NAME != "pull_request":
+    data_to_check = [
-        print(
+        (
-            "You are running this script in wrong event - expected pull_request, crashes may occur"
+            "result_valid_files.txt",
-        )
+            True,
+            "Found difference in converting images that properly converted in develop branch.",
+        ),
+        (
+            "result_not_valid_files.txt",
+            False,
+            "Found differences in converting images that were not properly converted in develop branch(this may be improvement).",
+        ),
+        (
+            "result_image_size.txt",
+            True,
+            "Generated png have different size in each run.",
+        ),
+        (
+            "result_crashes.txt",
+            True,
+            "Found crashes in crashes during when converting svg to png.",
+        ),
+    ]
 
-    print(os.getcwd())
+    fail_ci = False
+    comment = ""
+    for file_name, fail_ci_when_triggered, new_comment in data_to_check:
+        possible_problem = check_file(file_name)
+        if possible_problem is not None:
+            print(f">>>>> Found changes in {file_name} (look at CI steps from above)")
+            print(f"Comment: {new_comment}")
 
-    print()
+            if fail_ci_when_triggered:
-    fail_ci_v, comment_v = check_changes_in_valid_files()
+                fail_ci = True
-    print()
+            comment += new_comment + "\n"
-    fail_ci_i, comment_i = check_changes_in_invalid_files()
-    print()
-    fail_ci_s, comment_s = check_changes_in_image_size()
-    print()
 
-    fail_ci = fail_ci_v or fail_ci_i or fail_ci_s
+    to_write = "Regression report:\n"
-    comment = (comment_v + comment_i + comment_s).strip()
 
-    if GITHUB_EVENT_NAME == "pull_request":
+    if len(comment) > 0:
-        if len(comment) > 0:
+        to_write += comment
-            send_comment(comment)
+        to_write += "\nCheck CI for artifacts/logs for more details."
+    else:
+        to_write += "Not found any changes(both improvements and regressions) in this PR with test data."
+
+    print("\n\n" + to_write + "\n\n")
+
+    with open("comment.txt", "w") as file:
+        file.write(to_write)
 
     if fail_ci:
-        raise Exception("Found regression in image conversion")
+        with open("fail_ci.txt", "w") as file:
+            file.write("Fail")

test/regression/check_same_image_size.py

@@ -9,8 +9,10 @@ if (
     or not os.path.isfile(sys.argv[2])
 ):
     print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"')
-    print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"')
+    print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE NUMBER_OF_TRIES"')
-    raise ValueError("Missing or invalid input file or missing path to svg2png (panicked)")
+    raise ValueError(
+        "POSSIBLE_PROBLEM - Missing or invalid input file or missing path to svg2png"
+    )
 
 try_number = int(sys.argv[4])
 image_size = sys.argv[3]
@@ -33,8 +35,9 @@ for i in range(try_number + 1):
 sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True))
 
 if len(sizes) == 1:
-    print(f"Image size {str(sizes)}")
+    print(f"Not found problem with generating images, sizes - {str(sizes)}")
 else:
     print(
-        f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}"
+        f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}",
+        file=sys.stderr,
     )

test/regression/settings_comparison.toml

@@ -3,13 +3,13 @@ folder_with_files_to_check = "FilesToTest"
 problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
 output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
 ignored_files_path = "IgnoredSVG" # Place where to save ignored files
-timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds
+timeout = 120
 limit_threads = 0 # 0 will use all available threads
 px_size_of_generated_file = 400
 ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
 ignore_similarity_checking_step = false # Useful to finding problems with generating files
 ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
-similarity = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
+max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
 limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
 remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
 debug_show_always_output = false # Allows to find broken files
@@ -18,15 +18,21 @@ remove_problematic_files_after_copying = false # Remove from output folder probl
 remove_broken_files_after_copying = false # Remove from output folder broken svg files
 remove_ignored_files_after_copying = false # Removes not supported folders after copyting
 remove_generated_png_files_at_end = false # Remove all png from output folder at end
+lottie_path = ""
+lottie_broken_files_path = ""
+lottie_test = false
+thorvg_path = ""
+thorvg_broken_files_path = ""
+thorvg_test = false
 
 [first_tool]
 name = "thorvg_pr"
 path = "./build/src/tools/svg2png/svg2png"
 png_name_ending = "_thorvg_pr.png"
-arguments = "{FILE} -r {SIZE}x{SIZE}"
+arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"
 
 [other_tool]
 name = "thorvg_develop"
 path = "./thorvg_develop/build/src/tools/svg2png/svg2png"
 png_name_ending = "_thorvg_develop.png"
-arguments = "{FILE} -r {SIZE}x{SIZE}"
+arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"

test/regression/settings_crash_leak.toml

@@ -0,0 +1,39 @@
+[general]
+folder_with_files_to_check = "FilesToTest"
+problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
+output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
+ignored_files_path = "IgnoredSVG" # Place where to save ignored files
+timeout = 120
+limit_threads = 0 # 0 will use all available threads
+px_size_of_generated_file = 400
+ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
+ignore_similarity_checking_step = false # Useful to finding problems with generating files
+ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
+max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
+limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
+remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
+debug_show_always_output = false # Allows to find broken files
+return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1
+remove_problematic_files_after_copying = false # Remove from output folder problematic svg files
+remove_broken_files_after_copying = false # Remove from output folder broken svg files
+remove_ignored_files_after_copying = false # Removes not supported folders after copyting
+remove_generated_png_files_at_end = false # Remove all png from output folder at end
+lottie_path = ""
+lottie_broken_files_path = ""
+lottie_test = false
+thorvg_path = "./build/src/tools/svg2png/svg2png"
+thorvg_broken_files_path = "BrokenFILES"
+thorvg_test = true
+
+[first_tool]
+name = "thorvg_pr"
+path = "./build/src/tools/svg2png/svg2png"
+png_name_ending = "_thorvg_pr.png"
+arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"
+
+[other_tool]
+name = "thorvg_develop"
+path = "./thorvg_develop/build/src/tools/svg2png/svg2png"
+png_name_ending = "_thorvg_develop.png"
+arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"
+