diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index e5cfbe08..2fdeb41a 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -14,12 +14,12 @@ jobs: - name: Install things run: | sudo apt-get update - sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib + sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib earlyoom sudo pip3 install meson - name: Compile Thorvg PR run: | - meson . build -Dtools=svg2png + meson . build -Dtools=svg2png,lottie2gif -Dsavers=gif -Db_sanitize=address,undefined sudo ninja -C build install - name: Compile Thorvg Develop @@ -32,14 +32,52 @@ jobs: - name: Download SVG Regression finder and setup settings run: | - wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/svg_tester + wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.4.0/svg_tester chmod +x ./svg_tester - mv test/regression/settings.toml settings.toml + + - name: Prepare valid files to test + run: | + wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip + unzip -q files.zip + rm files.zip + wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip + unzip -q files.zip + rm files.zip + mv ThorvgValidFiles FilesToTest + find ThorvgNotValidFiles -type f -exec mv {} FilesToTest \; + rmdir ThorvgNotValidFiles + + - name: Setup settings for crash/leak/timeout tests + run: | + mv test/regression/settings_crash_leak.toml settings.toml + + - name: Run regression finder tests + run: | + ./svg_tester 2>&1 | tee result_crashes.txt + + - name: Setup settings for comparison tests + run: | + mv test/regression/settings_comparison.toml settings.toml + + - name: Store Crashing/Leaking/Timeouting Images + uses: actions/upload-artifact@v4 + with: + name: crashing-leaking-timeouting-images + path: BrokenFILES + if-no-files-found: ignore + + - name: Clean Data + run: | + rm -rf BrokenSVG || true + rm -rf FilesToTest || true + rm -rf ProblematicSVG || true + rm -rf IgnoredSVG || true + rm -rf BrokenFILES || true # Test valid files - name: Prepare valid files to test run: | - wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip + wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip unzip -q files.zip rm files.zip mv ThorvgValidFiles FilesToTest @@ -68,11 +106,13 @@ jobs: rm -rf FilesToTest || true rm -rf ProblematicSVG || true rm -rf IgnoredSVG || true + rm -rf BrokenFILES || true # Test files that may not work currently good with Thorvg, errors from this are not critical + # it is possible that this will find also some improvements - name: Prepare not valid to test run: | - wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip + wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip unzip -q files.zip rm files.zip mv ThorvgNotValidFiles FilesToTest @@ -101,19 +141,42 @@ jobs: rm -rf FilesToTest || true rm -rf ProblematicSVG || true rm -rf IgnoredSVG || true + rm -rf BrokenFILES || true - name: Test png reproducibility run: | - wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip + wget -q https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip unzip -q files.zip rm files.zip cp test/regression/check_same_image_size.py check_same_image_size.py # Forces to run tasks on different threads if possible, which should help find problem with data races taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/tools/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt - - name: Check output and send comment + - name: Check results run: | export PATH=$PATH:~/.local/bin/ python3 "${GITHUB_WORKSPACE}/.github/workflows/regression_check.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + + - name: Find Comment + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Regression report + +# TODO - not works +# - name: Create or update comment +# uses: peter-evans/create-or-update-comment@v4 +# with: +# comment-id: ${{ steps.fc.outputs.comment-id }} +# issue-number: ${{ github.event.pull_request.number }} +# body-path: 'comment.txt' +# edit-mode: replace + + - name: Fail CI if regression found + run: | + if [[ -f "fail_ci.txt" ]]; then + echo "Check Check results section for more details which tests failed" + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/regression_check.py b/.github/workflows/regression_check.py index 3f4aac12..1e12a065 100644 --- a/.github/workflows/regression_check.py +++ b/.github/workflows/regression_check.py @@ -1,140 +1,67 @@ -import os -import json -import subprocess +from typing import Optional -GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") -GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME") -GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH") or "Missing" - -RUST_PANIC = "panicked" POSSIBLE_PROBLEM = "POSSIBLE_PROBLEM - " -def check_github_token() -> None: - if not GITHUB_TOKEN: - print("The GITHUB_TOKEN is required.") - exit(1) - - -def check_file(file_name: str) -> tuple[list[str], list[str]]: - panics = [] - possible_problems = [] +def check_file(file_name: str) -> Optional[str]: with open(file_name, "r") as file: print(f"Checking file {file_name}") for line in file: line = line.strip() - if RUST_PANIC in line: - panics.append(line) - elif POSSIBLE_PROBLEM in line: - possible_problems.append(line) - - print(f"Panics in {file_name}", panics) - print(f"Possible problems in {file_name}", possible_problems) - return panics, possible_problems - - -def check_changes_in_valid_files() -> tuple[bool, str]: - panics, possible_problems = check_file("result_valid_files.txt") - - if len(panics) != 0 or len(possible_problems) != 0: - print(">>>>> Found changed in previously valid files (see above)") - - github_comment = "\nFound regression in converting images that properly converted in develop branch.\n" - for line in panics: - github_comment += ( - line.replace(RUST_PANIC, "") + " - panicked when checking\n" - ) - for line in possible_problems: - github_comment += ( - line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n" - ) - github_comment += "\n" - - return True, github_comment - return False, "" - - -def check_changes_in_invalid_files() -> tuple[bool, str]: - panics, possible_problems = check_file("result_not_valid_files.txt") - - if len(panics) != 0 or len(possible_problems) != 0: - print(">>>>> Found changed non valid files (see above)") - for line in panics + possible_problems: - print(line) - - github_comment = "\nFound differences in converting images that were not properly converted in develop branch.\n" - for line in panics: - github_comment += ( - line.replace(RUST_PANIC, "") + " - panicked when checking\n" - ) - for line in possible_problems: - github_comment += ( - line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n" - ) - github_comment += "\n" - - return False, github_comment - return False, "" - - -def check_changes_in_image_size() -> tuple[bool, str]: - panics, possible_problems = check_file("result_image_size.txt") - - if len(panics) != 0 or len(possible_problems) != 0: - print(">>>>> Found difference in size generated image(see above)") - for line in panics + possible_problems: - print(line) - - github_comment = "\nGenerated png have different size in each run.\n" - for line in panics: - github_comment += ( - line.replace(RUST_PANIC, "") + " - panicked when checking\n" - ) - for line in possible_problems: - github_comment += ( - line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n" - ) - github_comment += "\n" - - return True, github_comment - return False, "" - - -def send_comment(github_comment: str) -> None: - with open(GITHUB_EVENT_PATH, "r") as file: - data = json.load(file) - comments_url = data["pull_request"]["comments_url"] - - print(f"Sending comment to {comments_url}") - - payload = json.dumps({"body": github_comment}) - curl_command = f'curl -s -S -H "Authorization: token {GITHUB_TOKEN}" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "{payload}" "{comments_url}"' - # Print output from curl - subprocess.run(curl_command, shell=True) + if line.startswith(POSSIBLE_PROBLEM): + return line[len(POSSIBLE_PROBLEM) :] + return None if __name__ == "__main__": - if GITHUB_EVENT_NAME != "pull_request": - print( - "You are running this script in wrong event - expected pull_request, crashes may occur" - ) + data_to_check = [ + ( + "result_valid_files.txt", + True, + "Found difference in converting images that properly converted in develop branch.", + ), + ( + "result_not_valid_files.txt", + False, + "Found differences in converting images that were not properly converted in develop branch(this may be improvement).", + ), + ( + "result_image_size.txt", + True, + "Generated png have different size in each run.", + ), + ( + "result_crashes.txt", + True, + "Found crashes in crashes during when converting svg to png.", + ), + ] - print(os.getcwd()) + fail_ci = False + comment = "" + for file_name, fail_ci_when_triggered, new_comment in data_to_check: + possible_problem = check_file(file_name) + if possible_problem is not None: + print(f">>>>> Found changes in {file_name} (look at CI steps from above)") + print(f"Comment: {new_comment}") - print() - fail_ci_v, comment_v = check_changes_in_valid_files() - print() - fail_ci_i, comment_i = check_changes_in_invalid_files() - print() - fail_ci_s, comment_s = check_changes_in_image_size() - print() + if fail_ci_when_triggered: + fail_ci = True + comment += new_comment + "\n" - fail_ci = fail_ci_v or fail_ci_i or fail_ci_s - comment = (comment_v + comment_i + comment_s).strip() + to_write = "Regression report:\n" - if GITHUB_EVENT_NAME == "pull_request": - if len(comment) > 0: - send_comment(comment) + if len(comment) > 0: + to_write += comment + to_write += "\nCheck CI for artifacts/logs for more details." + else: + to_write += "Not found any changes(both improvements and regressions) in this PR with test data." + + print("\n\n" + to_write + "\n\n") + + with open("comment.txt", "w") as file: + file.write(to_write) if fail_ci: - raise Exception("Found regression in image conversion") + with open("fail_ci.txt", "w") as file: + file.write("Fail") diff --git a/test/regression/check_same_image_size.py b/test/regression/check_same_image_size.py index 9e25e02f..e3430695 100644 --- a/test/regression/check_same_image_size.py +++ b/test/regression/check_same_image_size.py @@ -9,8 +9,10 @@ if ( or not os.path.isfile(sys.argv[2]) ): print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"') - print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"') - raise ValueError("Missing or invalid input file or missing path to svg2png (panicked)") + print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE NUMBER_OF_TRIES"') + raise ValueError( + "POSSIBLE_PROBLEM - Missing or invalid input file or missing path to svg2png" + ) try_number = int(sys.argv[4]) image_size = sys.argv[3] @@ -33,8 +35,9 @@ for i in range(try_number + 1): sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True)) if len(sizes) == 1: - print(f"Image size {str(sizes)}") + print(f"Not found problem with generating images, sizes - {str(sizes)}") else: print( - f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}" + f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}", + file=sys.stderr, ) diff --git a/test/regression/settings.toml b/test/regression/settings_comparison.toml similarity index 82% rename from test/regression/settings.toml rename to test/regression/settings_comparison.toml index 4062b1b3..fa2ab5c9 100644 --- a/test/regression/settings.toml +++ b/test/regression/settings_comparison.toml @@ -3,13 +3,13 @@ folder_with_files_to_check = "FilesToTest" problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash) output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences) ignored_files_path = "IgnoredSVG" # Place where to save ignored files -timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds +timeout = 120 limit_threads = 0 # 0 will use all available threads px_size_of_generated_file = 400 ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files ignore_similarity_checking_step = false # Useful to finding problems with generating files ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters -similarity = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range +max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one debug_show_always_output = false # Allows to find broken files @@ -18,15 +18,21 @@ remove_problematic_files_after_copying = false # Remove from output folder probl remove_broken_files_after_copying = false # Remove from output folder broken svg files remove_ignored_files_after_copying = false # Removes not supported folders after copyting remove_generated_png_files_at_end = false # Remove all png from output folder at end +lottie_path = "" +lottie_broken_files_path = "" +lottie_test = false +thorvg_path = "" +thorvg_broken_files_path = "" +thorvg_test = false [first_tool] name = "thorvg_pr" path = "./build/src/tools/svg2png/svg2png" png_name_ending = "_thorvg_pr.png" -arguments = "{FILE} -r {SIZE}x{SIZE}" +arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff" [other_tool] name = "thorvg_develop" path = "./thorvg_develop/build/src/tools/svg2png/svg2png" png_name_ending = "_thorvg_develop.png" -arguments = "{FILE} -r {SIZE}x{SIZE}" +arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff" diff --git a/test/regression/settings_crash_leak.toml b/test/regression/settings_crash_leak.toml new file mode 100644 index 00000000..f29e1295 --- /dev/null +++ b/test/regression/settings_crash_leak.toml @@ -0,0 +1,39 @@ +[general] +folder_with_files_to_check = "FilesToTest" +problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash) +output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences) +ignored_files_path = "IgnoredSVG" # Place where to save ignored files +timeout = 120 +limit_threads = 0 # 0 will use all available threads +px_size_of_generated_file = 400 +ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files +ignore_similarity_checking_step = false # Useful to finding problems with generating files +ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters +max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range +limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files +remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one +debug_show_always_output = false # Allows to find broken files +return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1 +remove_problematic_files_after_copying = false # Remove from output folder problematic svg files +remove_broken_files_after_copying = false # Remove from output folder broken svg files +remove_ignored_files_after_copying = false # Removes not supported folders after copyting +remove_generated_png_files_at_end = false # Remove all png from output folder at end +lottie_path = "" +lottie_broken_files_path = "" +lottie_test = false +thorvg_path = "./build/src/tools/svg2png/svg2png" +thorvg_broken_files_path = "BrokenFILES" +thorvg_test = true + +[first_tool] +name = "thorvg_pr" +path = "./build/src/tools/svg2png/svg2png" +png_name_ending = "_thorvg_pr.png" +arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff" + +[other_tool] +name = "thorvg_develop" +path = "./thorvg_develop/build/src/tools/svg2png/svg2png" +png_name_ending = "_thorvg_develop.png" +arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff" +