From 56d18f37c3ad1018c986ebff7ce6efb348a45e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mikrut?= Date: Sun, 30 Apr 2023 12:04:11 +0200 Subject: [PATCH] infra: Improve regression finder --- .github/workflows/regression.yml | 82 +++++++++++++++++++----- .github/workflows/regression_check.sh | 66 ++++++++++++++----- test/regression/check_same_image_size.py | 34 ++++++++++ test/regression/settings.toml | 11 +++- 4 files changed, 160 insertions(+), 33 deletions(-) create mode 100644 test/regression/check_same_image_size.py diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 19e4e2cb..f090a129 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -28,40 +28,92 @@ jobs: sudo ninja -C build install cd .. - - name: Download SVG Regression finder + - name: Download SVG Regression finder and setup settings run: | - wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/svg_tester + wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/svg_tester chmod +x ./svg_tester - - - name: Prepare files to test - run: | - wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.2.0/SVG1000.zip - unzip SVG1000.zip mv test/regression/settings.toml settings.toml - mkdir BrokenSVG - - name: Run tests + # Test valid files + - name: Prepare valid files to test run: | - ./svg_tester 2>&1 | tee regression_logs.txt + wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip + unzip ThorvgValidFiles.zip + mv ThorvgValidFiles FilesToTest - - name: Store Broken Images + - name: Run regression finder tests + run: | + ./svg_tester 2>&1 | tee result_valid_files.txt + + - name: Store Broken Images for valid inputs uses: actions/upload-artifact@v3 with: - name: broken-images + name: differences-in-valid-files path: BrokenSVG if-no-files-found: ignore - - name: Store Problematic Images + - name: Store Problematic Images for valid inputs uses: actions/upload-artifact@v3 with: - name: problematic-images + name: problematic-images-in-valid-files path: ProblematicSVG if-no-files-found: ignore + - name: Clean Data + run: | + rm -rf BrokenSVG || true + rm -rf FilesToTest || true + rm -rf ProblematicSVG || true + rm -rf IgnoredSVG || true + + # Test files that may not work currently good with Thorvg, errors from this are not critical + - name: Prepare not valid to test + run: | + wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip + unzip ThorvgNotValidFiles.zip + mv ThorvgNotValidFiles FilesToTest + + - name: Run invalid files tests + run: | + ./svg_tester 2>&1 | tee result_not_valid_files.txt + + - name: Store Broken Images for not valid inputs + uses: actions/upload-artifact@v3 + with: + name: differences-in-not-valid-files + path: BrokenSVG + if-no-files-found: ignore + + - name: Store Problematic Images for not valid inputs + uses: actions/upload-artifact@v3 + with: + name: problematic-images-in-not-valid-files + path: ProblematicSVG + if-no-files-found: ignore + + - name: Clean Data + run: | + rm -rf BrokenSVG || true + rm -rf FilesToTest || true + rm -rf ProblematicSVG || true + rm -rf IgnoredSVG || true + + - name: Test png reproducibility + run: | + wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip + unzip AA_5.svg.zip + cp test/regression/check_same_image_size.py check_same_image_size.py + taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/bin/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt + - name: Add comment to PR run: | export PATH=$PATH:~/.local/bin/ chmod +x "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh" "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh" env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Failing should happen after creating comment PR and passing all tests + - name: Fail if needed + run: | + if [ -f "EXIT_REQUESTED" ]; then false; fi diff --git a/.github/workflows/regression_check.sh b/.github/workflows/regression_check.sh index 748a897e..3bc72ffd 100644 --- a/.github/workflows/regression_check.sh +++ b/.github/workflows/regression_check.sh @@ -1,28 +1,64 @@ #!/bin/bash if [[ -z "$GITHUB_TOKEN" ]]; then - echo "The GITHUB_TOKEN is required." - exit 1 + echo "The GITHUB_TOKEN is required." + exit 1 fi if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - pwd + pwd - PAYLOAD_REGRESSION=`cat regression_logs.txt | grep "Regression results:"` - COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url) + CREATE_COMMENT=0 + FAIL_CI=0 - echo $COMMENTS_URL - echo $PAYLOAD_REGRESSION + COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url) + echo "$COMMENTS_URL" + POSSIBLE_PROBLEM_SUBSTRING="POSSIBLE_PROBLEM - " - if [[ $PAYLOAD_REGRESSION == *"Regression results:"* ]]; then - echo "Found broken files" - OUTPUT+=$'\nFound difference in generated example png files, look into CI artifacts for list of files that was changed.' - OUTPUT+="$PAYLOAD_REGRESSION" - OUTPUT+=$'\n' - fi + VALID_FILES=$(cat result_valid_files.txt | grep "POSSIBLE_PROBLEM") + echo "$VALID_FILES" + if [[ $VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then + echo "Found changed valid files" + OUTPUT+=$'\nFound regression in converting images that properly converted in develop branch.\n' + OUTPUT+="${VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}" + OUTPUT+=$'\n' + CREATE_COMMENT=1 + FAIL_CI=1 + fi + + NOT_VALID_FILES=$(cat result_not_valid_files.txt | grep "POSSIBLE_PROBLEM") + echo "$NOT_VALID_FILES" + + if [[ $NOT_VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then + echo "Found changed non valid files" + OUTPUT+=$'\nFound differences in converting images that were not properly converted in develop branch.\n' + OUTPUT+="${NOT_VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}" + OUTPUT+=$'\n' + CREATE_COMMENT=1 + fi + + IMAGE_SIZE=$(cat result_image_size.txt | grep "POSSIBLE_PROBLEM") + echo "$IMAGE_SIZE" + + if [[ $IMAGE_SIZE == *"POSSIBLE_PROBLEM"* ]]; then + echo "Found difference in size generated image" + OUTPUT+=$'\nGenerated png have different size in each run.\n' + OUTPUT+="$IMAGE_SIZE" + OUTPUT+="${IMAGE_SIZE#$POSSIBLE_PROBLEM_SUBSTRING}" + OUTPUT+=$'\n' + CREATE_COMMENT=1 + FAIL_CI=1 + fi + + + if [ "$FAIL_CI" -eq 1 ]; then + touch "EXIT_REQUESTED" + fi + + if [ "$CREATE_COMMENT" -eq 1 ]; then PAYLOAD=$(echo '{}' | jq --arg body "$OUTPUT" '.body = $body') - curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "$PAYLOAD" "$COMMENTS_URL" -fi + fi +fi diff --git a/test/regression/check_same_image_size.py b/test/regression/check_same_image_size.py new file mode 100644 index 00000000..73efbaba --- /dev/null +++ b/test/regression/check_same_image_size.py @@ -0,0 +1,34 @@ +import os +import subprocess +import sys + +if len(sys.argv) != 5 or not sys.argv[1].endswith(".svg") or not os.path.isfile(sys.argv[1]) or not os.path.isfile( + sys.argv[2]): + print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"') + print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"') + raise ValueError(f'Missing or invalid input file or missing path to svg2png') + +try_number = int(sys.argv[4]) +image_size = sys.argv[3] +svg2png_path = sys.argv[2] +image_input = sys.argv[1] +image_output = image_input.replace(".svg", ".png") + +args = [svg2png_path, image_input, "-r", f"{image_size}x{image_size}"] +sizes = {} +for i in range(try_number + 1): + # if i % 100 == 0: + # print(f"{i + 1}/{try_number + 1}") + subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + new_size = os.path.getsize(image_output) + if new_size in sizes: + sizes[new_size] += 1 + else: + sizes[new_size] = 1 + +sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True)) + +if len(sizes) == 1: + print(f'Image size {str(sizes)}') +else: + print(f'POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}') diff --git a/test/regression/settings.toml b/test/regression/settings.toml index 90846ab3..232bfdc6 100644 --- a/test/regression/settings.toml +++ b/test/regression/settings.toml @@ -1,18 +1,23 @@ [general] -folder_with_files_to_check = "SVG1000" +folder_with_files_to_check = "FilesToTest" problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash) output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences) +ignored_files_path = "IgnoredSVG" # Place where to save ignored files timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds +limit_threads = 0 # 0 will use all available threads px_size_of_generated_file = 400 ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files ignore_similarity_checking_step = false # Useful to finding problems with generating files ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters -similarity = 10 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range +similarity = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one debug_show_always_output = false # Allows to find broken files -test_version = false # Tests `app --version`, this allows to earlier find but with problematic files instead in runtime, when checking files return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1 +remove_problematic_files_after_copying = false # Remove from output folder problematic svg files +remove_broken_files_after_copying = false # Remove from output folder broken svg files +remove_ignored_files_after_copying = false # Removes not supported folders after copyting +remove_generated_png_files_at_end = false # Remove all png from output folder at end [first_tool] name = "thorvg_pr"