git_clone/thorvg
1
0
Fork 0
mirror of https://github.com/thorvg/thorvg.git synced 2025-06-08 05:33:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Finding crashes and adding comments

Browse source
This commit is contained in:
Rafał Mikrut 2024-03-29 20:12:22 +01:00 • committed by Hermet Park
parent 2ff4e29949
commit 1c74e1e40a
5 changed files with 179 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
.github/workflows/regression.yml vendored
View file

@ -14,12 +14,12 @@ jobs:
- name: Install things
run: |
sudo apt-get update
sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib
sudo apt-get install -y wget2 librsvg2-dev librsvg2-bin ninja-build git gcc-multilib g++-multilib earlyoom
sudo pip3 install meson
- name: Compile Thorvg PR
run: |
meson . build -Dtools=svg2png
meson . build -Dtools=svg2png,lottie2gif -Dsavers=gif -Db_sanitize=address,undefined
sudo ninja -C build install
- name: Compile Thorvg Develop
@ -32,14 +32,52 @@ jobs:
- name: Download SVG Regression finder and setup settings
run: |
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/svg_tester
wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.4.0/svg_tester
chmod +x ./svg_tester
mv test/regression/settings.toml settings.toml
- name: Prepare valid files to test
run: |
wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
mv ThorvgValidFiles FilesToTest
find ThorvgNotValidFiles -type f -exec mv {} FilesToTest \;
rmdir ThorvgNotValidFiles
- name: Setup settings for crash/leak/timeout tests
run: |
mv test/regression/settings_crash_leak.toml settings.toml
- name: Run regression finder tests
run: |
./svg_tester 2>&1 | tee result_crashes.txt
- name: Setup settings for comparison tests
run: |
mv test/regression/settings_comparison.toml settings.toml
- name: Store Crashing/Leaking/Timeouting Images
uses: actions/upload-artifact@v4
with:
name: crashing-leaking-timeouting-images
path: BrokenFILES
if-no-files-found: ignore
- name: Clean Data
run: |
rm -rf BrokenSVG || true
rm -rf FilesToTest || true
rm -rf ProblematicSVG || true
rm -rf IgnoredSVG || true
rm -rf BrokenFILES || true
# Test valid files
- name: Prepare valid files to test
run: |
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
mv ThorvgValidFiles FilesToTest
@ -68,11 +106,13 @@ jobs:
rm -rf FilesToTest || true
rm -rf ProblematicSVG || true
rm -rf IgnoredSVG || true
rm -rf BrokenFILES || true
# Test files that may not work currently good with Thorvg, errors from this are not critical
# it is possible that this will find also some improvements
- name: Prepare not valid to test
run: |
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
wget -q https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
mv ThorvgNotValidFiles FilesToTest
@ -101,19 +141,42 @@ jobs:
rm -rf FilesToTest || true
rm -rf ProblematicSVG || true
rm -rf IgnoredSVG || true
rm -rf BrokenFILES || true
- name: Test png reproducibility
run: |
wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip
wget -q https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip
unzip -q files.zip
rm files.zip
cp test/regression/check_same_image_size.py check_same_image_size.py
# Forces to run tasks on different threads if possible, which should help find problem with data races
taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/tools/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt
- name: Check output and send comment
- name: Check results
run: |
export PATH=$PATH:~/.local/bin/
python3 "${GITHUB_WORKSPACE}/.github/workflows/regression_check.py"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Find Comment
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Regression report
# TODO - not works
# - name: Create or update comment
# uses: peter-evans/create-or-update-comment@v4
# with:
# comment-id: ${{ steps.fc.outputs.comment-id }}
# issue-number: ${{ github.event.pull_request.number }}
# body-path: 'comment.txt'
# edit-mode: replace
- name: Fail CI if regression found
run: |
if [[ -f "fail_ci.txt" ]]; then
echo "Check Check results section for more details which tests failed"
exit 1
fi

171
.github/workflows/regression_check.py vendored
View file

@ -1,140 +1,67 @@
import os
import json
import subprocess
from typing import Optional
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME")
GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH") or "Missing"
RUST_PANIC = "panicked"
POSSIBLE_PROBLEM = "POSSIBLE_PROBLEM - "
def check_github_token() -> None:
if not GITHUB_TOKEN:
print("The GITHUB_TOKEN is required.")
exit(1)
def check_file(file_name: str) -> tuple[list[str], list[str]]:
panics = []
possible_problems = []
def check_file(file_name: str) -> Optional[str]:
with open(file_name, "r") as file:
print(f"Checking file {file_name}")
for line in file:
line = line.strip()
if RUST_PANIC in line:
panics.append(line)
elif POSSIBLE_PROBLEM in line:
possible_problems.append(line)
print(f"Panics in {file_name}", panics)
print(f"Possible problems in {file_name}", possible_problems)
return panics, possible_problems
def check_changes_in_valid_files() -> tuple[bool, str]:
panics, possible_problems = check_file("result_valid_files.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found changed in previously valid files (see above)")
github_comment = "\nFound regression in converting images that properly converted in develop branch.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return True, github_comment
return False, ""
def check_changes_in_invalid_files() -> tuple[bool, str]:
panics, possible_problems = check_file("result_not_valid_files.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found changed non valid files (see above)")
for line in panics + possible_problems:
print(line)
github_comment = "\nFound differences in converting images that were not properly converted in develop branch.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return False, github_comment
return False, ""
def check_changes_in_image_size() -> tuple[bool, str]:
panics, possible_problems = check_file("result_image_size.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found difference in size generated image(see above)")
for line in panics + possible_problems:
print(line)
github_comment = "\nGenerated png have different size in each run.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return True, github_comment
return False, ""
def send_comment(github_comment: str) -> None:
with open(GITHUB_EVENT_PATH, "r") as file:
data = json.load(file)
comments_url = data["pull_request"]["comments_url"]
print(f"Sending comment to {comments_url}")
payload = json.dumps({"body": github_comment})
curl_command = f'curl -s -S -H "Authorization: token {GITHUB_TOKEN}" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "{payload}" "{comments_url}"'
# Print output from curl
subprocess.run(curl_command, shell=True)
if line.startswith(POSSIBLE_PROBLEM):
return line[len(POSSIBLE_PROBLEM) :]
return None
if __name__ == "__main__":
if GITHUB_EVENT_NAME != "pull_request":
print(
"You are running this script in wrong event - expected pull_request, crashes may occur"
)
data_to_check = [
(
"result_valid_files.txt",
True,
"Found difference in converting images that properly converted in develop branch.",
),
(
"result_not_valid_files.txt",
False,
"Found differences in converting images that were not properly converted in develop branch(this may be improvement).",
),
(
"result_image_size.txt",
True,
"Generated png have different size in each run.",
),
(
"result_crashes.txt",
True,
"Found crashes in crashes during when converting svg to png.",
),
]
print(os.getcwd())
fail_ci = False
comment = ""
for file_name, fail_ci_when_triggered, new_comment in data_to_check:
possible_problem = check_file(file_name)
if possible_problem is not None:
print(f">>>>> Found changes in {file_name} (look at CI steps from above)")
print(f"Comment: {new_comment}")
print()
fail_ci_v, comment_v = check_changes_in_valid_files()
print()
fail_ci_i, comment_i = check_changes_in_invalid_files()
print()
fail_ci_s, comment_s = check_changes_in_image_size()
print()
if fail_ci_when_triggered:
fail_ci = True
comment += new_comment + "\n"
fail_ci = fail_ci_v or fail_ci_i or fail_ci_s
comment = (comment_v + comment_i + comment_s).strip()
to_write = "Regression report:\n"
if GITHUB_EVENT_NAME == "pull_request":
if len(comment) > 0:
send_comment(comment)
to_write += comment
to_write += "\nCheck CI for artifacts/logs for more details."
else:
to_write += "Not found any changes(both improvements and regressions) in this PR with test data."
print("\n\n" + to_write + "\n\n")
with open("comment.txt", "w") as file:
file.write(to_write)
if fail_ci:
raise Exception("Found regression in image conversion")
with open("fail_ci.txt", "w") as file:
file.write("Fail")

11
test/regression/check_same_image_size.py
View file

@ -9,8 +9,10 @@ if (
or not os.path.isfile(sys.argv[2])
):
print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"')
print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"')
raise ValueError("Missing or invalid input file or missing path to svg2png (panicked)")
print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE NUMBER_OF_TRIES"')
raise ValueError(
"POSSIBLE_PROBLEM - Missing or invalid input file or missing path to svg2png"
)
try_number = int(sys.argv[4])
image_size = sys.argv[3]
@ -33,8 +35,9 @@ for i in range(try_number + 1):
sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True))
if len(sizes) == 1:
print(f"Image size {str(sizes)}")
print(f"Not found problem with generating images, sizes - {str(sizes)}")
else:
print(
f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}"
f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}",
file=sys.stderr,
)

14
test/regression/settings.toml → test/regression/settings_comparison.toml
View file

@ -3,13 +3,13 @@ folder_with_files_to_check = "FilesToTest"
problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
ignored_files_path = "IgnoredSVG" # Place where to save ignored files
timeout = 0 # TODO not working yet, use bigger value than 0 to enable timeout functionality, time in seconds
timeout = 120
limit_threads = 0 # 0 will use all available threads
px_size_of_generated_file = 400
ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
ignore_similarity_checking_step = false # Useful to finding problems with generating files
ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
similarity = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
debug_show_always_output = false # Allows to find broken files
@ -18,15 +18,21 @@ remove_problematic_files_after_copying = false # Remove from output folder probl
remove_broken_files_after_copying = false # Remove from output folder broken svg files
remove_ignored_files_after_copying = false # Removes not supported folders after copyting
remove_generated_png_files_at_end = false # Remove all png from output folder at end
lottie_path = ""
lottie_broken_files_path = ""
lottie_test = false
thorvg_path = ""
thorvg_broken_files_path = ""
thorvg_test = false
[first_tool]
name = "thorvg_pr"
path = "./build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_pr.png"
arguments = "{FILE} -r {SIZE}x{SIZE}"
arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"
[other_tool]
name = "thorvg_develop"
path = "./thorvg_develop/build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_develop.png"
arguments = "{FILE} -r {SIZE}x{SIZE}"
arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"

39
test/regression/settings_crash_leak.toml Normal file
View file

@ -0,0 +1,39 @@
[general]
folder_with_files_to_check = "FilesToTest"
problematic_files_path = "ProblematicSVG" # Where to store problematic files, in which conversion failed(e.g. due program crash)
output_folder = "BrokenSVG" # Place where to save files(Input and output that show differences)
ignored_files_path = "IgnoredSVG" # Place where to save ignored files
timeout = 120
limit_threads = 0 # 0 will use all available threads
px_size_of_generated_file = 400
ignore_conversion_step = false # Ignore step with conversion files from svg to png, just compare files
ignore_similarity_checking_step = false # Useful to finding problems with generating files
ignore_thorvg_not_supported_items = true # Thorvg not supports files with text, filters
max_difference = 5 # Bigger similiarity will show only broken files that are completelly different, looks that 0-100 is quite reasonable range
limit_files = 0 # Limit checked files, useful if you are just wanting to check how app works, 0 will remove limit of checked files
remove_files_from_output_folder_at_start = true # Useful if you run app with different settings and you don't want to remove files one by one
debug_show_always_output = false # Allows to find broken files
return_error_when_finding_invalid_files = false # When finding invalid files(broken or problematic) app will close with status 1
remove_problematic_files_after_copying = false # Remove from output folder problematic svg files
remove_broken_files_after_copying = false # Remove from output folder broken svg files
remove_ignored_files_after_copying = false # Removes not supported folders after copyting
remove_generated_png_files_at_end = false # Remove all png from output folder at end
lottie_path = ""
lottie_broken_files_path = ""
lottie_test = false
thorvg_path = "./build/src/tools/svg2png/svg2png"
thorvg_broken_files_path = "BrokenFILES"
thorvg_test = true
[first_tool]
name = "thorvg_pr"
path = "./build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_pr.png"
arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"
[other_tool]
name = "thorvg_develop"
path = "./thorvg_develop/build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_develop.png"
arguments = "{FILE} -r {SIZE}x{SIZE} -b ffffff"