git_clone/thorvg
1
0
Fork 0
mirror of https://github.com/thorvg/thorvg.git synced 2025-06-07 21:23:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Better regression testing

Browse source
This commit is contained in:
Rafał Mikrut 2024-03-14 22:35:45 +01:00 • committed by Jinny You
parent 22e16d560a
commit 25e6539ea7
10 changed files with 205 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/build_android.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
build_x86_64:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: nttld/setup-ndk@v1.3.1
id: setup-ndk
with:
@ -34,15 +34,15 @@ jobs:
meson setup build -Dlog=true -Dloaders="all" -Dsavers="all" -Dbindings="capi" --cross-file /tmp/android_cross.txt
sudo ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_x86_64
path: build/src/libthorvg*
build_aarch64:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: nttld/setup-ndk@v1.3.1
id: setup-ndk
with:
@ -64,7 +64,7 @@ jobs:
meson setup build -Dlog=true -Dloaders="all" -Dsavers="all" -Dbindings="capi" --cross-file /tmp/android_cross.txt
sudo ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_aarch64
path: build/src/libthorvg*

12
.github/workflows/build_ios.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
build_x86_64:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -33,15 +33,15 @@ jobs:
meson setup --backend=ninja build -Dlog=true -Dloaders="all" -Dstatic=true -Dsavers="all" -Dbindings="capi" --cross-file ./cross/ios_x86_64.txt
ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_x86_64
path: build/src/libthorvg*
build_aarch64:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -62,7 +62,7 @@ jobs:
meson setup --backend=ninja build -Dlog=true -Dloaders="all" -Dstatic=true -Dsavers="all" -Dbindings="capi" --cross-file ./cross/ios_aarch64.txt
ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_aarch64
path: build/src/libthorvg*

8
.github/workflows/build_macos.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
build:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -36,7 +36,7 @@ jobs:
static_loaders:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -59,7 +59,7 @@ jobs:
unit_test:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -79,7 +79,7 @@ jobs:
meson setup build -Dloaders="all" -Dsavers="all" -Dbindings="capi" -Dtests=true --errorlogs
ninja -C build install test
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: UnitTestReport
path: build/meson-logs/testlog.txt

8
.github/workflows/build_ubuntu.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -33,7 +33,7 @@ jobs:
static_loaders:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -52,7 +52,7 @@ jobs:
unit_test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: true
@ -71,7 +71,7 @@ jobs:
meson setup build -Dloaders="all" -Dsavers="all" -Dbindings="capi" -Dtests=true --errorlogs
sudo ninja -C build install test
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: UnitTestReport
path: build/meson-logs/testlog.txt

16
.github/workflows/build_windows.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
build:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: ilammy/msvc-dev-cmd@v1
- name: Install Packages
@ -25,15 +25,15 @@ jobs:
where link
ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_windows
path: build/src/thorvg*
static_loaders:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: ilammy/msvc-dev-cmd@v1
- name: Install Packages
@ -46,15 +46,15 @@ jobs:
where link
ninja -C build install
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: result
name: result_windows_static
path: build/src/thorvg*
unit_test:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: ilammy/msvc-dev-cmd@v1
- name: Install Packages
@ -67,7 +67,7 @@ jobs:
where link
ninja -C build install test
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: UnitTestReport
path: build/meson-logs/testlog.txt

43
.github/workflows/regression.yml vendored
View file

@ -1,5 +1,6 @@
name: Regression
# on: [push, pull_request] - enable when testing
on:
pull_request:
branches:
@ -9,7 +10,7 @@ jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install things
run: |
sudo apt-get update
@ -18,14 +19,14 @@ jobs:
- name: Compile Thorvg PR
run: |
meson . build -Dtools=svg2png -Db_sanitize=address,undefined
meson . build -Dtools=svg2png
sudo ninja -C build install
- name: Compile Thorvg Develop
run: |
git clone https://github.com/thorvg/thorvg.git thorvg_develop
cd thorvg_develop
meson . build -Dtools=svg2png -Db_sanitize=address,undefined
meson . build -Dtools=svg2png
sudo ninja -C build install
cd ..
@ -38,8 +39,9 @@ jobs:
# Test valid files
- name: Prepare valid files to test
run: |
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip
unzip ThorvgValidFiles.zip
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
mv ThorvgValidFiles FilesToTest
- name: Run regression finder tests
@ -47,14 +49,14 @@ jobs:
./svg_tester 2>&1 | tee result_valid_files.txt
- name: Store Broken Images for valid inputs
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: differences-in-valid-files
path: BrokenSVG
if-no-files-found: ignore
- name: Store Problematic Images for valid inputs
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: problematic-images-in-valid-files
path: ProblematicSVG
@ -70,8 +72,9 @@ jobs:
# Test files that may not work currently good with Thorvg, errors from this are not critical
- name: Prepare not valid to test
run: |
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip
unzip ThorvgNotValidFiles.zip
wget https://github.com/qarmin/SVG-regression-finder/releases/download/0.3.0/ThorvgNotValidFiles.zip -O files.zip
unzip -q files.zip
rm files.zip
mv ThorvgNotValidFiles FilesToTest
- name: Run invalid files tests
@ -79,14 +82,14 @@ jobs:
./svg_tester 2>&1 | tee result_not_valid_files.txt
- name: Store Broken Images for not valid inputs
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: differences-in-not-valid-files
path: BrokenSVG
if-no-files-found: ignore
- name: Store Problematic Images for not valid inputs
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: problematic-images-in-not-valid-files
path: ProblematicSVG
@ -101,20 +104,16 @@ jobs:
- name: Test png reproducibility
run: |
wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip
unzip AA_5.svg.zip
wget https://github.com/thorvg/thorvg/files/11356766/AA_5.svg.zip -O files.zip
unzip -q files.zip
rm files.zip
cp test/regression/check_same_image_size.py check_same_image_size.py
taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/bin/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt
# Forces to run tasks on different threads if possible, which should help find problem with data races
taskset -c 0-15 python3 check_same_image_size.py AA_5.svg ./build/src/tools/svg2png/svg2png 100 500 2>&1 | tee result_image_size.txt
- name: Add comment to PR
- name: Check output and send comment
run: |
export PATH=$PATH:~/.local/bin/
chmod +x "${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
"${GITHUB_WORKSPACE}/.github/workflows/regression_check.sh"
python3 "${GITHUB_WORKSPACE}/.github/workflows/regression_check.py"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Failing should happen after creating comment PR and passing all tests
- name: Fail if needed
run: |
if [ -f "EXIT_REQUESTED" ]; then false; fi

140
.github/workflows/regression_check.py vendored Normal file
View file

@ -0,0 +1,140 @@
import os
import json
import subprocess
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
GITHUB_EVENT_NAME = os.getenv("GITHUB_EVENT_NAME")
GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH") or "Missing"
RUST_PANIC = "panicked"
POSSIBLE_PROBLEM = "POSSIBLE_PROBLEM - "
def check_github_token() -> None:
if not GITHUB_TOKEN:
print("The GITHUB_TOKEN is required.")
exit(1)
def check_file(file_name: str) -> tuple[list[str], list[str]]:
panics = []
possible_problems = []
with open(file_name, "r") as file:
print(f"Checking file {file_name}")
for line in file:
line = line.strip()
if RUST_PANIC in line:
panics.append(line)
elif POSSIBLE_PROBLEM in line:
possible_problems.append(line)
print(f"Panics in {file_name}", panics)
print(f"Possible problems in {file_name}", possible_problems)
return panics, possible_problems
def check_changes_in_valid_files() -> tuple[bool, str]:
panics, possible_problems = check_file("result_valid_files.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found changed in previously valid files (see above)")
github_comment = "\nFound regression in converting images that properly converted in develop branch.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return True, github_comment
return False, ""
def check_changes_in_invalid_files() -> tuple[bool, str]:
panics, possible_problems = check_file("result_not_valid_files.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found changed non valid files (see above)")
for line in panics + possible_problems:
print(line)
github_comment = "\nFound differences in converting images that were not properly converted in develop branch.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return False, github_comment
return False, ""
def check_changes_in_image_size() -> tuple[bool, str]:
panics, possible_problems = check_file("result_image_size.txt")
if len(panics) != 0 or len(possible_problems) != 0:
print(">>>>> Found difference in size generated image(see above)")
for line in panics + possible_problems:
print(line)
github_comment = "\nGenerated png have different size in each run.\n"
for line in panics:
github_comment += (
line.replace(RUST_PANIC, "") + " - panicked when checking\n"
)
for line in possible_problems:
github_comment += (
line.replace(POSSIBLE_PROBLEM, "") + " - changed visually \n"
)
github_comment += "\n"
return True, github_comment
return False, ""
def send_comment(github_comment: str) -> None:
with open(GITHUB_EVENT_PATH, "r") as file:
data = json.load(file)
comments_url = data["pull_request"]["comments_url"]
print(f"Sending comment to {comments_url}")
payload = json.dumps({"body": github_comment})
curl_command = f'curl -s -S -H "Authorization: token {GITHUB_TOKEN}" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "{payload}" "{comments_url}"'
# Print output from curl
subprocess.run(curl_command, shell=True)
if __name__ == "__main__":
if GITHUB_EVENT_NAME != "pull_request":
print(
"You are running this script in wrong event - expected pull_request, crashes may occur"
)
print(os.getcwd())
print()
fail_ci_v, comment_v = check_changes_in_valid_files()
print()
fail_ci_i, comment_i = check_changes_in_invalid_files()
print()
fail_ci_s, comment_s = check_changes_in_image_size()
print()
fail_ci = fail_ci_v or fail_ci_i or fail_ci_s
comment = (comment_v + comment_i + comment_s).strip()
if GITHUB_EVENT_NAME == "pull_request":
if len(comment) > 0:
send_comment(comment)
if fail_ci:
raise Exception("Found regression in image conversion")

64
.github/workflows/regression_check.sh vendored
View file

@ -1,64 +0,0 @@
#!/bin/bash
if [[ -z "$GITHUB_TOKEN" ]]; then
echo "The GITHUB_TOKEN is required."
exit 1
fi
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
pwd
CREATE_COMMENT=0
FAIL_CI=0
COMMENTS_URL=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.comments_url)
echo "$COMMENTS_URL"
POSSIBLE_PROBLEM_SUBSTRING="POSSIBLE_PROBLEM - "
VALID_FILES=$(cat result_valid_files.txt | grep "POSSIBLE_PROBLEM")
echo "$VALID_FILES"
if [[ $VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then
echo "Found changed valid files"
OUTPUT+=$'\nFound regression in converting images that properly converted in develop branch.\n'
OUTPUT+="${VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}"
OUTPUT+=$'\n'
CREATE_COMMENT=1
FAIL_CI=1
fi
NOT_VALID_FILES=$(cat result_not_valid_files.txt | grep "POSSIBLE_PROBLEM")
echo "$NOT_VALID_FILES"
if [[ $NOT_VALID_FILES == *"POSSIBLE_PROBLEM"* ]]; then
echo "Found changed non valid files"
OUTPUT+=$'\nFound differences in converting images that were not properly converted in develop branch.\n'
OUTPUT+="${NOT_VALID_FILES#$POSSIBLE_PROBLEM_SUBSTRING}"
OUTPUT+=$'\n'
CREATE_COMMENT=1
fi
IMAGE_SIZE=$(cat result_image_size.txt | grep "POSSIBLE_PROBLEM")
echo "$IMAGE_SIZE"
if [[ $IMAGE_SIZE == *"POSSIBLE_PROBLEM"* ]]; then
echo "Found difference in size generated image"
OUTPUT+=$'\nGenerated png have different size in each run.\n'
OUTPUT+="$IMAGE_SIZE"
OUTPUT+="${IMAGE_SIZE#$POSSIBLE_PROBLEM_SUBSTRING}"
OUTPUT+=$'\n'
CREATE_COMMENT=1
FAIL_CI=1
fi
if [ "$FAIL_CI" -eq 1 ]; then
touch "EXIT_REQUESTED"
fi
if [ "$CREATE_COMMENT" -eq 1 ]; then
PAYLOAD=$(echo '{}' | jq --arg body "$OUTPUT" '.body = $body')
curl -s -S -H "Authorization: token $GITHUB_TOKEN" --header "Content-Type: application/vnd.github.VERSION.text+json" --data "$PAYLOAD" "$COMMENTS_URL"
fi
fi

18
test/regression/check_same_image_size.py
View file

@ -2,11 +2,15 @@ import os
import subprocess
import sys
if len(sys.argv) != 5 or not sys.argv[1].endswith(".svg") or not os.path.isfile(sys.argv[1]) or not os.path.isfile(
sys.argv[2]):
if (
len(sys.argv) != 5
or not sys.argv[1].endswith(".svg")
or not os.path.isfile(sys.argv[1])
or not os.path.isfile(sys.argv[2])
):
print('Proper usage "python app.py AA.svg /path/to/svg2png 400 100"')
print('Proper usage "python app.py SVG_FILE SVG_PNG_PATH SIZE_IMAGE TRYING"')
raise ValueError(f'Missing or invalid input file or missing path to svg2png')
raise ValueError("Missing or invalid input file or missing path to svg2png (panicked)")
try_number = int(sys.argv[4])
image_size = sys.argv[3]
@ -15,7 +19,7 @@ image_input = sys.argv[1]
image_output = image_input.replace(".svg", ".png")
args = [svg2png_path, image_input, "-r", f"{image_size}x{image_size}"]
sizes = {}
sizes: dict = {}
for i in range(try_number + 1):
# if i % 100 == 0:
# print(f"{i + 1}/{try_number + 1}")
@ -29,6 +33,8 @@ for i in range(try_number + 1):
sizes = dict(sorted(sizes.items(), key=lambda item: item[1], reverse=True))
if len(sizes) == 1:
print(f'Image size {str(sizes)}')
print(f"Image size {str(sizes)}")
else:
print(f'POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}')
print(
f"POSSIBLE_PROBLEM - Converting svg to png is not reproducible - file sizes {str(sizes)}"
)

6
test/regression/settings.toml
View file

@ -21,12 +21,12 @@ remove_generated_png_files_at_end = false # Remove all png from output folder at
[first_tool]
name = "thorvg_pr"
path = "./build/src/bin/svg2png/svg2png"
path = "./build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_pr.png"
arguments = "{FILE} -r {SIZE}x{SIZE}"
[other_tool]
name = "thorvg_develop"
path = "./thorvg_develop/build/src/bin/svg2png/svg2png"
png_name_ending = "_thorvg_pr.png"
path = "./thorvg_develop/build/src/tools/svg2png/svg2png"
png_name_ending = "_thorvg_develop.png"
arguments = "{FILE} -r {SIZE}x{SIZE}"