.github/workflows/pull_request.yaml - avm - Git at Google

 name: Pull Request
 on:
   pull_request:
   workflow_dispatch:

 defaults:
   run:
     shell: bash

 concurrency:
   group: pr-${{ github.ref }}
   cancel-in-progress: true

 jobs:
   # This job is a hack because GitHub doesn't support referencing env vars for setting runners or container image
   # See https://github.com/orgs/community/discussions/26324
   vars:
     name: Set common variables
     runs-on: ubuntu-latest
     outputs:
       runner: self-hosted
       container-image: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404:lnicolas
       container-image-multilib: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404-multilib:lnicolas
       timeout-minutes: 660
     steps:
       - name: Do nothing
         run: |
           echo noop

   style-checks:
     name: Style Checks
     if: (!cancelled())
     needs:
       - vars
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       DIFF_REF: ${{ github.event.pull_request.base.sha }}
     steps:
       - uses: actions/checkout@v4.1.0
         with:
           fetch-depth: 50
           lfs: true

       - uses: ./.github/actions/common-setup

       - name: Run clang-format
         run: |
           echo "lang-format version: $(clang-format --version)"

           # Run clang-format check.
           for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.[hc]pp' '*.cc' '*.[ch]' | grep -v third_party); do
             exit_code=0
             clang-format -i --style=file $f -n -Werror || exit_code=$?
             if [ ${exit_code} -ne 0 ]; then
               echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
               echo -e "\e[31mREDACTED://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
               exit 1
             fi
           done

       - name: Run cmake-format
         run: |
           echo "cmake-format version: $(cmake-format --version)"

           # Run cmake-format check.
           for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.cmake' 'CMakeLists.txt' | grep -v third_party); do
             exit_code=0
             cmake-format --check $f || exit_code=$?
             if [ ${exit_code} -ne 0 ]; then
               echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
               echo -e "\e[31mREDACTED://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
               exit 1
             fi
           done

   pr-checks:
     name: PR Checks
     if: (!cancelled())
     needs:
       - vars
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       DIFF_REF: ${{ github.event.pull_request.base.sha }}
     steps:
       - uses: actions/checkout@v4.1.0
         with:
           fetch-depth: 50
           lfs: true

       - uses: ./.github/actions/common-setup

       - name: Validate files with the executable bit set
         run: |
           echo lol=${{ github.ref }}
           git diff "${DIFF_REF}" --check

           # Validate files with the executable bit set
           files=$(git diff --name-only "${DIFF_REF}" | tr '\n' ' ')
           git ls-tree -r HEAD $files | while read mode type sha1 file; do
             if [ "$mode" = "100755" ]; then
               case "$file" in
                 configure|*.php|*.pl|*.py|*.sh)
                   ;;
                 *)
                   echo "File $file should not be executable."
                   echo "Only configure|*.php|*.pl|*.py|*.sh are accepted."
                   exit 1
                   ;;
               esac
             fi
           done

   test-data-checks:
     name: Test Data Checks
     if: (!cancelled())
     needs:
       - vars
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       DIFF_REF: ${{ github.event.pull_request.base.sha }}
     steps:
       - uses: actions/checkout@v4.1.0
         with:
           fetch-depth: 50
           lfs: true

       - uses: ./.github/actions/common-setup

       - name: Ensure added test files add declared in test data
         run: |
           # TODO: replace this test with a proper job/rule filter
           if [ -z "$(git diff ${DIFF_REF} "test/")" ]; then
             echo "No test file added."
             exit 0
           fi
           ADDED_TEST_FILES="$(git diff ${DIFF_REF} "test/" \
             | perl -ne  '/^\+.*?"((?!\$).*?\.(ivf|webm|res|mkv|y4m|yuv))"/g and print "$1\n"')" \
           printf "Checking for files:\n%s\n\n" "$ADDED_TEST_FILES"
           echo "$ADDED_TEST_FILES" | while read -r f; do
             if [ -n "$(grep -L "${f}"  test/test-data.sha1 test/test_data_util.cmake)" ]; then
               echo "file: ${f} was not found in test-data.sha1 or test_data_util.cmake"
               exit 1
             fi
           done

   previous-build-x86_64-linux-gcc:
     name: Previous Build (x86_64-linux-gcc)
     uses: ./.github/workflows/build-job-reusable.yaml
     needs:
       - style-checks
       - pr-checks
       - test-data-checks
       - vars
     if: (!cancelled())
     with:
       avm-build-config: ${{ matrix.avm-build-config }}
       parent-job-name: previous-build-x86_64-linux-gcc
       runner: '["${{ needs.vars.outputs.runner }}"]'
       container-image: ${{ needs.vars.outputs.container-image }}
       show-github-context: true
       extra-env-vars: |
         DIFF_REF=${{ github.event.pull_request.base.sha }}
         INSTALLROOT_FOLDER=installroot_old
         # Allow build warnings.
         EXTRA_CMAKE_FLAGS=-DENABLE_WERROR=0
       before-script: |
         #TODO: git fetch --unshallow
         git checkout "${DIFF_REF}"
       artifact-paths: /installroot_old
     strategy:
       matrix:
         avm-build-config:
           - encode-only

   test-data:
     name: Test Data
     uses: ./.github/workflows/common-test-data-reusable.yaml
     needs:
       - vars
     if: (!cancelled())
     with:
       runner: '["${{ needs.vars.outputs.runner }}"]'
       container-image: ${{ needs.vars.outputs.container-image }}

   common-builds:
     name: Common Builds
     uses: ./.github/workflows/common-builds-reusable.yaml
     needs:
       - style-checks
       - pr-checks
       - test-data-checks
       - vars
     if: (!cancelled())
     with:
       runner: '["${{ needs.vars.outputs.runner }}"]'
       container-image: ${{ needs.vars.outputs.container-image }}
       container-image-multilib: ${{ needs.vars.outputs.container-image-multilib }}

   example-test-x86_64-gcc:
     name: Example Test (x86_64-gcc)
     needs:
       - common-builds
       - test-data
       - vars
     if: (!cancelled())
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       LIBAVM_TEST_DATA_PATH: ${{ github.workspace }}/libavm-test-data
     steps:
       - uses: actions/checkout@v4.1.0
         with:
           fetch-depth: 0
           fetch-tags: true
           lfs: true

       - uses: ./.github/actions/common-setup

       - name: Get test data
         uses: actions/download-artifact@v5
         with:
           name: test-data
           path: libavm-test-data

       - name: Get example build
         uses: actions/download-artifact@v5
         with:
           name: example-build
           path: avm_example_build

       - name: Fix executable bit permissions
         run: |
           # See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

           # Look for files starting with ELF header and set executable but
           for file in $(find avm_example_build/ -type f); do
             if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
               echo "Found binary ${file}, fixing permissions."
               chmod a+x "${file}"
             fi
           done

       - name: Run example test
         run: |
           cpu_cores=$(nproc)
           commands=/tmp/parallel.commands.$$
           ret=0

           printf "" >${commands}

           cd avm_example_build
           for test in $(../test/examples.sh --list-tests); do
             echo "sh ../test/examples.sh --bin-path examples --verbose --show-program-output --filter \"\b${test}\b\" 2> >(tee -a ${GITHUB_WORKSPACE}/example.test.${test}.log >&2)" >>${commands}
           done

           cat ${commands}
           time parallel --line-buffer --jobs ${cpu_cores} < ${commands} || ret=$?

           if [ ${ret} -ne 0 ]; then
             echo "Failure running example tests, see logs for details." >&2
             exit 1
           fi

       - name: Upload logs
         uses: actions/upload-artifact@v4.1.0
         if: ${{ always() }}
         with:
           name: example-test-logs
           path: example.test.*.log

   linux-sanitizer-test:
     name: Linux Sanitizer Test
     uses: ./.github/workflows/sanitizer-job-reusable.yaml
     needs:
       - common-builds
       - vars
     if: (!cancelled())
     with:
       avm-sanitizer-type: ${{ matrix.avm-sanitizer-type }}
       max-total-shards: ${{ matrix.max-total-shards }}
       runner: '["${{ needs.vars.outputs.runner }}"]'
       container-image: ${{ needs.vars.outputs.container-image }}
       show-github-context: true
     strategy:
       fail-fast: false
       matrix:
         avm-sanitizer-type:
           - address
           - integer
           - thread
           - undefined
           # CFI Sanitizer commented for now, as lto build does not work
           # - cfi
         include:
           # default to unlimited
           - max-total-shards: 0

   encdec-run:
     name: Enc/Dec
     needs:
       - common-builds
       - previous-build-x86_64-linux-gcc
       - vars
     if: (!cancelled())
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       AVMENC_OUTPUT: ${{ matrix.avm-enc-output }}
       AVMENC_LIMIT: ${{ matrix.avm-enc-limit }}
       AVMENC_QP: 210
       AVMENC_INPUT: Vertical_Bayshore_270x480_2997.y4m
       AVMENC_COMMON_ARGS: "\
         --debug \
         --cpu-used=0 \
         --passes=1 \
         --deltaq-mode=0 \
         --enable-keyframe-filtering=0 \
         --enable-tpl-model=0 \
         --use-fixed-qp-offsets=1 \
         --end-usage=q \
         --obu \
         --psnr \
       "

     steps:
       - name: Configure AVM encoder
         run: |
           if [ -z "${AVMENC_OUTPUT}" ]; then
             echo "AVMENC_OUTPUT is not set." >&2
             exit 1
           fi

           # common encoder arguments to all outputs
           AVMENC_ARGS="${AVMENC_COMMON_ARGS} \
             --qp=${AVMENC_QP} \
             --limit=${AVMENC_LIMIT} \
           "

           # output-specific arguments
           case ${AVMENC_OUTPUT} in
             all-intra)
               AVMENC_ARGS="${AVMENC_ARGS} \
               --kf-max-dist=0 \
               --kf-min-dist=0 \
               "
               ;;

             random-access)
               AVMENC_ARGS="${AVMENC_ARGS} \
                 --auto-alt-ref=1 \
                 --gf-max-pyr-height=4 \
                 --gf-min-pyr-height=4 \
                 --kf-max-dist=65 \
                 --kf-min-dist=65 \
                 --lag-in-frames=19 \
                 --max-gf-interval=16 \
                 --min-gf-interval=16 \
               "
               ;;

             low-delay)
               AVMENC_ARGS="${AVMENC_ARGS} \
                 --gf-max-pyr-height=4 \
                 --gf-min-pyr-height=4 \
                 --kf-max-dist=9999 \
                 --kf-min-dist=9999 \
                 --lag-in-frames=0 \
                 --subgop-config-str=ld \
               "
               ;;

             *)
               echo "Unknown encoder output: '${AVMENC_OUTPUT}'" >&2
               exit 1
               ;;
           esac

           # Append to workflow environment
           echo "AVMENC_ARGS=${AVMENC_ARGS}" >>${GITHUB_ENV}

       - name: Pull input file
         run: |
           curl -s -S -f -O https://gitlab.com/AOMediaCodec/aom-testing/-/raw/master/test-files/${AVMENC_INPUT}.xz
           unxz ${AVMENC_INPUT}.xz

       - name: Pull previous encoder build
         uses: actions/download-artifact@v5
         with:
           name: previous-build-x86_64-linux-gcc-encode-only
           path: enc-previous

       - name: Pull current encoder build
         uses: actions/download-artifact@v5
         with:
           name: example-build
           path: enc-current

       - name: Fix executable bit permissions
         run: |
           # See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

           # Look for files starting with ELF header and set executable but
           for file in $(find . -type f); do
             if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
               echo "Found binary ${file}, fixing permissions."
               chmod a+x "${file}"
             fi
           done

       - name: Encode
         run: |
           printf -- "\n---------------------------------------------------\n"
           echo "Encoder input:  ${AVMENC_INPUT}"
           echo "Encoder output: ${AVMENC_OUTPUT}"
           echo "Encoder args:   ${AVMENC_ARGS}"
           printf -- "---------------------------------------------------\n\n"

           commands=/tmp/parallel.commands.$$
           cat >${commands} <<EOF
           enc-previous/usr/local/bin/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.previous.obu ${AVMENC_INPUT} 2>&1 | tee ${AVMENC_OUTPUT}.previous.psnr.log
           enc-current/avmenc                ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.current.obu  ${AVMENC_INPUT} 2>&1 | tee ${AVMENC_OUTPUT}.current.psnr.log
           EOF

           time parallel --line-buffer < ${commands} || true

           for obu_file in ${AVMENC_OUTPUT}.previous.obu ${AVMENC_OUTPUT}.current.obu; do
             if [ ! -f ${obu_file} ]; then
               echo "Cannot find OBU file ${obu_file}" >&2
               exit 1
             fi

             md5sum --binary ${obu_file} | awk '{ print $1; }' > ${obu_file}.md5
           done

       - name: Decode
         run: |
           enc-current/avmdec \
             ${AVMENC_OUTPUT}.current.obu \
             --output=${AVMDEC_OUTPUT}.current.decoded.y4m \
             --summary \
             2>&1 | tee ${AVMDEC_OUTPUT}.current.summary.log

           if [ ! -f ${AVMDEC_OUTPUT}.current.decoded.y4m ]; then
             echo "Cannot find decoded Y4M file" >&2
             exit 1
           fi

           if [ ! -f ${AVMDEC_OUTPUT}.current.summary.log ]; then
             echo "Cannot find decoder summary log file" >&2
             exit 1
           fi

           for str in 'decoded frames' 'showed frames'; do
             frame_count=$(grep -E -o "[0-9]+ ${str}" "${AVMDEC_OUTPUT}.current.summary.log" | sed -E "s/([0-9]+) ${str}/\1/g")
             echo "${str} = ${frame_count}"
             if [ ${frame_count} -ne ${AVMENC_LIMIT} ]; then
               echo "ERROR: Unexpected number of ${str}. Got ${frame_count}, expected ${AVMENC_LIMIT}" >&2
               exit 1
             fi
           done

       - name: Upload artifacts
         uses: actions/upload-artifact@v4.1.0
         if: ${{ always() }}
         with:
           name: encdec-${{ matrix.avm-enc-output }}
           path: ${{ matrix.avm-enc-output }}.*

       - name: Compare hashes
         run: |
           ALLOW_FAIL=0
           FAILED=0

           if [ -n "$(git --no-pager log --grep STATS_CHANGED --format=format:"%H" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}")" ]; then
             ALLOW_FAIL=1
           fi

           variant=${AVMDEC_OUTPUT}

           if diff -- "${AVMENC_OUTPUT}.previous.obu.md5" "${AVMENC_OUTPUT}.current.obu.md5" &>/dev/null; then
             echo "OK: No differences in ${AVMENC_OUTPUT} outputs found."
             echo ""
           else
             echo "WARNING: ${AVMENC_OUTPUT}.previous.obu.md5 and ${AVMENC_OUTPUT}.current.obu.md5 differ!"
             echo "${AVMENC_OUTPUT}.previous.obu.md5 : $(cat ${AVMENC_OUTPUT}.previous.obu.md5)"
             echo "${AVMENC_OUTPUT}.current.obu.md5  : $(cat ${AVMENC_OUTPUT}.current.obu.md5)"
             echo ""
             FAILED=1
           fi

           if [ ${FAILED} -eq 1 ]; then
             if [ ${ALLOW_FAIL} -eq 1 ]; then
               echo "SUCCESS: Even though a mismatch was detected, one or more"
               echo "         commits contain the STATS_CHANGED keyword, so mismatch"
               echo "         of encode outputs is allowed."
               echo "Commits containing STATS_CHANGED added in this MR:"
               git --no-pager log --grep STATS_CHANGED --format=format:"%h: %s; \"%aN\" <%aE>" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}"
               echo ""
               exit 0
             fi

             echo "FAIL: Failing job due to mismatch and no STATS_CHANGED keyword"
             echo "      was found in any of the commits added in this MR!"
             exit 1
           fi

     strategy:
       fail-fast: false
       matrix:
         avm-enc-output:
           - all-intra
           - random-access
           - low-delay
         include:
           - avm-enc-limit: 30
           - avm-enc-limit: 15
             avm-enc-output: all-intra

   static-analyzer:
     name: Static Analyzer
     needs:
       - vars
     if: (!cancelled())
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     env:
       ANALYZER_MODE: ${{ matrix.analyzer-mode }}
     steps:
       - uses: actions/checkout@v4.1.0
         with:
           fetch-depth: 50
           fetch-tags: true
           lfs: true

       - uses: ./.github/actions/common-setup

       - name: Analyze
         run: |
           mkdir analyzer-${ANALYZER_MODE}

           scan_build() {
             scan-build \
               --exclude third_party \
               --exclude _deps \
               --exclude abseil-cpp \
               --exclude benchmark \
               --exclude cpuinfo \
               --exclude eigen \
               --exclude farmhash \
               --exclude fft2d \
               --exclude flatbuffers \
               --exclude flatbuffers-flatc \
               --exclude fp16 \
               --exclude FP16 \
               --exclude FXdiv \
               --exclude psimd \
               --exclude ml_dtypes \
               --exclude neon2sse \
               --exclude protobuf \
               --exclude pthreadpool \
               --exclude pthreadpool-source \
               --exclude ruy \
               --exclude xnnpack \
               --exclude XNNPACK \
               --exclude googletest \
               -o analyzer-${ANALYZER_MODE} \
               -analyzer-config mode=${ANALYZER_MODE} \
               $*
           }

           # CMake 4.0.0 removed compatibility with CMake < 3.5. Add
           # -DCMAKE_POLICY_VERSION_MINIMUM=3.5 to try configuring anyway.
           scan_build cmake -B avm_build -GNinja -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Debug
           exit_code=0
           scan_build --status-bugs cmake --build avm_build || exit_code=$?
           if [ ${exit_code} -ne 0 ]; then
             echo -e "\e[31mstatic analyzer HTML warning report can be browsed by following this wiki page:\e[0m"
             echo -e "\e[31mhttps://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#viewing-static-analysis-report-from-gitlab-ci\e[0m"
             exit 1
           fi

       - name: Upload artifacts
         uses: actions/upload-artifact@v4.1.0
         if: ${{ failure() }}
         with:
           name: analyzer-${{ matrix.analyzer-mode }}
           path: analyzer-${{ matrix.analyzer-mode }}
           retention-days: 14

     strategy:
       fail-fast: false
       matrix:
         analyzer-mode:
           - deep
           - shallow

   so-checks:
     name: SO checks
     needs:
       - common-builds
       - vars
     if: (!cancelled())
     timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
     runs-on: ${{ needs.vars.outputs.runner }}
     container:
       image: ${{ needs.vars.outputs.container-image }}
     steps:
       - name: Pull build
         uses: actions/download-artifact@v5
         with:
           name: build-x86_64-linux-gcc-shared

       - name: Check SO
         run: |
           for so in $(find -name '*.so*'); do
             echo "Checking ${so} for textrels..."
             if readelf -dW ${so} | grep TEXTREL; then
               textrels=$(eu-findtextrel ${so} | sort | uniq)
               if [ -n "${textrels}" ]; then
                 echo "${textrels}"
                 exit 1
               fi
             fi

             echo "Checking ${so} ELF header/section headers..."
             if readelf -lW ${so} | grep WE; then
               echo "Invalid ELF header/section headers"
               echo "https://android.googlesource.com/platform/bionic/+/master/android-changes-for-ndk-developers.md#Writable-and-Executable-Segments-Enforced-for-API-level-26"
               exit 1
             fi
           done
	name: Pull Request
	on:
	pull_request:
	workflow_dispatch:

	defaults:
	run:
	shell: bash

	concurrency:
	group: pr-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# This job is a hack because GitHub doesn't support referencing env vars for setting runners or container image
	# See https://github.com/orgs/community/discussions/26324
	vars:
	name: Set common variables
	runs-on: ubuntu-latest
	outputs:
	runner: self-hosted
	container-image: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404:lnicolas
	container-image-multilib: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404-multilib:lnicolas
	timeout-minutes: 660
	steps:
	- name: Do nothing
	run: \|
	echo noop

	style-checks:
	name: Style Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v4.1.0
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Run clang-format
	run: \|
	echo "lang-format version: $(clang-format --version)"

	# Run clang-format check.
	for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '.[hc]pp' '.cc' '*.[ch]' \| grep -v third_party); do
	exit_code=0
	clang-format -i --style=file $f -n -Werror \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
	echo -e "\e[31mREDACTED://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
	exit 1
	fi
	done

	- name: Run cmake-format
	run: \|
	echo "cmake-format version: $(cmake-format --version)"

	# Run cmake-format check.
	for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.cmake' 'CMakeLists.txt' \| grep -v third_party); do
	exit_code=0
	cmake-format --check $f \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
	echo -e "\e[31mREDACTED://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
	exit 1
	fi
	done

	pr-checks:
	name: PR Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v4.1.0
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Validate files with the executable bit set
	run: \|
	echo lol=${{ github.ref }}
	git diff "${DIFF_REF}" --check

	# Validate files with the executable bit set
	files=$(git diff --name-only "${DIFF_REF}" \| tr '\n' ' ')
	git ls-tree -r HEAD $files \| while read mode type sha1 file; do
	if [ "$mode" = "100755" ]; then
	case "$file" in
	configure\|.php\|.pl\|.py\|.sh)
	;;
	*)
	echo "File $file should not be executable."
	echo "Only configure\|.php\|.pl\|.py\|.sh are accepted."
	exit 1
	;;
	esac
	fi
	done

	test-data-checks:
	name: Test Data Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v4.1.0
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Ensure added test files add declared in test data
	run: \|
	# TODO: replace this test with a proper job/rule filter
	if [ -z "$(git diff ${DIFF_REF} "test/")" ]; then
	echo "No test file added."
	exit 0
	fi
	ADDED_TEST_FILES="$(git diff ${DIFF_REF} "test/" \
	\| perl -ne '/^\+.?"((?!\$).?\.(ivf\|webm\|res\|mkv\|y4m\|yuv))"/g and print "$1\n"')" \
	printf "Checking for files:\n%s\n\n" "$ADDED_TEST_FILES"
	echo "$ADDED_TEST_FILES" \| while read -r f; do
	if [ -n "$(grep -L "${f}" test/test-data.sha1 test/test_data_util.cmake)" ]; then
	echo "file: ${f} was not found in test-data.sha1 or test_data_util.cmake"
	exit 1
	fi
	done

	previous-build-x86_64-linux-gcc:
	name: Previous Build (x86_64-linux-gcc)
	uses: ./.github/workflows/build-job-reusable.yaml
	needs:
	- style-checks
	- pr-checks
	- test-data-checks
	- vars
	if: (!cancelled())
	with:
	avm-build-config: ${{ matrix.avm-build-config }}
	parent-job-name: previous-build-x86_64-linux-gcc
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	show-github-context: true
	extra-env-vars: \|
	DIFF_REF=${{ github.event.pull_request.base.sha }}
	INSTALLROOT_FOLDER=installroot_old
	# Allow build warnings.
	EXTRA_CMAKE_FLAGS=-DENABLE_WERROR=0
	before-script: \|
	#TODO: git fetch --unshallow
	git checkout "${DIFF_REF}"
	artifact-paths: /installroot_old
	strategy:
	matrix:
	avm-build-config:
	- encode-only

	test-data:
	name: Test Data
	uses: ./.github/workflows/common-test-data-reusable.yaml
	needs:
	- vars
	if: (!cancelled())
	with:
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}

	common-builds:
	name: Common Builds
	uses: ./.github/workflows/common-builds-reusable.yaml
	needs:
	- style-checks
	- pr-checks
	- test-data-checks
	- vars
	if: (!cancelled())
	with:
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	container-image-multilib: ${{ needs.vars.outputs.container-image-multilib }}

	example-test-x86_64-gcc:
	name: Example Test (x86_64-gcc)
	needs:
	- common-builds
	- test-data
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	LIBAVM_TEST_DATA_PATH: ${{ github.workspace }}/libavm-test-data
	steps:
	- uses: actions/checkout@v4.1.0
	with:
	fetch-depth: 0
	fetch-tags: true
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Get test data
	uses: actions/download-artifact@v5
	with:
	name: test-data
	path: libavm-test-data

	- name: Get example build
	uses: actions/download-artifact@v5
	with:
	name: example-build
	path: avm_example_build

	- name: Fix executable bit permissions
	run: \|
	# See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

	# Look for files starting with ELF header and set executable but
	for file in $(find avm_example_build/ -type f); do
	if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
	echo "Found binary ${file}, fixing permissions."
	chmod a+x "${file}"
	fi
	done

	- name: Run example test
	run: \|
	cpu_cores=$(nproc)
	commands=/tmp/parallel.commands.$$
	ret=0

	printf "" >${commands}

	cd avm_example_build
	for test in $(../test/examples.sh --list-tests); do
	echo "sh ../test/examples.sh --bin-path examples --verbose --show-program-output --filter \"\b${test}\b\" 2> >(tee -a ${GITHUB_WORKSPACE}/example.test.${test}.log >&2)" >>${commands}
	done

	cat ${commands}
	time parallel --line-buffer --jobs ${cpu_cores} < ${commands} \|\| ret=$?

	if [ ${ret} -ne 0 ]; then
	echo "Failure running example tests, see logs for details." >&2
	exit 1
	fi

	- name: Upload logs
	uses: actions/upload-artifact@v4.1.0
	if: ${{ always() }}
	with:
	name: example-test-logs
	path: example.test.*.log

	linux-sanitizer-test:
	name: Linux Sanitizer Test
	uses: ./.github/workflows/sanitizer-job-reusable.yaml
	needs:
	- common-builds
	- vars
	if: (!cancelled())
	with:
	avm-sanitizer-type: ${{ matrix.avm-sanitizer-type }}
	max-total-shards: ${{ matrix.max-total-shards }}
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	show-github-context: true
	strategy:
	fail-fast: false
	matrix:
	avm-sanitizer-type:
	- address
	- integer
	- thread
	- undefined
	# CFI Sanitizer commented for now, as lto build does not work
	# - cfi
	include:
	# default to unlimited
	- max-total-shards: 0

	encdec-run:
	name: Enc/Dec
	needs:
	- common-builds
	- previous-build-x86_64-linux-gcc
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	AVMENC_OUTPUT: ${{ matrix.avm-enc-output }}
	AVMENC_LIMIT: ${{ matrix.avm-enc-limit }}
	AVMENC_QP: 210
	AVMENC_INPUT: Vertical_Bayshore_270x480_2997.y4m
	AVMENC_COMMON_ARGS: "\
	--debug \
	--cpu-used=0 \
	--passes=1 \
	--deltaq-mode=0 \
	--enable-keyframe-filtering=0 \
	--enable-tpl-model=0 \
	--use-fixed-qp-offsets=1 \
	--end-usage=q \
	--obu \
	--psnr \
	"

	steps:
	- name: Configure AVM encoder
	run: \|
	if [ -z "${AVMENC_OUTPUT}" ]; then
	echo "AVMENC_OUTPUT is not set." >&2
	exit 1
	fi

	# common encoder arguments to all outputs
	AVMENC_ARGS="${AVMENC_COMMON_ARGS} \
	--qp=${AVMENC_QP} \
	--limit=${AVMENC_LIMIT} \
	"

	# output-specific arguments
	case ${AVMENC_OUTPUT} in
	all-intra)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--kf-max-dist=0 \
	--kf-min-dist=0 \
	"
	;;

	random-access)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--auto-alt-ref=1 \
	--gf-max-pyr-height=4 \
	--gf-min-pyr-height=4 \
	--kf-max-dist=65 \
	--kf-min-dist=65 \
	--lag-in-frames=19 \
	--max-gf-interval=16 \
	--min-gf-interval=16 \
	"
	;;

	low-delay)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--gf-max-pyr-height=4 \
	--gf-min-pyr-height=4 \
	--kf-max-dist=9999 \
	--kf-min-dist=9999 \
	--lag-in-frames=0 \
	--subgop-config-str=ld \
	"
	;;

	*)
	echo "Unknown encoder output: '${AVMENC_OUTPUT}'" >&2
	exit 1
	;;
	esac

	# Append to workflow environment
	echo "AVMENC_ARGS=${AVMENC_ARGS}" >>${GITHUB_ENV}

	- name: Pull input file
	run: \|
	curl -s -S -f -O https://gitlab.com/AOMediaCodec/aom-testing/-/raw/master/test-files/${AVMENC_INPUT}.xz
	unxz ${AVMENC_INPUT}.xz

	- name: Pull previous encoder build
	uses: actions/download-artifact@v5
	with:
	name: previous-build-x86_64-linux-gcc-encode-only
	path: enc-previous

	- name: Pull current encoder build
	uses: actions/download-artifact@v5
	with:
	name: example-build
	path: enc-current

	- name: Fix executable bit permissions
	run: \|
	# See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

	# Look for files starting with ELF header and set executable but
	for file in $(find . -type f); do
	if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
	echo "Found binary ${file}, fixing permissions."
	chmod a+x "${file}"
	fi
	done

	- name: Encode
	run: \|
	printf -- "\n---------------------------------------------------\n"
	echo "Encoder input: ${AVMENC_INPUT}"
	echo "Encoder output: ${AVMENC_OUTPUT}"
	echo "Encoder args: ${AVMENC_ARGS}"
	printf -- "---------------------------------------------------\n\n"

	commands=/tmp/parallel.commands.$$
	cat >${commands} <<EOF
	enc-previous/usr/local/bin/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.previous.obu ${AVMENC_INPUT} 2>&1 \| tee ${AVMENC_OUTPUT}.previous.psnr.log
	enc-current/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.current.obu ${AVMENC_INPUT} 2>&1 \| tee ${AVMENC_OUTPUT}.current.psnr.log
	EOF

	time parallel --line-buffer < ${commands} \|\| true

	for obu_file in ${AVMENC_OUTPUT}.previous.obu ${AVMENC_OUTPUT}.current.obu; do
	if [ ! -f ${obu_file} ]; then
	echo "Cannot find OBU file ${obu_file}" >&2
	exit 1
	fi

	md5sum --binary ${obu_file} \| awk '{ print $1; }' > ${obu_file}.md5
	done

	- name: Decode
	run: \|
	enc-current/avmdec \
	${AVMENC_OUTPUT}.current.obu \
	--output=${AVMDEC_OUTPUT}.current.decoded.y4m \
	--summary \
	2>&1 \| tee ${AVMDEC_OUTPUT}.current.summary.log

	if [ ! -f ${AVMDEC_OUTPUT}.current.decoded.y4m ]; then
	echo "Cannot find decoded Y4M file" >&2
	exit 1
	fi

	if [ ! -f ${AVMDEC_OUTPUT}.current.summary.log ]; then
	echo "Cannot find decoder summary log file" >&2
	exit 1
	fi

	for str in 'decoded frames' 'showed frames'; do
	frame_count=$(grep -E -o "[0-9]+ ${str}" "${AVMDEC_OUTPUT}.current.summary.log" \| sed -E "s/([0-9]+) ${str}/\1/g")
	echo "${str} = ${frame_count}"
	if [ ${frame_count} -ne ${AVMENC_LIMIT} ]; then
	echo "ERROR: Unexpected number of ${str}. Got ${frame_count}, expected ${AVMENC_LIMIT}" >&2
	exit 1
	fi
	done

	- name: Upload artifacts
	uses: actions/upload-artifact@v4.1.0
	if: ${{ always() }}
	with:
	name: encdec-${{ matrix.avm-enc-output }}
	path: ${{ matrix.avm-enc-output }}.*

	- name: Compare hashes
	run: \|
	ALLOW_FAIL=0
	FAILED=0

	if [ -n "$(git --no-pager log --grep STATS_CHANGED --format=format:"%H" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}")" ]; then
	ALLOW_FAIL=1
	fi

	variant=${AVMDEC_OUTPUT}

	if diff -- "${AVMENC_OUTPUT}.previous.obu.md5" "${AVMENC_OUTPUT}.current.obu.md5" &>/dev/null; then
	echo "OK: No differences in ${AVMENC_OUTPUT} outputs found."
	echo ""
	else
	echo "WARNING: ${AVMENC_OUTPUT}.previous.obu.md5 and ${AVMENC_OUTPUT}.current.obu.md5 differ!"
	echo "${AVMENC_OUTPUT}.previous.obu.md5 : $(cat ${AVMENC_OUTPUT}.previous.obu.md5)"
	echo "${AVMENC_OUTPUT}.current.obu.md5 : $(cat ${AVMENC_OUTPUT}.current.obu.md5)"
	echo ""
	FAILED=1
	fi

	if [ ${FAILED} -eq 1 ]; then
	if [ ${ALLOW_FAIL} -eq 1 ]; then
	echo "SUCCESS: Even though a mismatch was detected, one or more"
	echo " commits contain the STATS_CHANGED keyword, so mismatch"
	echo " of encode outputs is allowed."
	echo "Commits containing STATS_CHANGED added in this MR:"
	git --no-pager log --grep STATS_CHANGED --format=format:"%h: %s; \"%aN\" <%aE>" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}"
	echo ""
	exit 0
	fi

	echo "FAIL: Failing job due to mismatch and no STATS_CHANGED keyword"
	echo " was found in any of the commits added in this MR!"
	exit 1
	fi

	strategy:
	fail-fast: false
	matrix:
	avm-enc-output:
	- all-intra
	- random-access
	- low-delay
	include:
	- avm-enc-limit: 30
	- avm-enc-limit: 15
	avm-enc-output: all-intra

	static-analyzer:
	name: Static Analyzer
	needs:
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	ANALYZER_MODE: ${{ matrix.analyzer-mode }}
	steps:
	- uses: actions/checkout@v4.1.0
	with:
	fetch-depth: 50
	fetch-tags: true
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Analyze
	run: \|
	mkdir analyzer-${ANALYZER_MODE}

	scan_build() {
	scan-build \
	--exclude third_party \
	--exclude _deps \
	--exclude abseil-cpp \
	--exclude benchmark \
	--exclude cpuinfo \
	--exclude eigen \
	--exclude farmhash \
	--exclude fft2d \
	--exclude flatbuffers \
	--exclude flatbuffers-flatc \
	--exclude fp16 \
	--exclude FP16 \
	--exclude FXdiv \
	--exclude psimd \
	--exclude ml_dtypes \
	--exclude neon2sse \
	--exclude protobuf \
	--exclude pthreadpool \
	--exclude pthreadpool-source \
	--exclude ruy \
	--exclude xnnpack \
	--exclude XNNPACK \
	--exclude googletest \
	-o analyzer-${ANALYZER_MODE} \
	-analyzer-config mode=${ANALYZER_MODE} \
	$*
	}

	# CMake 4.0.0 removed compatibility with CMake < 3.5. Add
	# -DCMAKE_POLICY_VERSION_MINIMUM=3.5 to try configuring anyway.
	scan_build cmake -B avm_build -GNinja -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Debug
	exit_code=0
	scan_build --status-bugs cmake --build avm_build \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mstatic analyzer HTML warning report can be browsed by following this wiki page:\e[0m"
	echo -e "\e[31mhttps://gitlab.com/AOMediaCodec/avm/-/wikis/Reproducing-CI-Test-Failures-Locally#viewing-static-analysis-report-from-gitlab-ci\e[0m"
	exit 1
	fi

	- name: Upload artifacts
	uses: actions/upload-artifact@v4.1.0
	if: ${{ failure() }}
	with:
	name: analyzer-${{ matrix.analyzer-mode }}
	path: analyzer-${{ matrix.analyzer-mode }}
	retention-days: 14

	strategy:
	fail-fast: false
	matrix:
	analyzer-mode:
	- deep
	- shallow

	so-checks:
	name: SO checks
	needs:
	- common-builds
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	steps:
	- name: Pull build
	uses: actions/download-artifact@v5
	with:
	name: build-x86_64-linux-gcc-shared

	- name: Check SO
	run: \|
	for so in $(find -name '.so'); do
	echo "Checking ${so} for textrels..."
	if readelf -dW ${so} \| grep TEXTREL; then
	textrels=$(eu-findtextrel ${so} \| sort \| uniq)
	if [ -n "${textrels}" ]; then
	echo "${textrels}"
	exit 1
	fi
	fi

	echo "Checking ${so} ELF header/section headers..."
	if readelf -lW ${so} \| grep WE; then
	echo "Invalid ELF header/section headers"
	echo "https://android.googlesource.com/platform/bionic/+/master/android-changes-for-ndk-developers.md#Writable-and-Executable-Segments-Enforced-for-API-level-26"
	exit 1
	fi
	done