.gitlab/ci_nightly.yml

#
# Nightly-only jobs
#
# This file is conditionally included in the main
# .gitlab-ci.yml only when doing nightly builds
# so the builds here do not explicitly need to be
# made nightly-only using rules.
#

default:
  timeout: 11 hours

# Simple x86_64 Linux build
Linux Nightly Build:
  stage: build
  interruptible: true
  # CMake 4.0.0 removed compatibility with CMake < 3.5. Add
  # -DCMAKE_POLICY_VERSION_MINIMUM=3.5 to try configuring anyway.
  variables:
    CMAKE_FLAGS: >-
      -DCMAKE_POLICY_VERSION_MINIMUM=3.5
      -DCMAKE_BUILD_TYPE=RelWithDebInfo
      -DAOM_EXTRA_C_FLAGS="--coverage"
      -DAOM_EXTRA_CXX_FLAGS="--coverage"
      -DAOM_EXTRA_EXE_LINKER_FLAGS="--coverage"
  script:
    - 'echo "CMake Flags: $CMAKE_FLAGS"'
    - cmake -B "aom_nightly_build" -GNinja $CMAKE_FLAGS
    - cmake --build "aom_nightly_build" --target aomenc
    - cmake --build "aom_nightly_build" --target test_libaom
  needs: []
  artifacts:
    expire_in: 1 day
    paths:
      - 'aom_nightly_build/aomenc'
      - 'aom_nightly_build/test_libaom'
      - 'aom_nightly_build/**/*.gcno'
      - 'aom_nightly_build/**/*.[hc]'
      - 'aom_nightly_build/**/*.[hc]pp'
      - 'aom_nightly_build/**/*.cc'
      - 'aom_nightly_build/**/*.inc'

# Full unit tests run (with coverage report)
Linux All Unit Test:
  stage: test
  interruptible: true
  parallel: 24
  variables:
    LIBAOM_TEST_DATA_PATH: ${CI_PROJECT_DIR}/libaom-test-data
    GTEST_TOTAL_SHARDS: ${CI_NODE_TOTAL}
    GTEST_OUTPUT: "xml:report.xml"
  before_script:
    - export GTEST_SHARD_INDEX=$((CI_NODE_INDEX - 1))
  script:
    - cd aom_nightly_build
    - ./test_libaom
    # Ignore parse errors due to a bug in `gcovr 7.0`:
    # https://github.com/gcovr/gcovr/issues/882
    # Ignore errors about 'no_working_dir_found' because
    # tarballs for third_party code (e.g. tensorflow and dependencies) are NOT
    # extracted here and so the source files from those are NOT available.
    - >-
      gcovr
      --gcov-ignore-parse-errors
      --gcov-ignore-errors=no_working_dir_found
      --gcov-executable gcov-14
      --exclude ../third_party --exclude _deps --exclude abseil-cpp --exclude
      benchmark --exclude cpuinfo --exclude eigen --exclude farmhash --exclude
      fft2d --exclude flatbuffers --exclude flatbuffers-flatc --exclude fp16
      --exclude FP16 --exclude FXdiv --exclude psimd  --exclude ml_dtypes
      --exclude neon2sse  --exclude protobuf --exclude pthreadpool --exclude
      pthreadpool-source --exclude ruy --exclude xnnpack --exclude XNNPACK
      --exclude gemmlowp  --exclude googletest
      --exclude-directories "third_party"
      --exclude-directories "aom_nightly_build/third_party"
      --json -o "coverage_${CI_NODE_INDEX}.json"
      -r ../ . >  "coverage_${CI_NODE_INDEX}.log" 2>&1
  needs:
    - 'Test Data'
    - 'Linux Nightly Build'
  artifacts:
    when: always
    reports:
      junit: ./aom_nightly_build/report.xml
    paths:
      - aom_nightly_build/coverage_*.json
      - aom_nightly_build/coverage_*.log

# Produce actual coverage report from the
# "Linux All Unit Test" job that ran in the
# test stage before.
Coverage Report:
  stage: report
  interruptible: true
  script:
    - mkdir -p coverage-html
    - cd aom_nightly_build
    - >-
      gcovr
      --gcov-executable gcov-14
      --add-tracefile "coverage_*.json"
      --exclude ../third_party --exclude _deps --exclude abseil-cpp --exclude
      benchmark --exclude cpuinfo --exclude eigen --exclude farmhash --exclude
      fft2d --exclude flatbuffers --exclude flatbuffers-flatc --exclude fp16
      --exclude FP16 --exclude FXdiv --exclude psimd  --exclude ml_dtypes
      --exclude neon2sse  --exclude protobuf --exclude pthreadpool --exclude
      pthreadpool-source --exclude ruy --exclude xnnpack --exclude XNNPACK
      --exclude gemmlowp  --exclude googletest
      --xml ../coverage.xml
      --html --html-details ../coverage-html/coverage.html
      --print-summary
      -r ../ .
  needs:
    - 'Linux Nightly Build'
    - 'Linux All Unit Test'
  coverage: '/lines:\s(\d+.\d+\%)/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    paths:
      - coverage-html


#
# Tests run with varying SIMD levels
# [sse2, sse3, ssse3, sse4_1, sse4_2, avx2]
#
# This is not done with a big matrix, because
# GitLab CI has a limitation on expanding matrices
# being limited to 50 jobs only. If that limit is
# ever raised, this should be re-written using a
# matrix!
#

# Common unit-optim template
# This is inherited by all the SIMD-level jobs
.common-linux-unit-optim:
  stage: test
  interruptible: true
  parallel: 10
  variables:
    LIBAOM_TEST_DATA_PATH: ${CI_PROJECT_DIR}/libaom-test-data
    GTEST_TOTAL_SHARDS: ${CI_NODE_TOTAL}
    GTEST_OUTPUT: "xml:report.xml"
  script:
    - cd aom_nightly_build
    - |
      case "${AOM_OPTIM_LEVEL}" in
        sse) export AOM_SIMD_CAPS_MASK=0x3 ;;
        sse2) export AOM_SIMD_CAPS_MASK=0x7 ;;
        sse3) export AOM_SIMD_CAPS_MASK=0xf ;;
        ssse3) export AOM_SIMD_CAPS_MASK=0x1f ;;
        sse4_1) export AOM_SIMD_CAPS_MASK=0x3f ;;
        # sse4_2 was added *after* avx2 so the mask is a little weird
        sse4_2) export AOM_SIMD_CAPS_MASK=0x13f ;;
        avx2) export AOM_SIMD_CAPS_MASK=0x1ff ;;
      esac

      test_filter="-C*:MMX*:SSE*:SSSE*:AVX*:*Large*"

    - ./test_libaom --gtest_filter=${test_filter}
  needs:
    - 'Test Data'
    - 'Linux Nightly Build'
  artifacts:
    when: always
    reports:
      junit: ./aom_nightly_build/report.xml
  before_script:
    - export GTEST_SHARD_INDEX=$((CI_NODE_INDEX - 1))

Linux Unit (x86_64, sse2):
  extends: .common-linux-unit-optim
  variables:
    AOM_OPTIM_LEVEL: sse2

Linux Unit (x86_64, ssse3):
  extends: .common-linux-unit-optim
  variables:
    AOM_OPTIM_LEVEL: ssse3

Linux Unit (x86_64, sse4_2):
  extends: .common-linux-unit-optim
  variables:
    AOM_OPTIM_LEVEL: sse4_2

#
# Sanitizer jobs
# The following jobs run the unit tests but
# with a built where the respective sanitizers
# are enabled.
#
# This is not using a matrix as the jobs
# depend on other jobs that were built by
# a matrix job before and a matrix can not
# depend on other matrix-jobs unfortunately.
#

# Common sanitizer unit tests template
# This is inherited by all sanitizer test jobs
.sanitizer-common-nigtly:
  stage: test
  interruptible: true
  parallel: 24
  variables:
    LIBAOM_TEST_DATA_PATH: ${CI_PROJECT_DIR}/libaom-test-data
    GTEST_TOTAL_SHARDS: ${CI_NODE_TOTAL}
    GTEST_OUTPUT: "xml:report.xml"
    SANITIZER_OPTIONS: "\
     :handle_segv=1\
     :handle_abort=1\
     :handle_sigfpe=1\
     :fast_unwind_on_fatal=1\
     :allocator_may_return_null=1"
  before_script:
    - export PATH="/usr/lib/llvm-18/bin:${PATH}"
    - export GTEST_SHARD_INDEX=$((CI_NODE_INDEX - 1))
  script:
    - |
      case $AOM_SANITIZER_TYPE in
        address)
          SANITIZER_OPTIONS="${SANITIZER_OPTIONS}:detect_stack_use_after_return=1"
          SANITIZER_OPTIONS="${SANITIZER_OPTIONS}:max_uar_stack_size_log=17"
          export ASAN_OPTIONS="${SANITIZER_OPTIONS}"
          ;;
        thread)
          # The thread sanitizer uses a subset.
          TSAN_OPTIONS="handle_sigfpe=1"
          TSAN_OPTIONS="$TSAN_OPTIONS handle_segv=1"
          TSAN_OPTIONS="$TSAN_OPTIONS handle_abort=1"
          export TSAN_OPTIONS
          ;;
        undefined|integer)
          SANITIZER_OPTIONS="${SANITIZER_OPTIONS}:print_stacktrace=1:suppressions=.gitlab/UBSan.supp"
          export UBSAN_OPTIONS="${SANITIZER_OPTIONS}"
          ;;
      esac
    - |
      filter="*Large*"
      # As of 17 May 2018 there are no large threading tests.
      [ "${AOM_SANITIZER_TYPE}" = "thread" ] && filter="*Thread*Large*" || :
    - ./aom_build/${AOM_SANITIZER_TYPE}/test_libaom --gtest_filter="${filter}" 2> >(tee -a sanitizer.log >&2)
    - |
      # Looking for sanitizer output in log...
      grep -q "\(ERROR\|WARNING\): \(Address\|Thread\|Memory\|Leak\)Sanitizer:" sanitizer.log && {
        echo "Found sanitizer errors or warnings, check the log:"
        cat sanitizer.log
        exit 1
      }
      # Looking for UBSan output in log (differs from the common format)
      grep -q ":[[:digit:]]\+:[[:digit:]]\+: runtime error:" sanitizer.log && {
        echo "Found sanitizer errors or warnings, check the log:"
        cat sanitizer.log
        exit 1
      }
      echo "No sanitizer errors found"
  artifacts:
    expose_as: 'Nightly sanitizer logs'
    name: "sanitizer.log-$CI_JOB_ID"
    when: on_failure
    paths:
      - sanitizer.log

Linux Sanitizer (address) Test Nightly:
  extends: .sanitizer-common-nigtly
  parallel: 34
  variables:
    AOM_SANITIZER_TYPE: address
  needs:
    - 'Test Data'
    - 'Linux Sanitizers (x86_64-clang): [address]'

Linux Sanitizer (undefined) Test Nightly:
  extends: .sanitizer-common-nigtly
  parallel: 34
  variables:
    AOM_SANITIZER_TYPE: undefined
  needs:
    - 'Test Data'
    - 'Linux Sanitizers (x86_64-clang): [undefined]'

Linux Sanitizer (integer) Test Nightly:
  extends: .sanitizer-common-nigtly
  variables:
    AOM_SANITIZER_TYPE: integer
  needs:
    - 'Test Data'
    - 'Linux Sanitizers (x86_64-clang): [integer]'

Linux Sanitizer (thread) Test Nightly:
  extends: .sanitizer-common-nigtly
  parallel: 16
  variables:
    AOM_SANITIZER_TYPE: thread
  needs:
    - 'Test Data'
    - 'Linux Sanitizers (x86_64-clang): [thread]'


# Memory sanitizer (MSan) needs special setup where some standard libraries
# have to be built with memory sanitizer. Hence, the separate build and test
# jobs for MSan.
Linux Sanitizers (x86_64-clang) (Memory):
  stage: build
  interruptible: true
  script:
    - LLVM_VERSION_MAJOR=18
    - LLVM_VERSION="${LLVM_VERSION_MAJOR}.1.3"
    - export PATH="/usr/lib/llvm-${LLVM_VERSION_MAJOR}/bin:${PATH}"
    # Build cxx and cxxabi with msan.
    - git clone --depth=1 --branch llvmorg-${LLVM_VERSION}  https://github.com/llvm/llvm-project
    - LLVM_MSAN_BUILD_DIR=${PWD}/llvm-project/msan_out
    - cmake -G Ninja -S llvm-project/runtimes -B "${LLVM_MSAN_BUILD_DIR}"
        -DCMAKE_BUILD_TYPE=Release
        -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind"
        -DCMAKE_C_COMPILER=clang
        -DCMAKE_CXX_COMPILER=clang++
        -DLLVM_USE_SANITIZER=MemoryWithOrigins
        -DLIBCXXABI_USE_LLVM_UNWINDER=OFF
    - cmake --build "${LLVM_MSAN_BUILD_DIR}" --config Release -- cxx cxxabi unwind
    # Build AVM with msan.
    - AVM_MSAN_BUILD_DIR="aom_build/memory"
    - |
      MSAN_CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2"
      MSAN_CFLAGS="${MSAN_CFLAGS} -fno-omit-frame-pointer -g -O2"
      MSAN_CFLAGS="${MSAN_CFLAGS} -I${LLVM_MSAN_BUILD_DIR}/include"
    - |
      MSAN_CXXFLAGS="${MSAN_CFLAGS}"
      MSAN_CXXFLAGS="${MSAN_CXXFLAGS} -I${LLVM_MSAN_BUILD_DIR}/include/c++/v1"
      MSAN_CXXFLAGS="${MSAN_CXXFLAGS} -stdlib=libc++"
    - MSAN_LD_FLAGS="${MSAN_CXXFLAGS} -L${LLVM_MSAN_BUILD_DIR}/lib -lc++abi"
    - export CFLAGS="${MSAN_CFLAGS}"
    - export CXXFLAGS="${MSAN_CXXFLAGS}"
    - export LDFLAGS="${MSAN_LD_FLAGS}"
    - echo "CFLAGS = ${CFLAGS}"
    - echo "CXXFLAGS = ${CXXFLAGS}"
    - echo "LDFLAGS = ${LDFLAGS}"
    - cmake -B "${AVM_MSAN_BUILD_DIR}" -G Ninja
        -DAOM_TARGET_CPU=generic
        -DCMAKE_C_COMPILER=clang
        -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_POLICY_VERSION_MINIMUM=3.5
        -DENABLE_CCACHE=1
        -DENABLE_EXAMPLES=0
    - cmake --build "${AVM_MSAN_BUILD_DIR}" --target test_libaom
  needs: []
  artifacts:
    expire_in: 1 day
    paths:
      - aom_build/*/test_libaom
      - llvm-project/msan_out/lib/

Linux Sanitizer (memory) Test Nightly:
  stage: test
  interruptible: true
  parallel: 20
  variables:
    LIBAOM_TEST_DATA_PATH: ${CI_PROJECT_DIR}/libaom-test-data
    GTEST_TOTAL_SHARDS: ${CI_NODE_TOTAL}
    GTEST_OUTPUT: "xml:report.xml"
    SANITIZER_OPTIONS: "\
     :handle_segv=1\
     :handle_abort=1\
     :handle_sigfpe=1\
     :fast_unwind_on_fatal=1\
     :allocator_may_return_null=1"
  before_script:
    - export PATH="/usr/lib/llvm-18/bin:${PATH}"
    - export GTEST_SHARD_INDEX=$((CI_NODE_INDEX - 1))
  script:
    - export MSAN_OPTIONS="${SANITIZER_OPTIONS}"
    - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${PWD}/llvm-project/msan_out/lib"
    - filter="-*Large*:*MD5Test*:*TestVectorTest*:*InvalidFile*:*ExternalFrameBuffer*:*UninstantiatedParameterizedTestSuite*"
    - ./aom_build/memory/test_libaom --gtest_filter="${filter}"
  needs:
    - 'Test Data'
    - 'Linux Sanitizers (x86_64-clang) (Memory)'


# Encode 2 GOPs in parallel and serial manner and ensure that results match.
.gop-enc-common:
  stage: test
  interruptible: true
  variables:
    AOMENC_LIMIT: 130
    AOMENC_SKIP: 0
    AOMENC_QP: 160
    AOMENC_INPUT: Vertical_Bayshore_270x480_2997.y4m
    AOMENC_OUTPUT: serial_2gops
  before_script:
    - curl -s -S -f -O https://gitlab.com/AOMediaCodec/aom-testing/-/raw/master/test-files/${AOMENC_INPUT}.xz
    - unxz ${AOMENC_INPUT}.xz
  script:
    - aom_nightly_build/aomenc
        --debug
        --cpu-used=0
        --passes=1
        --lag-in-frames=19
        --auto-alt-ref=1
        --min-gf-interval=16
        --max-gf-interval=16
        --gf-min-pyr-height=4
        --gf-max-pyr-height=4
        --kf-min-dist=65
        --kf-max-dist=65
        --use-fixed-qp-offsets=1
        --deltaq-mode=0
        --enable-tpl-model=0
        --end-usage=q
        --qp=${AOMENC_QP}
        --enable-keyframe-filtering=0
        --obu
        --limit=${AOMENC_LIMIT}
        --skip=${AOMENC_SKIP}
        -o "${AOMENC_OUTPUT}.obu"
        "${AOMENC_INPUT}" 2>&1 | tee "${AOMENC_OUTPUT}.psnr.log"
    - '[ -f "${AOMENC_OUTPUT}.obu" ] || exit 1'
  artifacts:
    when: always
    paths:
      - ${AOMENC_OUTPUT}.*
  needs:
    - 'Linux Nightly Build'


Serial Encode (2 GOPs):
  extends: .gop-enc-common


Parallel Encode (1st GOP):
  extends: .gop-enc-common
  variables:
    AOMENC_LIMIT: 65
    AOMENC_SKIP: 0
    AOMENC_OUTPUT: parallel_gop1


Parallel Encode (2nd GOP):
  extends: .gop-enc-common
  variables:
    AOMENC_LIMIT: 130
    AOMENC_SKIP: 65
    AOMENC_OUTPUT: parallel_gop2


Serial Parallel compare:
  stage: report
  interruptible: true
  script:
    - cat parallel_gop1.obu parallel_gop2.obu > parallel_2gops.obu
    - diff serial_2gops.obu parallel_2gops.obu
  needs:
    - 'Serial Encode (2 GOPs)'
    - 'Parallel Encode (1st GOP)'
    - 'Parallel Encode (2nd GOP)'