CWG-E135 TFlite local dependencies This commit provides local dependencies for TFlite in CWG-E135. Fixes #467
diff --git a/.gitignore b/.gitignore index a488609..d38da9d 100644 --- a/.gitignore +++ b/.gitignore
@@ -4,5 +4,7 @@ .project .vscode .gitignore +third_party/deps_cmake/ +third_party/tensorflow/ TAGS __pycache__ \ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41d76d4..6b6986d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml
@@ -934,7 +934,7 @@ - mkdir output-${ANALYZER_MODE} - | scan_build() { - scan-build --exclude third_party --exclude _deps --exclude xnnpack --exclude flatbuffers --exclude cpuinfo --exclude abseil-cpp --exclude eigen --exclude ruy --exclude pthreadpool-source -o output-${ANALYZER_MODE} -analyzer-config mode=${ANALYZER_MODE} $* + scan-build --exclude third_party --exclude _deps --exclude abseil-cpp --exclude benchmark --exclude cpuinfo --exclude eigen --exclude farmhash --exclude fft2d --exclude flatbuffers --exclude flatbuffers-flatc --exclude fp16 --exclude FP16 --exclude FXdiv --exclude psimd --exclude ml_dtypes --exclude neon2sse --exclude protobuf --exclude pthreadpool --exclude pthreadpool-source --exclude ruy --exclude xnnpack --exclude XNNPACK --exclude googletest -o output-${ANALYZER_MODE} -analyzer-config mode=${ANALYZER_MODE} $* } - scan_build cmake -B aom_build -GNinja -DCMAKE_BUILD_TYPE=Debug - scan_build --status-bugs cmake --build aom_build
diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b99661..f171587 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -586,11 +586,101 @@ ${AOM_ENCODER_TOOL_TARGETS}) if(CONFIG_TENSORFLOW_LITE) - include(FetchContent) + # Building TFLite from third_party sources. Previously TFLite was fetched + # from github as part of the build, using the following syntax: + # cmake-format: off + # include(FetchContent) + # set(TFLITE_TAG "v2.17.0") + # fetchcontent_declare( + # tflite + # GIT_REPOSITORY https://github.com/tensorflow/tensorflow + # GIT_TAG ${TFLITE_TAG} + # GIT_SHALLOW TRUE) + # fetchcontent_getproperties(tflite) + # if(NOT tflite_POPULATED) + # fetchcontent_populate(tflite) + # add_subdirectory(${tflite_SOURCE_DIR}/tensorflow/lite/c + # ${tflite_BINARY_DIR}) + # endif() + # cmake-format: on + # + # However some enviroments don't allow internet access during cmake build, + # so the new approach is to store all the necessary TFLite dependencies as + # archives inside of the third_party directory. These archives are extracted + # below and built as sub-projects. The fetchcontent based snippet above can + # be used in the future to update the stored archives for the dependencies + # should TFlite version be updated, + message(STATUS "Building TFLite from local sources...") + # These are the paths of the stored TFLite dependencies, use the the + # fetchcontent snippet above, then look for messages like: "Performing + # download step (download, verify and extract) for 'xyz'" go to + # "xyz-source", Then archive the source into "xyz.tar.gz" and and add the + # variable "XYZ_SOURCE_DIR" pointing to "third_party/xyz" like below, also + # update the command line to extrac the archive below. - set(TFLITE_TAG "v2.17.0") + string( + CONCAT + EXTRACT_TF_UPDATE_DEPS_CMAKE_CONFIG + "cd ${CMAKE_CURRENT_SOURCE_DIR}/third_party;" + "tar -xzf tensorflow.tar.gz; " + "tar -xzf deps_cmake.tar.gz; " + "cp deps_cmake/abseil-cpp.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/cpuinfo.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/eigen.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/farmhash.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/fft2d.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/flatbuffers.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/gemmlowp.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/ml_dtypes.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/neon2sse.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/protobuf.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/ruy.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules; " + "cp deps_cmake/xnnpack.cmake ${CMAKE_CURRENT_SOURCE_DIR}/third_party/tensorflow/tensorflow/lite/tools/cmake/modules" + ) - message(STATUS "Fetching TFLite ${TFLITE_TAG}...") + execute_process(COMMAND bash -c "${EXTRACT_TF_UPDATE_DEPS_CMAKE_CONFIG}") + string( + CONCAT EXTRACT_DEPS_TO_BUILD + "cd ${CMAKE_CURRENT_SOURCE_DIR}/third_party; " + "tar -xzf abseil-cpp.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf eigen.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf neon2sse.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf xnnpack.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf ruy.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf farmhash.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf fft2d.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf gemmlowp.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf ml_dtypes.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf cpuinfo.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf protobuf.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf flatbuffers.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf fp16.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf pthreadpool.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf psimd.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf FXdiv.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf benchmark.tar.gz -C ${CMAKE_BINARY_DIR}; " + "tar -xzf googletest.tar.gz -C ${CMAKE_BINARY_DIR}") + + execute_process(COMMAND bash -c "${EXTRACT_DEPS_TO_BUILD}") + + set(FP16_SOURCE_DIR ${CMAKE_BINARY_DIR}/fp16) + set(PTHREADPOOL_SOURCE_DIR ${CMAKE_BINARY_DIR}/pthreadpool) + set(PSIMD_SOURCE_DIR ${CMAKE_BINARY_DIR}/psimd) + set(FXDIV_SOURCE_DIR ${CMAKE_BINARY_DIR}/FXdiv) + set(GOOGLEBENCHMARK_SOURCE_DIR ${CMAKE_BINARY_DIR}/benchmark) + set(GOOGLETEST_SOURCE_DIR ${CMAKE_BINARY_DIR}/googletest) + set(abseil-cpp_SOURCE_DIR ${CMAKE_BINARY_DIR}/abseil-cpp) + set(CPUINFO_SOURCE_DIR ${CMAKE_BINARY_DIR}/cpuinfo) + set(EIGEN_SOURCE_DIR ${CMAKE_BINARY_DIR}/eigen) + set(FARMHASH_SOURCE_DIR ${CMAKE_BINARY_DIR}/farmhash) + set(FFT2D_SOURCE_DIR ${CMAKE_BINARY_DIR}/fft2d) + set(FLATBUFFERS_SOURCE_DIR ${CMAKE_BINARY_DIR}/flatbuffers) + set(GEMMLOWP_SOURCE_DIR ${CMAKE_BINARY_DIR}/gemmlowp) + set(ML_DTYPES_SOURCE_DIR ${CMAKE_BINARY_DIR}/ml_dtypes) + set(PROTOBUF_SOURCE_DIR ${CMAKE_BINARY_DIR}/protobuf) + set(RUY_SOURCE_DIR ${CMAKE_BINARY_DIR}/ruy) + set(NEON2SSE_SOURCE_DIR ${CMAKE_BINARY_DIR}/neon2sse) + set(XNNPACK_SOURCE_DIR ${CMAKE_BINARY_DIR}/XNNPACK) # static linking makes life with TFLite much easier set(TFLITE_C_BUILD_SHARED_LIBS OFF) @@ -600,30 +690,17 @@ # Turn XNNPack on. set(TFLITE_ENABLE_XNNPACK ON) - fetchcontent_declare( - tflite - GIT_REPOSITORY https://github.com/tensorflow/tensorflow - GIT_TAG ${TFLITE_TAG} - GIT_SHALLOW TRUE) - - fetchcontent_getproperties(tflite) - if(NOT tflite_POPULATED) - fetchcontent_populate(tflite) - # Some of the subprojects (e.g. Eigen) are very noisy and emit status - # messages all the time. Temporary ignore status messages while adding - # this to silence it. Ugly but effective. - set(OLD_CMAKE_MESSAGE_LOG_LEVEL ${CMAKE_MESSAGE_LOG_LEVEL}) - set(CMAKE_MESSAGE_LOG_LEVEL WARNING) - add_subdirectory(${tflite_SOURCE_DIR}/tensorflow/lite/c - ${tflite_BINARY_DIR}) - set(CMAKE_MESSAGE_LOG_LEVEL ${OLD_CMAKE_MESSAGE_LOG_LEVEL}) - endif() + # Some of the subprojects (e.g. Eigen) are very noisy and emit status + # messages all the time. Temporary ignore status messages while adding this + # to silence it. Ugly but effective. + set(OLD_CMAKE_MESSAGE_LOG_LEVEL ${CMAKE_MESSAGE_LOG_LEVEL}) + set(CMAKE_MESSAGE_LOG_LEVEL WARNING) + add_subdirectory(third_party/tensorflow/tensorflow/lite/c) + set(CMAKE_MESSAGE_LOG_LEVEL ${OLD_CMAKE_MESSAGE_LOG_LEVEL}) # Disable some noisy warnings in tflite target_compile_options(tensorflow-lite PRIVATE -w) - - # tensorflowlite_c is implicitly declared by this FetchContent - include_directories(${tflite_SOURCE_DIR}) + include_directories(${AOM_ROOT}/third_party/tensorflow) include_directories("${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/include/") add_dependencies(aom_av1_common tensorflow-lite) endif()
diff --git a/third_party/FXdiv.tar.gz b/third_party/FXdiv.tar.gz new file mode 100644 index 0000000..9cf9c63 --- /dev/null +++ b/third_party/FXdiv.tar.gz Binary files differ
diff --git a/third_party/abseil-cpp.tar.gz b/third_party/abseil-cpp.tar.gz new file mode 100644 index 0000000..3289cf5 --- /dev/null +++ b/third_party/abseil-cpp.tar.gz Binary files differ
diff --git a/third_party/benchmark.tar.gz b/third_party/benchmark.tar.gz new file mode 100644 index 0000000..f0ca791 --- /dev/null +++ b/third_party/benchmark.tar.gz Binary files differ
diff --git a/third_party/cpuinfo.tar.gz b/third_party/cpuinfo.tar.gz new file mode 100644 index 0000000..56d08e7 --- /dev/null +++ b/third_party/cpuinfo.tar.gz Binary files differ
diff --git a/third_party/deps_cmake.tar.gz b/third_party/deps_cmake.tar.gz new file mode 100644 index 0000000..a03a629 --- /dev/null +++ b/third_party/deps_cmake.tar.gz Binary files differ
diff --git a/third_party/eigen.tar.gz b/third_party/eigen.tar.gz new file mode 100644 index 0000000..31e2134 --- /dev/null +++ b/third_party/eigen.tar.gz Binary files differ
diff --git a/third_party/farmhash.tar.gz b/third_party/farmhash.tar.gz new file mode 100644 index 0000000..98116af --- /dev/null +++ b/third_party/farmhash.tar.gz Binary files differ
diff --git a/third_party/fft2d.tar.gz b/third_party/fft2d.tar.gz new file mode 100644 index 0000000..28a17c1 --- /dev/null +++ b/third_party/fft2d.tar.gz Binary files differ
diff --git a/third_party/flatbuffers.tar.gz b/third_party/flatbuffers.tar.gz new file mode 100644 index 0000000..64813c4 --- /dev/null +++ b/third_party/flatbuffers.tar.gz Binary files differ
diff --git a/third_party/fp16.tar.gz b/third_party/fp16.tar.gz new file mode 100644 index 0000000..263715c --- /dev/null +++ b/third_party/fp16.tar.gz Binary files differ
diff --git a/third_party/gemmlowp.tar.gz b/third_party/gemmlowp.tar.gz new file mode 100644 index 0000000..27df7e0 --- /dev/null +++ b/third_party/gemmlowp.tar.gz Binary files differ
diff --git a/third_party/googletest.tar.gz b/third_party/googletest.tar.gz new file mode 100644 index 0000000..afad72d --- /dev/null +++ b/third_party/googletest.tar.gz Binary files differ
diff --git a/third_party/ml_dtypes.tar.gz b/third_party/ml_dtypes.tar.gz new file mode 100644 index 0000000..5a0cd2e --- /dev/null +++ b/third_party/ml_dtypes.tar.gz Binary files differ
diff --git a/third_party/neon2sse.tar.gz b/third_party/neon2sse.tar.gz new file mode 100644 index 0000000..96e307f --- /dev/null +++ b/third_party/neon2sse.tar.gz Binary files differ
diff --git a/third_party/protobuf.tar.gz b/third_party/protobuf.tar.gz new file mode 100644 index 0000000..6b3d8c0 --- /dev/null +++ b/third_party/protobuf.tar.gz Binary files differ
diff --git a/third_party/psimd.tar.gz b/third_party/psimd.tar.gz new file mode 100644 index 0000000..ce43a04 --- /dev/null +++ b/third_party/psimd.tar.gz Binary files differ
diff --git a/third_party/pthreadpool.tar.gz b/third_party/pthreadpool.tar.gz new file mode 100644 index 0000000..54dd9df --- /dev/null +++ b/third_party/pthreadpool.tar.gz Binary files differ
diff --git a/third_party/ruy.tar.gz b/third_party/ruy.tar.gz new file mode 100644 index 0000000..02673f1 --- /dev/null +++ b/third_party/ruy.tar.gz Binary files differ
diff --git a/third_party/tensorflow.tar.gz b/third_party/tensorflow.tar.gz new file mode 100644 index 0000000..8610154 --- /dev/null +++ b/third_party/tensorflow.tar.gz Binary files differ
diff --git a/third_party/xnnpack.tar.gz b/third_party/xnnpack.tar.gz new file mode 100644 index 0000000..7275e0a --- /dev/null +++ b/third_party/xnnpack.tar.gz Binary files differ