Merge "Handle 64b box sizes" into main
diff --git a/CMakeLists.txt b/CMakeLists.txt index ce831e6..e44cba0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -13,10 +13,12 @@ option(AVIFINFO_BUILD_TESTS "Build and enable tests (GoogleTest must be installed)" OFF) +option(AVIFINFO_BUILD_TOOLS "Build tools" OFF) # C library add_library(avifinfo avifinfo.c) +target_include_directories(avifinfo PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) # C++ tests @@ -26,11 +28,42 @@ enable_testing() add_executable(avifinfo_test tests/avifinfo_test.cc) - target_include_directories(avifinfo_test PRIVATE ${GTEST_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR}) + target_include_directories(avifinfo_test PRIVATE ${GTEST_INCLUDE_DIRS}) target_link_libraries(avifinfo_test PRIVATE ${GTEST_BOTH_LIBRARIES} avifinfo) add_test( NAME avifinfo_test COMMAND ${CMAKE_CURRENT_BINARY_DIR}/avifinfo_test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests) endif() + +# C++ tools + +if(AVIFINFO_BUILD_TOOLS) + find_package(Threads REQUIRED) # For aom + + # Clone and build libavif and its dependency aom. + include(ExternalProject) + set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) + ExternalProject_Add( + libavif + GIT_REPOSITORY https://github.com/AOMediaCodec/libavif + PATCH_COMMAND cd ${CMAKE_BINARY_DIR}/libavif-prefix/src/libavif/ext && + ./aom.cmd + UPDATE_DISCONNECTED 1 # Avoid building aom everytime. + # aom.cmd builds aom as static so libavif should be built as static too. + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} + -DAVIF_CODEC_AOM=ON -DAVIF_LOCAL_AOM=ON -DBUILD_SHARED_LIBS=OFF) + + add_executable(avifinfo_tool tools/avifinfo_tool.cc tests/avifinfo_fuzz.cc) + set_property(TARGET avifinfo_tool PROPERTY CXX_STANDARD 17) # for filesystem + target_include_directories(avifinfo_tool SYSTEM + PRIVATE ${EXTERNAL_INSTALL_LOCATION}/include) + # libavif's CMakeLists.txt uses a PRIVATE target_link_libraries() so + # avifinfo_tool must be linked to aom too (and thus also to pthread). + target_link_directories( + avifinfo_tool PRIVATE ${EXTERNAL_INSTALL_LOCATION}/lib + ${CMAKE_BINARY_DIR}/libavif-prefix/src/libavif/ext/aom/build.libavif) + target_link_libraries(avifinfo_tool PRIVATE avifinfo avif aom + Threads::Threads) + add_dependencies(avifinfo_tool libavif) +endif()
diff --git a/tools/avifinfo_tool.cc b/tools/avifinfo_tool.cc new file mode 100644 index 0000000..b16281c --- /dev/null +++ b/tools/avifinfo_tool.cc
@@ -0,0 +1,271 @@ +// Copyright (c) 2021, Alliance for Open Media. All rights reserved +// +// This source code is subject to the terms of the BSD 2 Clause License and +// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License +// was not distributed with this source code in the LICENSE file, you can +// obtain it at www.aomedia.org/license/software. If the Alliance for Open +// Media Patent License 1.0 was not distributed with this source code in the +// PATENTS file, you can obtain it at www.aomedia.org/license/patent. + +#include <algorithm> +#include <cstring> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <iterator> +#include <string> +#include <unordered_map> +#include <vector> + +#include "avifinfo.h" +#include "avif/avif.h" + +namespace { + +//------------------------------------------------------------------------------ + +std::string GetHelpStr() { + std::string str; + str += "Command line tool to compare libavif and libavifinfo results.\n"; + str += "Usage: avifparse [options] <directory>\n"; + str += "Options:\n"; + str += " -h, --help ...... Print this help\n"; + str += " --fast .......... Skip libavif decoding, only use libavifinfo\n"; + str += " --min-size ...... Find minimum size to extract features per file\n"; + str += " --validate ...... Check libavifinfo consistency on each file\n"; + return str; +} + +//------------------------------------------------------------------------------ + +// Decoding result. +struct Result { + bool success; // True if the 'features' were correctly decoded. + AvifInfoFeatures features; +}; + +// Decodes the AVIF at 'data' of 'data_size' bytes using libavif. +Result DecodeAvif(const uint8_t data[], size_t data_size) { + Result result; + avifImage* const image = avifImageCreateEmpty(); + avifDecoder* const decoder = avifDecoderCreate(); + decoder->strictFlags = AVIF_STRICT_DISABLED; + const avifResult status = + avifDecoderReadMemory(decoder, image, data, data_size); + avifDecoderDestroy(decoder); + if (status == AVIF_RESULT_OK) { + const uint32_t num_channels = + ((image->yuvFormat == AVIF_PIXEL_FORMAT_NONE) ? 0 + : (image->yuvFormat == AVIF_PIXEL_FORMAT_YUV400) ? 1 + : 3) + + ((image->alphaPlane != nullptr) ? 1 : 0); + result = {true, {image->width, image->height, image->depth, num_channels}}; + } else { + result = {false}; + } + avifImageDestroy(image); + return result; +} + +// Parses the AVIF at 'data' of 'data_size' bytes using libavifinfo. +Result ParseAvif(const uint8_t data[], size_t data_size) { + Result result; + const AvifInfoStatus status = AvifInfoGetWithSize( + data, data_size, &result.features, /*file_size=*/data_size); + result.success = (status == kAvifInfoOk); + return result; +} + +// Same as above but also returns the 'min_data_size' for which 'data' can be +// successfully parsed. +Result ParseAvifForSize(const uint8_t data[], size_t data_size, + size_t& min_data_size) { + const Result result = ParseAvif(data, data_size); + if (!result.success) { + min_data_size = data_size; + return result; + } + min_data_size = 1; + size_t max_data_size = data_size; + while (min_data_size < max_data_size) { + const size_t middle = (min_data_size + max_data_size) / 2; + if (AvifInfoGetWithSize(data, middle, nullptr, /*file_size=*/data_size) == + kAvifInfoOk) { + max_data_size = middle; + } else { + min_data_size = middle + 1; + } + } + return result; +} + +// Reuses the fuzz target for easy library validation. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t data_size); + +// Aggregated stats about the decoded/parsed AVIF files. +struct Stats { + uint32_t num_files_invalid_at_decode = 0; + uint32_t num_files_invalid_at_parse = 0; + uint32_t num_files_invalid_at_both = 0; + std::unordered_map<size_t, uint32_t> min_size_to_count; +}; + +//------------------------------------------------------------------------------ + +// Recursively adds all files at 'path' to 'file_paths'. +void FindFiles(const std::string& path, std::vector<std::string>& file_paths) { + if (std::filesystem::is_directory(path)) { + for (const std::filesystem::directory_entry& entry : + std::filesystem::directory_iterator(path)) { + FindFiles(entry.path(), file_paths); + } + } else { + file_paths.emplace_back(path); + } +} + +// Find the longest common prefix of all input 'paths'. +std::string FindCommonLongestPrefix(const std::vector<std::string>& paths) { + std::string prefix = paths.empty() ? "" : paths.front(); + for (const std::string& path : paths) { + const auto mismatch = + std::mismatch(prefix.begin(), prefix.end(), path.begin(), path.end()); + prefix = prefix.substr(0, std::distance(prefix.begin(), mismatch.first)); + } + return std::filesystem::path(prefix).remove_filename().string(); +} + +//------------------------------------------------------------------------------ + +// Uses libavifinfo to extract the features of an AVIF file stored in 'data' at +// 'path'. The AVIF file is 'data_size'-byte long. +void ParseFile(const std::string& path, const uint8_t* data, size_t data_size, + Stats& stats) { + const Result parse = ParseAvif(data, data_size); + if (!parse.success) { + ++stats.num_files_invalid_at_parse; + std::cout << "parsing failure for " << path << std::endl; + } +} + +// Uses libavif then libavifinfo to extract the features of an AVIF file. +void DecodeAndParseFile(const std::string& path, const uint8_t* data, + size_t data_size, Stats& stats) { + const Result decode = DecodeAvif(data, data_size); + const Result parse = ParseAvif(data, data_size); + if (!decode.success) ++stats.num_files_invalid_at_decode; + if (!parse.success) ++stats.num_files_invalid_at_parse; + if (!decode.success && !parse.success) ++stats.num_files_invalid_at_both; + + if (!parse.success || + (decode.success && + (decode.features.width != parse.features.width || + decode.features.height != parse.features.height || + decode.features.bit_depth != parse.features.bit_depth || + decode.features.num_channels != parse.features.num_channels))) { + if (decode.success && parse.success) { + std::cout << "decoded " << decode.features.width << "x" + << decode.features.height << "," << decode.features.bit_depth + << "b*" << decode.features.num_channels << " / " + << "parsed " << parse.features.width << "x" + << parse.features.height << "," << parse.features.bit_depth + << "b*" << parse.features.num_channels; + } else { + std::cout << "decoding " << (decode.success ? "success" : "failure") + << " / parsing " << (parse.success ? "success" : "failure"); + } + std::cout << " for " << path << std::endl; + } +} + +// Returns the minimum number of bytes of AVIF 'data' for features to be +// extracted. +void FindMinSizeOfFile(const std::string& path, const uint8_t* data, + size_t data_size, Stats& stats) { + size_t min_size; + const Result parse = ParseAvifForSize(data, data_size, min_size); + if (parse.success) { + ++stats.min_size_to_count[min_size]; + } else { + ++stats.num_files_invalid_at_parse; + } +} + +// Checks the consistency of libavifinfo over an AVIF file. +void ValidateFile(const std::string& path, const uint8_t* data, + size_t data_size) { + if (LLVMFuzzerTestOneInput(data, data_size) != 0) { + std::cout << "validation failed for " << path << std::endl; + } +} + +} // namespace + +//------------------------------------------------------------------------------ + +int main(int argc, char** argv) { + std::vector<std::string> file_paths; + bool only_parse = false; + bool find_min_size = false; + bool validate = false; + + for (int arg = 1; arg < argc; ++arg) { + if (!std::strcmp(argv[arg], "-h")) { + std::cout << GetHelpStr(); + return 0; + } else if (!std::strcmp(argv[arg], "--fast")) { + only_parse = true; + } else if (!std::strcmp(argv[arg], "--min-size")) { + find_min_size = true; + only_parse = true; + } else if (!std::strcmp(argv[arg], "--validate")) { + validate = true; + } else { + FindFiles(argv[arg], file_paths); + } + } + if (file_paths.empty()) { + std::cerr << "No input specified" << std::endl; + return 1; + } + std::cout << "Found " << file_paths.size() << " files" << std::endl; + const std::string prefix = FindCommonLongestPrefix(file_paths); + for (std::string& file_path : file_paths) { + file_path = file_path.substr(prefix.size()); + } + + Stats stats; + for (const std::string& file_path : file_paths) { + std::vector<uint8_t> bytes(std::filesystem::file_size(prefix + file_path)); + std::ifstream file(prefix + file_path, std::ios::binary); + file.read(reinterpret_cast<char*>(bytes.data()), bytes.size()); + if (find_min_size) { + FindMinSizeOfFile(file_path, bytes.data(), bytes.size(), stats); + } else if (only_parse) { + ParseFile(file_path, bytes.data(), bytes.size(), stats); + } else { + DecodeAndParseFile(file_path, bytes.data(), bytes.size(), stats); + } + if (validate) { + ValidateFile(file_path, bytes.data(), bytes.size()); + } + } + + std::cout << stats.num_files_invalid_at_parse << " files failed to parse" + << std::endl; + if (!only_parse) { + std::cout << stats.num_files_invalid_at_decode << " files failed to decode" + << std::endl; + std::cout << stats.num_files_invalid_at_both + << " files failed to parse and decode" << std::endl; + } + + if (find_min_size) { + std::cout << std::endl; + for (const auto& it : stats.min_size_to_count) { + std::cout << it.second << " files need " << it.first + << " bytes to extract features" << std::endl; + } + } + return 0; +}