| #!/usr/bin/env python | |
| ## Copyright (c) 2021, Alliance for Open Media. All rights reserved | |
| ## | |
| ## This source code is subject to the terms of the BSD 3-Clause Clear License and the | |
| ## Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear License was | |
| ## not distributed with this source code in the LICENSE file, you can obtain it | |
| ## at aomedia.org/license/software-license/bsd-3-c-c/. If the Alliance for Open Media Patent | |
| ## License 1.0 was not distributed with this source code in the PATENTS file, you | |
| ## can obtain it at aomedia.org/license/patent-license/. | |
| ## | |
| __author__ = "maggie.sun@intel.com, ryanlei@meta.com" | |
| import os | |
| import re | |
| import sys | |
| import argparse | |
| import subprocess | |
| from Utils import CreateNewSubfolder | |
| from Config import WorkPath, FrameNum, EnableParallelGopEncoding, GOP_SIZE | |
| root_path = '/project/tenjin/fba/design/ryanlei/AV2-CTC/AV2-CTC-v9.0.0/' | |
| test_path = os.path.join(root_path, 'test') | |
| dec_path = os.path.join(test_path, 'decodedYUVs') | |
| #ra_dec_log_path = os.path.join(test_path, 'RA_decLogs') | |
| dec_log_path = os.path.join(test_path, 'decLogs') | |
| ra_cmd_log_file = os.path.join(test_path, "AV2CTC_TestCmd_RA.log") | |
| as_cmd_log_file = os.path.join(test_path, "AV2CTC_TestCmd_AS.log") | |
| decoder = "%s/bin/aomdec-v9.0.0-rc1" % root_path | |
| ra_dec_error_log = os.path.join(test_path, "ra_decode_error.log") | |
| as_dec_error_log = os.path.join(test_path, "as_decode_error.log") | |
| updated_ra_cmd_log = os.path.join(test_path, "AV2CTC_TestCmd_Update_RA.log") | |
| updated_as_cmd_log = os.path.join(test_path, "AV2CTC_TestCmd_Update_AS.log") | |
| def get_expected_frames(cfg): | |
| if cfg in ['LD', 'AI', 'STILL']: | |
| total_frame = FrameNum[cfg] | |
| elif cfg in ['RA', 'AS']: | |
| if EnableParallelGopEncoding: | |
| total_frame = GOP_SIZE | |
| else: | |
| total_frame = FrameNum[cfg] | |
| return total_frame | |
| def check_decode_log(dec_log_file, cfg): | |
| if not os.path.isfile(dec_log_file): | |
| print("%s does not exists\n" % dec_log_file) | |
| return False | |
| else: | |
| dec_log = open(dec_log_file, 'r') | |
| for line in dec_log: | |
| m = re.search(r"(\d+) decoded frames", line) | |
| if m: | |
| frames = int(m.group(1)) | |
| #print ("%s has %d frames" % (dec_log_file, frames)) | |
| if (frames == get_expected_frames(cfg)): | |
| return True | |
| else: | |
| print("%s has %d frames\n" % (dec_log_file, frames)) | |
| return False | |
| dec_log.close() | |
| return False | |
| def run_decode(cmd_log_file): | |
| CreateNewSubfolder(WorkPath, dec_log_path) | |
| cmd_log = open(cmd_log_file, 'r') | |
| for line in cmd_log: | |
| m = re.search(r"-o (.*).obu",line) | |
| if m: | |
| bitstream = m.group(1) | |
| output_file = bitstream + ".obu" | |
| video = bitstream.split("/")[-1] | |
| dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") | |
| dec_output = os.path.join(dec_path, video + ".y4m") | |
| #print("bitstrams file is %s\n" % output_file) | |
| #print("decoding log file is %s\n" % dec_log) | |
| args = " --codec=av1 --summary -o %s %s" % (dec_output, output_file) | |
| dec_cmd = decoder + args + "> %s 2>&1"%dec_log | |
| #print("=== decoding %s \n" %video) | |
| #print(dec_cmd) | |
| #os.system(dec_cmd) | |
| cmd = "ENABLE_CONTAINER_CONFIG=1 grid run -autokill 0 -C modeling_c9 -r+ RAM/0" + " '" + dec_cmd + "'" | |
| #print(cmd) | |
| subprocess.call(cmd, shell=True) | |
| cmd_log.close() | |
| def check_decoding(cmd_log_file, dec_error_log): | |
| cmd_log = open(cmd_log_file, 'r') | |
| error_list = [] | |
| error_log = open(dec_error_log, 'wt') | |
| for line in cmd_log: | |
| m = re.search(r"-o (.*).obu",line) | |
| if m: | |
| bitstream = m.group(1) | |
| output_file = bitstream + ".obu" | |
| video = bitstream.split("/")[-1] | |
| dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") | |
| dec_output = os.path.join(dec_path, video + ".y4m") | |
| cfg = get_config(output_file) | |
| #print("checking %s for %s" % (dec_log, cfg)) | |
| if check_decode_log(dec_log, cfg) == False: | |
| print("%s decoding error" % video) | |
| error_log.write(output_file + "\n") | |
| error_list.append(video) | |
| if os.path.exists(dec_output): | |
| del_cmd = "rm %s" % dec_output | |
| #print("=== deleting %s \n" %video) | |
| os.system(del_cmd) | |
| #input("press any key to continue") | |
| cmd_log.close() | |
| error_log.close() | |
| return error_list | |
| def filter_cmd_log(cmd_log_file, updated_cmd_log_file, error_list): | |
| cmd_log = open(cmd_log_file, 'r') | |
| updated_cmd_log = open(updated_cmd_log_file, 'wt') | |
| start_copy = False | |
| for line in cmd_log: | |
| if start_copy: | |
| updated_cmd_log.write(line) | |
| m = re.search(r"============== (.*) Job Start =================", line) | |
| if m: | |
| video = m.group(1) | |
| if video in error_list: | |
| start_copy = True | |
| updated_cmd_log.write(line) | |
| else: | |
| video = video.replace("_AS_3840x2160_Preset", "_AS_Preset") | |
| if video in error_list: | |
| start_copy = True | |
| updated_cmd_log.write(line) | |
| m = re.search(r"============== (.*) Job End =================", line) | |
| if m: | |
| video = m.group(1) | |
| if video in error_list: | |
| start_copy = False | |
| else: | |
| video = video.replace("_AS_3840x2160_Preset", "_AS_Preset") | |
| if video in error_list: | |
| start_copy = False | |
| cmd_log.close() | |
| updated_cmd_log.close() | |
| def get_config(file_name): | |
| if ('_LD_' in file_name): | |
| return 'LD' | |
| elif ('_AS_' in file_name): | |
| return 'AS' | |
| elif ('_AI_' in file_name): | |
| return 'AI' | |
| elif ('_STILL_' in file_name): | |
| return 'STILL' | |
| elif ('_RA_' in file_name): | |
| return 'RA' | |
| else: | |
| print("unsupported configuration") | |
| return '' | |
| def check_ld_decoding(cmd_log_file, dec_error_log): | |
| cmd_log = open(cmd_log_file, 'r') | |
| error_list = [] | |
| error_log = open(dec_error_log, 'wt') | |
| for line in cmd_log: | |
| m = re.search(r"=* (.*) Job Start", line) | |
| #m = re.search(r"-o (.*).obu",line) | |
| if m: | |
| bitstream = m.group(1) | |
| output_file = bitstream + ".obu" | |
| video = bitstream.split("/")[-1] | |
| cfg = get_config(video) | |
| if cfg in ['RA', 'AS']: | |
| dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") | |
| elif cfg in ['LD', 'AI', 'STILL']: | |
| dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") | |
| dec_output = os.path.join(dec_path, video + ".y4m") | |
| if check_decode_log(dec_log, cfg) == False: | |
| #print(line) | |
| print("%s decoding error" % video) | |
| error_log.write(output_file + "\n") | |
| error_list.append(video) | |
| if os.path.exists(dec_output): | |
| del_cmd = "rm %s" % dec_output | |
| #print("=== deleting %s \n" %video) | |
| os.system(del_cmd) | |
| #input("press any key to continue") | |
| cmd_log.close() | |
| error_log.close() | |
| return error_list | |
| ###################################### | |
| # main | |
| ###################################### | |
| if __name__ == "__main__": | |
| #1. trigger decoding process, needed for RA | |
| #run_decode(cmd_log_file) | |
| #2. check decoding errors | |
| #error_list = check_decoding(ra_cmd_log_file, ra_dec_error_log) | |
| #print("There are %d missing encodings" % len(error_list)) | |
| #filter_cmd_log(ra_cmd_log_file, updated_ra_cmd_log, error_list) | |
| error_list = check_decoding(as_cmd_log_file, as_dec_error_log) | |
| print("There are %d missing encodings" % len(error_list)) | |
| filter_cmd_log(as_cmd_log_file, updated_as_cmd_log, error_list) |