update CTC test script 1. add more logging files for decoding and vmaf execution 2. add utilities script for checking RA parallel gop encoding status
diff --git a/tools/convexhull_framework/src/AV2CTCTest.py b/tools/convexhull_framework/src/AV2CTCTest.py index 783d36e..0a5b1be 100644 --- a/tools/convexhull_framework/src/AV2CTCTest.py +++ b/tools/convexhull_framework/src/AV2CTCTest.py
@@ -46,7 +46,15 @@ def setupWorkFolderStructure(): global Path_Bitstreams, Path_DecodedYuv, Path_QualityLog, Path_TestLog,\ - Path_CfgFiles, Path_TimingLog, Path_EncLog, Path_CmdLog + Path_CfgFiles, Path_TimingLog, Path_EncLog, Path_DecLog, Path_CmdLog,\ + Path_VmafLog + abs_path = os.path.abspath(WorkPath) + print(abs_path) + if(os.path.exists(abs_path) == False): + cmd = "mkscratchdir " + abs_path + print(cmd) + subprocess.call(cmd, shell=True) + Path_Bitstreams = CreateNewSubfolder(WorkPath, "bitstreams") Path_DecodedYuv = CreateNewSubfolder(WorkPath, "decodedYUVs") Path_QualityLog = CreateNewSubfolder(WorkPath, "qualityLogs") @@ -55,12 +63,16 @@ Path_TimingLog = CreateNewSubfolder(WorkPath, "perfLogs") Path_EncLog = CreateNewSubfolder(WorkPath, "encLogs") Path_CmdLog = CreateNewSubfolder(WorkPath, "cmdLogs") + Path_DecLog = CreateNewSubfolder(WorkPath, "decLogs") + Path_VmafLog = CreateNewSubfolder(WorkPath, "vmafLogs") + ############################################################################### ######### Major Functions ##################################################### def CleanUp_workfolders(): folders = [Path_Bitstreams, Path_DecodedYuv, Path_QualityLog, - Path_TestLog, Path_CfgFiles, Path_TimingLog, Path_EncLog] + Path_TestLog, Path_CfgFiles, Path_TimingLog, Path_EncLog, + Path_DecLog, Path_VmafLog] for folder in folders: Cleanfolder(folder) @@ -83,12 +95,12 @@ Utils.Logger.info("start decode file %s" % os.path.basename(bsFile)) #decode decodedYUV = Decode(clip, method, test_cfg, codec, bsFile, Path_DecodedYuv, Path_TimingLog, - False, LogCmdOnly) + False, Path_DecLog, LogCmdOnly) #calcualte quality distortion Utils.Logger.info("start quality metric calculation") CalculateQualityMetric(clip.file_path, FrameNum[test_cfg], decodedYUV, - clip.fmt, clip.width, clip.height, clip.bit_depth, - Path_QualityLog, LogCmdOnly) + clip.fmt, clip.width, clip.height, clip.bit_depth, + Path_QualityLog, Path_VmafLog, LogCmdOnly) if SaveMemory: DeleteFile(decodedYUV, LogCmdOnly) Utils.Logger.info("finish running encode with QP %d" % (QP)) @@ -150,12 +162,12 @@ method, codec, test_cfg, preset, QP) decodedYUV = Decode(clip, method, test_cfg, codec, bsfile, Path_DecodedYuv, Path_TimingLog, - False, LogCmdOnly) + False, Path_DecLog, LogCmdOnly) #calcualte quality distortion Utils.Logger.info("start quality metric calculation") CalculateQualityMetric(clip.file_path, FrameNum[test_cfg], decodedYUV, clip.fmt, clip.width, clip.height, clip.bit_depth, - Path_QualityLog, LogCmdOnly) + Path_QualityLog, Path_VmafLog, LogCmdOnly) if SaveMemory: DeleteFile(decodedYUV, LogCmdOnly) Utils.Logger.info("finish running encode with QP %d" % (QP))
diff --git a/tools/convexhull_framework/src/CalcQtyWithVmafTool.py b/tools/convexhull_framework/src/CalcQtyWithVmafTool.py index 5aa327e..9a950c6 100644 --- a/tools/convexhull_framework/src/CalcQtyWithVmafTool.py +++ b/tools/convexhull_framework/src/CalcQtyWithVmafTool.py
@@ -120,10 +120,17 @@ file = os.path.join(path, filename) return file +def GetVMAFExecLogFile(recfile, path): + filename = GetShortContentName(recfile, False) + '_vmafexec.log' + file = os.path.join(path, filename) + return file + ################################################################################ ##################### Exposed Functions ######################################## -def VMAF_CalQualityMetrics(origfile, recfile, logfilePath, LogCmdOnly=False): - vmaf_log = GetVMAFLogFile(recfile, logfilePath) +def VMAF_CalQualityMetrics(origfile, recfile, QualityLogPath, VmafLogPath, LogCmdOnly=False): + vmaf_log = GetVMAFLogFile(recfile, QualityLogPath) + vmafexec_log = GetVMAFLogFile(recfile, VmafLogPath) + args = " -r %s -d %s -q --threads 4 -o %s" \ % (origfile, recfile, vmaf_log) @@ -134,7 +141,7 @@ else: args += " --aom_ctc v1.0" - cmd = VMAF + args + cmd = VMAF + args + "> %s 2>&1"%vmafexec_log ExecuteCmd(cmd, LogCmdOnly) def VMAF_GatherQualityMetrics(recfile, logfilePath):
diff --git a/tools/convexhull_framework/src/CalculateQualityMetrics.py b/tools/convexhull_framework/src/CalculateQualityMetrics.py index 4dc32ff..23c9208 100644 --- a/tools/convexhull_framework/src/CalculateQualityMetrics.py +++ b/tools/convexhull_framework/src/CalculateQualityMetrics.py
@@ -21,9 +21,9 @@ logger = logging.getLogger(loggername) def CalculateQualityMetric(src_file, framenum, reconYUV, fmt, width, height, - bit_depth, logfilePath, LogCmdOnly=False): + bit_depth, QualityLogPath, VmafLogPath, LogCmdOnly=False): Utils.CmdLogger.write("::Quality Metrics\n") - VMAF_CalQualityMetrics(src_file, reconYUV, logfilePath, LogCmdOnly) + VMAF_CalQualityMetrics(src_file, reconYUV, QualityLogPath, VmafLogPath, LogCmdOnly) def GatherQualityMetrics(reconYUV, logfilePath): qresult, per_frame_log = VMAF_GatherQualityMetrics(reconYUV, logfilePath)
diff --git a/tools/convexhull_framework/src/CheckRAEncoding.py b/tools/convexhull_framework/src/CheckRAEncoding.py new file mode 100644 index 0000000..c3a4fe1 --- /dev/null +++ b/tools/convexhull_framework/src/CheckRAEncoding.py
@@ -0,0 +1,125 @@ +#!/usr/bin/env python +## Copyright (c) 2021, Alliance for Open Media. All rights reserved +## +## This source code is subject to the terms of the BSD 3-Clause Clear License and the +## Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear License was +## not distributed with this source code in the LICENSE file, you can obtain it +## at aomedia.org/license/software-license/bsd-3-c-c/. If the Alliance for Open Media Patent +## License 1.0 was not distributed with this source code in the PATENTS file, you +## can obtain it at aomedia.org/license/patent-license/. +## +__author__ = "maggie.sun@intel.com, ryanlei@meta.com" + +import os +import re +import sys +import argparse +import subprocess + + +root_path = '/project/tenjin/fba/design/ryanlei/AV2-CTC/AV2-CTC-v7.0.0-new/' +test_path = os.path.join(root_path, 'test') +dec_path = os.path.join(test_path, 'decodedYUVs') +dec_log_path = os.path.join(test_path, 'RA_decLogs') +cmd_log_file = os.path.join(test_path, "AV2CTC_TestCmd.log") +decoder = "%s/bin/aomdec-v7.0.0" % root_path +dec_error_log = os.path.join(test_path, "decode_error.log") +expected_frames = 65 +updated_cmd_log = os.path.join(test_path, "AV2CTC_TestCmd_Update.log") + +def check_decode_log(dec_log_file): + dec_log = open(dec_log_file, 'r') + for line in dec_log: + m = re.search(r"(\d+) decoded frames", line) + if m: + frames = int(m.group(1)) + if (frames == expected_frames): + return True + + dec_log.close() + return False + +def run_decode(cmd_log_file): + cmd_log = open(cmd_log_file, 'r') + for line in cmd_log: + m = re.search(r"-o (.*).obu",line) + if m: + bitstream = m.group(1) + output_file = bitstream + ".obu" + video = bitstream.split("/")[-1] + dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") + dec_output = os.path.join(dec_path, video + ".y4m") + #print("bitstrams file is %s\n" % output_file) + #print("decoding log file is %s\n" % dec_log) + args = " --codec=av1 --summary -o %s %s" % (dec_output, output_file) + dec_cmd = decoder + args + "> %s 2>&1"%dec_log + #print("=== decoding %s \n" %video) + #print(dec_cmd) + #os.system(dec_cmd) + cmd = "ENABLE_CONTAINER_CONFIG=1 grid run -autokill 0 -C modeling_c9 -r+ RAM/0" + " '" + dec_cmd + "'" + #print(cmd) + subprocess.call(cmd, shell=True) + cmd_log.close() + +def check_decoding(): + cmd_log = open(cmd_log_file, 'r') + error_list = [] + error_log = open(dec_error_log, 'wt') + for line in cmd_log: + m = re.search(r"-o (.*).obu",line) + if m: + bitstream = m.group(1) + output_file = bitstream + ".obu" + video = bitstream.split("/")[-1] + dec_log = os.path.join(dec_log_path, video + "_DecLog.txt") + dec_output = os.path.join(dec_path, video + ".y4m") + + if check_decode_log(dec_log) == False: + print("%s decoding error" % video) + error_log.write(output_file + "\n") + error_list.append(video) + + if os.path.exists(dec_output): + del_cmd = "rm %s" % dec_output + #print("=== deleting %s \n" %video) + os.system(del_cmd) + #input("press any key to continue") + cmd_log.close() + error_log.close() + return error_list + + +def filter_cmd_log(cmd_log_file, updated_cmd_log_file, error_list): + cmd_log = open(cmd_log_file, 'r') + updated_cmd_log = open(updated_cmd_log_file, 'wt') + start_copy = False + + for line in cmd_log: + if start_copy: + updated_cmd_log.write(line) + + m = re.search(r"============== (.*) Job Start =================", line) + if m: + video = m.group(1) + if video in error_list: + start_copy = True + updated_cmd_log.write(line) + + m = re.search(r"============== (.*) Job End =================", line) + if m: + video = m.group(1) + if video in error_list: + start_copy = False + + cmd_log.close() + updated_cmd_log.close() + + +###################################### +# main +###################################### +if __name__ == "__main__": + #run_decode(cmd_log_file) + error_list = check_decoding() + #print(error_list) + filter_cmd_log(cmd_log_file, updated_cmd_log, error_list) \ No newline at end of file
diff --git a/tools/convexhull_framework/src/ConvexHullTest.py b/tools/convexhull_framework/src/ConvexHullTest.py index fd15e70..aff1dd4 100755 --- a/tools/convexhull_framework/src/ConvexHullTest.py +++ b/tools/convexhull_framework/src/ConvexHullTest.py
@@ -35,7 +35,7 @@ def setupWorkFolderStructure(): global Path_Bitstreams, Path_DecodedYuv, Path_UpScaleYuv, Path_DnScaleYuv, \ Path_QualityLog, Path_TestLog, Path_CfgFiles, Path_DecUpScaleYuv, Path_PerfLog, \ - Path_EncLog, Path_CmdLog + Path_EncLog, Path_DecLog, Path_VmafLog, Path_CmdLog Path_Bitstreams = CreateNewSubfolder(WorkPath, "bitstreams") Path_DecodedYuv = CreateNewSubfolder(WorkPath, "decodedYUVs") Path_UpScaleYuv = CreateNewSubfolder(WorkPath, "upscaledYUVs") @@ -46,13 +46,16 @@ Path_CfgFiles = CreateNewSubfolder(WorkPath, "configFiles") Path_PerfLog = CreateNewSubfolder(WorkPath, "perfLogs") Path_EncLog = CreateNewSubfolder(WorkPath, "encLogs") + Path_DecLog = CreateNewSubfolder(WorkPath, "decLogs") Path_CmdLog = CreateNewSubfolder(WorkPath, "cmdLogs") + Path_VmafLog = CreateNewSubfolder(WorkPath, "vmafLogs") ############################################################################### ######### Major Functions ##################################################### def CleanUp_workfolders(): folders = [Path_DnScaleYuv, Path_Bitstreams, Path_DecodedYuv, Path_QualityLog, - Path_TestLog, Path_CfgFiles, Path_PerfLog, Path_EncLog] + Path_TestLog, Path_CfgFiles, Path_PerfLog, Path_EncLog, Path_DecLog, + Path_VmafLog] if not KeepUpscaledOutput: folders += [Path_UpScaleYuv, Path_DecUpScaleYuv] @@ -89,13 +92,13 @@ ds_clip, 'AS', QP, FrameNum['AS'], clip.width, clip.height, Path_Bitstreams, Path_DecodedYuv, Path_DecUpScaleYuv, - Path_CfgFiles, Path_PerfLog, Path_EncLog, upScalAlgo, - ScaleMethod, SaveMemory, LogCmdOnly) + Path_CfgFiles, Path_PerfLog, Path_EncLog, Path_DecLog, + upScalAlgo, ScaleMethod, SaveMemory, LogCmdOnly) #calcualte quality distortion Utils.Logger.info("start quality metric calculation") CalculateQualityMetric(clip.file_path, FrameNum['AS'], reconyuv, clip.fmt, clip.width, clip.height, - clip.bit_depth, Path_QualityLog, LogCmdOnly) + clip.bit_depth, Path_QualityLog, Path_VmafLog, LogCmdOnly) if SaveMemory: DeleteFile(reconyuv, LogCmdOnly) if LogCmdOnly:
diff --git a/tools/convexhull_framework/src/EncDecUpscale.py b/tools/convexhull_framework/src/EncDecUpscale.py index e897eae..be64cdd 100644 --- a/tools/convexhull_framework/src/EncDecUpscale.py +++ b/tools/convexhull_framework/src/EncDecUpscale.py
@@ -14,9 +14,9 @@ from VideoEncoder import VideoEncode from VideoDecoder import VideoDecode from VideoScaler import UpScaling, GetDownScaledOutFile, GetUpScaledOutFile -from Config import SUFFIX, LoggerName, EnableParallelGopEncoding +from Config import SUFFIX, LoggerName, EnableParallelGopEncoding, GOP_SIZE from Utils import GetShortContentName, Clip, GetEncLogFile, GetDecPerfFile, \ - GetEncPerfFile, DeleteFile + GetEncPerfFile, DeleteFile, GetDecLogFile import logging subloggername = "EncDecUpscale" @@ -56,24 +56,28 @@ enc_log, start_frame, LogCmdOnly) return bsfile -def Decode(clip, method, test_cfg, codec, bsfile, path, perf_path, decode_to_yuv, LogCmdOnly=False): +def Decode(clip, method, test_cfg, codec, bsfile, path, perf_path, decode_to_yuv, + log_path, LogCmdOnly=False): decodedfile = GetDecodedFile(bsfile, path, decode_to_yuv) dec_perf = GetDecPerfFile(bsfile, perf_path) + dec_log = GetDecLogFile(bsfile, log_path) + #call VideoDecoder to do the decoding - VideoDecode(clip, method, test_cfg, codec, bsfile, decodedfile, dec_perf, decode_to_yuv, LogCmdOnly) + VideoDecode(clip, method, test_cfg, codec, bsfile, decodedfile, dec_perf, + decode_to_yuv, dec_log, LogCmdOnly) return decodedfile def Run_EncDec_Upscale(method, codec, preset, clip, test_cfg, QP, num, outw, outh, path_bs, path_decoded, path_upscaled, path_cfg, - path_perf, path_enc_log, upscale_algo, scale_method, save_memory, - LogCmdOnly = False): + path_perf, path_enc_log, path_dec_log, + upscale_algo, scale_method, save_memory, LogCmdOnly = False): logger.info("%s %s start encode file %s with QP = %d" % (method, codec, clip.file_name, QP)) bsFile = Encode(method, codec, preset, clip, test_cfg, QP, num, path_bs, path_perf, path_enc_log, 0, LogCmdOnly) logger.info("start decode file %s" % os.path.basename(bsFile)) decodedYUV = Decode(clip, method, test_cfg, codec, bsFile, path_decoded, path_perf, False, - LogCmdOnly) + path_dec_log, LogCmdOnly) logger.info("start upscale file %s" % os.path.basename(decodedYUV)) #hard code frame rate to 0 before upscaling. #TODO: change to real frame rate after decoder fix the issue
diff --git a/tools/convexhull_framework/src/Launch.py b/tools/convexhull_framework/src/Launch.py new file mode 100644 index 0000000..23295ce --- /dev/null +++ b/tools/convexhull_framework/src/Launch.py
@@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import re +import sys +import subprocess +from Config import WorkPath + +###################################### +# main +###################################### +if __name__ == "__main__": + cmd_log_file = sys.argv[1] + curr_path = os.getcwd() + Path_CmdLog = os.path.join(WorkPath, "cmdLogs") + cmd_log_path = os.path.abspath(Path_CmdLog) + os.chdir(WorkPath) + + cmd_log = open(cmd_log_file, "rt") + + width = 0; height = 0; job_name = None; job_file = None + for line in cmd_log: + if line == "\n": + continue + + m = re.search(r"=+\s+(.*)\s+Job\s+Start\s+=+", line) + if m: + job_name = m.group(1) + #print(job_name) + job_file_name = os.path.join(Path_CmdLog, job_name + ".sh") + job_file = open(job_file_name, 'wt') + job_file.write("#!/bin/bash\n\n") + m = re.search(r"_(\d+)x(\d+)", job_name) + if m: + width = int(m.group(1)) + height = int(m.group(2)) + continue + + m = re.search(r"::", line) + if m: + continue + + m = re.search(r"=+\s+(.*)\s+Job\s+End\s+=+",line) + if m: + job_file.close() + os.chmod(job_file_name, 0o755) + extra_memory = "0" + max_wh = max(width, height) + if max_wh >= 3840: + extra_memory = "4500" + + width = 0; height = 0; job_name = None; job_file = None + #cmd = "nc run -C nolic_batch -r+ RAM/" + extra_memory + " " + job_file_name + #cmd = "nc run -C modeling -r+ RAM/" + extra_memory + " " + job_file_name + #cmd = "nc run -autokill 0 -C modeling -r+ \"osversion>=8\" -r+ RAM/" + extra_memory + " " + job_file_name + cmd = "ENABLE_CONTAINER_CONFIG=1 grid run -autokill 0 -C modeling_c9 -r+ RAM/" + extra_memory + " " + job_file_name + print(cmd) + subprocess.call(cmd, shell=True) + else: + job_file.write(line) + job_file.write("wait\n") + + cmd_log.close() \ No newline at end of file
diff --git a/tools/convexhull_framework/src/Utils.py b/tools/convexhull_framework/src/Utils.py index 6f50e5a..cd6f727 100755 --- a/tools/convexhull_framework/src/Utils.py +++ b/tools/convexhull_framework/src/Utils.py
@@ -194,6 +194,15 @@ filename = GetShortContentName(bsfile, False) + '_EncLog.txt' return os.path.join(logpath, filename) +def GetDecLogFile(bsfile, logpath): + filename = GetShortContentName(bsfile, False) + '_DecLog.txt' + return os.path.join(logpath, filename) + +def GetVmafLogFile(bsfile, logpath): + filename = GetShortContentName(bsfile, False) + '_VmafLog.txt' + return os.path.join(logpath, filename) + + def parseY4MHeader(y4m): """ Parse y4m information from its header.
diff --git a/tools/convexhull_framework/src/VideoDecoder.py b/tools/convexhull_framework/src/VideoDecoder.py index cf688e2..9c76f5c 100644 --- a/tools/convexhull_framework/src/VideoDecoder.py +++ b/tools/convexhull_framework/src/VideoDecoder.py
@@ -15,12 +15,12 @@ from Config import AOMDEC, AV1DEC, EnableTimingInfo, Platform, UsePerfUtil, FFMPEG from Utils import ExecuteCmd, GetShortContentName, ConvertYUVToY4M, DeleteFile -def DecodeWithAOM(test_cfg, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly=False): +def DecodeWithAOM(test_cfg, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly=False): if decode_to_yuv: args = " --codec=av1 --summary --rawvideo -o %s %s" % (outfile, infile) else: args = " --codec=av1 --summary -o %s %s" % (outfile, infile) - cmd = AOMDEC + args + cmd = AOMDEC + args + "> %s 2>&1"%dec_log if EnableTimingInfo: if Platform == "Windows": cmd = "ptime " + cmd + " >%s"%dec_perf @@ -34,12 +34,12 @@ ExecuteCmd(cmd, LogCmdOnly) -def DecodeWithAV1(test_cfg, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly=False): +def DecodeWithAV1(test_cfg, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly=False): if decode_to_yuv: args = " --codec=av1 --summary --rawvideo -o %s %s" % (outfile, infile) else: args = " --codec=av1 --summary -o %s %s" % (outfile, infile) - cmd = AV1DEC + args + cmd = AV1DEC + args + "> %s 2>&1"%dec_log if EnableTimingInfo: if Platform == "Windows": cmd = "ptime " + cmd + " >%s" % dec_perf @@ -53,7 +53,7 @@ ExecuteCmd(cmd, LogCmdOnly) -def DecodeWithHEVC(test_cfg, clip, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly=False): +def DecodeWithHEVC(test_cfg, clip, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly=False): args = " -i %s " % infile if clip.fmt == '420' and clip.bit_depth == 8: pix_fmt = "yuv420p" @@ -63,7 +63,7 @@ print("Unsupported color format") args += " -pix_fmt %s %s" % (pix_fmt, outfile) - cmd = FFMPEG + args + cmd = FFMPEG + args + "> %s 2>&1"%dec_log if EnableTimingInfo: if Platform == "Windows": cmd = "ptime " + cmd + " >%s" % dec_perf @@ -77,13 +77,13 @@ ExecuteCmd(cmd, LogCmdOnly) -def VideoDecode(clip, method, test_cfg, codec, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly=False): +def VideoDecode(clip, method, test_cfg, codec, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly=False): Utils.CmdLogger.write("::Decode\n") if codec == 'av2' and method == 'aom': - DecodeWithAOM(test_cfg, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly) + DecodeWithAOM(test_cfg, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly) elif codec == 'av1': - DecodeWithAV1(test_cfg, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly) + DecodeWithAV1(test_cfg, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly) elif codec == 'hevc': - DecodeWithHEVC(test_cfg, clip, infile, outfile, dec_perf, decode_to_yuv, LogCmdOnly) + DecodeWithHEVC(test_cfg, clip, infile, outfile, dec_perf, decode_to_yuv, dec_log, LogCmdOnly) else: raise ValueError("invalid parameter for decode.")