| #!/usr/bin/env python |
| ## Copyright (c) 2021, Alliance for Open Media. All rights reserved |
| ## |
| ## This source code is subject to the terms of the BSD 3-Clause Clear License and the |
| ## Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear License was |
| ## not distributed with this source code in the LICENSE file, you can obtain it |
| ## at aomedia.org/license/software-license/bsd-3-c-c/. If the Alliance for Open Media Patent |
| ## License 1.0 was not distributed with this source code in the PATENTS file, you |
| ## can obtain it at aomedia.org/license/patent-license/. |
| ## |
| __author__ = "maggie.sun@intel.com, ryanlei@fb.com" |
| |
| import os |
| import xlsxwriter |
| import xlrd |
| import logging |
| from CalculateQualityMetrics import CalculateQualityMetric, GatherQualityMetrics |
| from VideoScaler import GetDownScaledOutFile, DownScaling, UpScaling,\ |
| GetUpScaledOutFile |
| from Config import DnScaleRatio, FrameNum, QualityList, LoggerName,\ |
| Path_ScalingResults, ScalQty_WtCols, ScalQty_startRow, LineColors, \ |
| ScalSumQty_WtCols, AS_DOWNSCALE_ON_THE_FLY |
| from Utils import Cleanfolder, GetShortContentName, CreateChart_Scatter, \ |
| AddSeriesToChart_Scatter, UpdateChart, InsertChartsToSheet, \ |
| CalcRowsClassAndContentDict, Clip, CreateChart_Bar, AddSeriesToChart_Bar, \ |
| DeleteFile |
| import Utils |
| |
| subloggername = "ScalingTest" |
| loggername = LoggerName + '.' + '%s' % subloggername |
| logger = logging.getLogger(loggername) |
| |
| StatsMetrics = ['Max', 'Std+', 'Average', 'Std-', 'Min'] |
| |
| def GetScalingResultExcelFile(method, rationum, scalAlgoNum): |
| filename = "ScalingResults_RatioNum_%d_%s_AlgoNum_%d.xlsx"\ |
| % (rationum, method, scalAlgoNum) |
| file = os.path.join(Path_ScalingResults, filename) |
| return file |
| |
| def GetScalingResultExcelFile_PerContent(content, method): |
| filename = GetShortContentName(content, False) |
| filename = "ScalingResults_%s_%s.xlsx" % (method, filename) |
| file = os.path.join(Path_ScalingResults, filename) |
| return file |
| |
| def Run_Scaling_Test(clip, dnScalAlgo, upScalAlgo, path_dnscl, path_upscl, |
| path_log, path_cfg, savememory, keepupscaledyuv, ScaleMethod, |
| LogCmdOnly=False): |
| logger.info("start running scaling tests with content %s" |
| % os.path.basename(clip.file_name)) |
| DnScaledRes = [(int(clip.width / ratio), int(clip.height / ratio)) |
| for ratio in DnScaleRatio] |
| for i in range(len(DnScaledRes)): |
| DnScaledW = DnScaledRes[i][0] |
| DnScaledH = DnScaledRes[i][1] |
| if (DnScaledW == clip.width and DnScaledH == clip.height): |
| continue |
| logger.info("start downscaling content to %dx%d" % (DnScaledW, DnScaledH)) |
| JobName = '%s_Downscaling_%dx%d' % \ |
| (GetShortContentName(clip.file_name, False), DnScaledW, DnScaledH) |
| if LogCmdOnly: |
| Utils.CmdLogger.write("============== %s Job Start =================\n"%JobName) |
| # downscaling |
| dnscalyuv = GetDownScaledOutFile(clip, DnScaledW, DnScaledH, |
| path_dnscl, ScaleMethod, dnScalAlgo) |
| if not os.path.isfile(dnscalyuv): |
| dnscalyuv = DownScaling(ScaleMethod, clip, FrameNum['AS'], DnScaledW, DnScaledH, |
| path_dnscl, path_cfg, dnScalAlgo, LogCmdOnly) |
| dnscaled_clip = Clip(GetShortContentName(dnscalyuv, False)+'.y4m', |
| dnscalyuv, "", DnScaledW, DnScaledH, |
| clip.fmt, clip.fps_num, clip.fps_denom, |
| clip.bit_depth) |
| upscaleyuv = UpScaling(ScaleMethod, dnscaled_clip, FrameNum['AS'], clip.width, clip.height, |
| path_upscl, path_cfg, upScalAlgo, LogCmdOnly) |
| CalculateQualityMetric(clip.file_path, FrameNum['AS'], upscaleyuv, clip.fmt, |
| clip.width, clip.height, clip.bit_depth, |
| path_log, LogCmdOnly) |
| if savememory: |
| if dnscalyuv != clip.file_path: |
| DeleteFile(dnscalyuv, LogCmdOnly) |
| if not keepupscaledyuv: |
| DeleteFile(upscaleyuv, LogCmdOnly) |
| |
| if LogCmdOnly: |
| Utils.CmdLogger.write("============== %s Job End =================\n" % JobName) |
| logger.info("finish running scaling test.") |
| |
| def GeneratePerClipExcelFile(ScaleMethod, dnScalAlgos, upScalAlgos, clip, path_log): |
| logger.info("start generate excel file for content :%s" % clip.file_name) |
| excFile = GetScalingResultExcelFile_PerContent(clip.file_name, ScaleMethod) |
| wb = xlsxwriter.Workbook(excFile) |
| shtname = GetShortContentName(clip.file_name) |
| sht = wb.add_worksheet(shtname) |
| |
| scaleRatios = DnScaleRatio |
| if (1.0 in scaleRatios): |
| scaleRatios.remove(1.0) |
| |
| sht.write(1, 0, 'Content Name') |
| sht.write(2, 0, clip.file_name) |
| sht.write(1, 1, 'Scaling Ratio') |
| sht.write_column(2, 1, scaleRatios) |
| pre_title = ['Width', 'Height', 'DnScaledWidth', 'DnScaledHeight'] |
| sht.write_row(1, 2, pre_title) |
| for dn_algo, up_algo, col in zip(dnScalAlgos, upScalAlgos, ScalQty_WtCols): |
| algos = dn_algo + '--' + up_algo |
| sht.write(0, col, algos) |
| sht.write_row(1, col, QualityList) |
| |
| rows = [ScalQty_startRow + i for i in range(len(scaleRatios))] |
| continfos = [] |
| for ratio, row in zip(scaleRatios, rows): |
| dw = int(clip.width / ratio) |
| dh = int(clip.height / ratio) |
| info = [clip.width, clip.height, dw, dh] |
| sht.write_row(row, 2, info) |
| continfos.append(info) |
| |
| charts = []; y_mins = {}; y_maxs = {} |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| chart_title = 'Scaling Quality - %s' % qty |
| xaxis_name = 'scaling ratio' |
| chart = CreateChart_Scatter(wb, chart_title, xaxis_name, qty) |
| charts.append(chart) |
| y_mins[x] = []; y_maxs[x] = [] |
| |
| for dn_algo, up_algo, col, i in zip(dnScalAlgos, upScalAlgos, ScalQty_WtCols, |
| range(len(dnScalAlgos))): |
| qualities = [] |
| seriname = dn_algo + '--' + up_algo |
| for ratio, row, idx in zip(scaleRatios, rows, range(len(scaleRatios))): |
| w = continfos[idx][0] |
| h = continfos[idx][1] |
| dw = continfos[idx][2] |
| dh = continfos[idx][3] |
| dnScalOut = GetDownScaledOutFile(clip, dw, dh, path_log, ScaleMethod, dn_algo) |
| ds_clip = Clip(GetShortContentName(dnScalOut, False) + '.y4m', |
| dnScalOut, "", dw, dh, clip.fmt, clip.fps_num, clip.fps_denom, |
| clip.bit_depth) |
| upScalOut = GetUpScaledOutFile(ds_clip, clip.width, clip.height, |
| ScaleMethod, up_algo, path_log) |
| qtys, perframe_vmaf_log = GatherQualityMetrics(upScalOut, path_log) |
| sht.write_row(row, col, qtys) |
| qualities.append(qtys) |
| for x in range(len(QualityList)): |
| AddSeriesToChart_Scatter(shtname, rows, col + x, 1, charts[x], |
| seriname, LineColors[i]) |
| |
| # get min and max of y-axis for a certain dn up scaling algo |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| qs = [row[x] for row in qualities] |
| y_mins[x].append(min(qs)) |
| y_maxs[x].append(max(qs)) |
| |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| ymin = min(y_mins[x]) |
| ymax = max(y_maxs[x]) |
| margin = 0.1 # add 10% on min and max value for y_axis range |
| num_precsn = 5 if 'MS-SSIM' in qty else 3 |
| UpdateChart(charts[x], ymin, ymax, margin, qty, num_precsn) |
| |
| startrow = rows[-1] + 2; startcol = 1 |
| InsertChartsToSheet(sht, startrow, startcol, charts) |
| |
| wb.close() |
| logger.info("finish export scaling quality results to excel file.") |
| |
| def GenerateSummarySheet(wb, ScaleMethod, dnScalAlgos, upScalAlgos, ratio, path_log): |
| logger.info("start generate summary sheet for ratio %2.2f" % ratio) |
| |
| shts = [] |
| shtname = "Ratio=%1.2f" % ratio |
| sht = wb.add_worksheet(shtname) |
| shts.append(sht) |
| sht.write(1, 0, 'Content Class') |
| sht.write(1, 1, 'Content NO.') |
| sht.write(1, 2, 'Content Name') |
| pre_title = ['Width', 'Height', 'DnScaledWidth', 'DnScaledHeight'] |
| sht.write_row(1, 3, pre_title) |
| |
| for dn_algo, up_algo, col in zip(dnScalAlgos, upScalAlgos, ScalSumQty_WtCols): |
| algos = dn_algo + '--' + up_algo |
| sht.write(0, col, algos) |
| sht.write_row(1, col, QualityList) |
| |
| content_infos = {}; totalnum_contents = 0 |
| for (clss, clip_list), row_clss in zip(ClipDict.items(), Rows_Class): |
| sht.write(row_clss, 0, clss) |
| totalnum_contents = totalnum_contents + len(clip_list) |
| for clip, row_cont in zip(clip_list, range(len(clip_list))): |
| dw = int(clip.width / ratio) |
| dh = int(clip.height / ratio) |
| shortcntname = GetShortContentName(clip.file_name, False) |
| sht.write(row_clss + row_cont, 2, shortcntname) |
| infos = [clip.width, clip.height, dw, dh] |
| sht.write_row(row_clss + row_cont, 3, infos) |
| content_infos[shortcntname] = infos |
| |
| charts = []; y_mins = {}; y_maxs = {} |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| chart_title = '%s of %s' % (qty, shtname) |
| chart = CreateChart_Bar(wb, chart_title, 'Contents', qty) |
| charts.append(chart) |
| y_mins[x] = []; y_maxs[x] = [] |
| |
| rows_all = [ScalQty_startRow + i for i in range(totalnum_contents)] |
| sht.write_column(ScalQty_startRow, 1, range(totalnum_contents)) |
| for dn_algo, up_algo, col, i in zip(dnScalAlgos, upScalAlgos, |
| ScalSumQty_WtCols, |
| range(len(dnScalAlgos))): |
| qualities = [] |
| seriname = dn_algo + '--' + up_algo |
| for (clss, clip_list), row_clss in zip(ClipDict.items(), Rows_Class): |
| for clip, row_cont in zip(clip_list, range(len(clip_list))): |
| key = GetShortContentName(clip.file_name, False) |
| w = content_infos[key][0] |
| h = content_infos[key][1] |
| dw = content_infos[key][2] |
| dh = content_infos[key][3] |
| dnScalOut = GetDownScaledOutFile(clip, dw, dh, path_log, ScaleMethod, dn_algo) |
| ds_clip = Clip(GetShortContentName(dnScalOut, False) + '.y4m', |
| dnScalOut, clss, dw, dh, clip.fmt, clip.fps_num, |
| clip.fps_denom, clip.bit_depth) |
| upScalOut = GetUpScaledOutFile(ds_clip, w, h, ScaleMethod, up_algo, path_log) |
| qtys, perframe_vmaf_log = GatherQualityMetrics(upScalOut, path_log) |
| sht.write_row(row_clss + row_cont, col, qtys) |
| qualities.append(qtys) |
| |
| for x in range(len(QualityList)): |
| AddSeriesToChart_Bar(shtname, rows_all, col + x, 1, charts[x], seriname) |
| |
| # get min and max of y-axis for a certain dn up scaling algo |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| qs = [row[x] for row in qualities] |
| y_mins[x].append(min(qs)) |
| y_maxs[x].append(max(qs)) |
| |
| for qty, x in zip(QualityList, range(len(QualityList))): |
| ymin = min(y_mins[x]) |
| ymax = max(y_maxs[x]) |
| margin = 0.1 # add 10% on min and max value for y_axis range |
| num_precsn = 5 if 'MS-SSIM' in qty else 3 |
| UpdateChart(charts[x], ymin, ymax, margin, qty, num_precsn) |
| |
| startrow = rows_all[-1] + 2; startcol = 1 |
| InsertChartsToSheet(sht, startrow, startcol, charts) |
| logger.info("finish average sheet for ratio:%2.2f." % ratio) |
| |
| return sht |
| |
| def GenerateAverageSheet(wb, sumsht, dnScalAlgos, upScalAlgos, ratio): |
| logger.info("start generate average sheet for ratio %2.2f" % ratio) |
| |
| rdsht = sumsht |
| rdshtname = rdsht.get_name() |
| shtname = "AverageForRatio=%1.2f" % ratio |
| sht = wb.add_worksheet(shtname) |
| sht.write(1, 0, 'QualityMetrics') |
| sht.write(1, 1, 'Content Class') |
| sht.write(1, 2, 'Content Number') |
| |
| interval = 1 |
| step = len(StatsMetrics) + interval |
| startcol = 3 |
| cols_avg = [startcol + step * i for i in range(len(dnScalAlgos))] |
| for col, dn_algo, up_algo in zip(cols_avg, dnScalAlgos, upScalAlgos): |
| algos = dn_algo + '--' + up_algo |
| sht.write(0, col, algos) |
| sht.write_row(1, col, StatsMetrics) |
| |
| step = len(ClipDict) + 1 # 1 extra row for total of each class |
| startrow = 2 |
| rows_qtymtr = [startrow + step * i for i in range(len(QualityList))] |
| for qty, row_qm, y in zip(QualityList, rows_qtymtr, range(len(QualityList))): |
| sht.write(row_qm, 0, qty) |
| |
| #charts = [] |
| #titlename = 'Quality Statistics %s' % qty |
| #chart = CreateChart_Line(wb, titlename, qty) |
| #charts.append(chart) |
| |
| totalnum_contents = 0 |
| for (cls, clip_list), idx, rdrow_cls in zip(ClipDict.items(), |
| range(len(ClipDict)), |
| Rows_Class): |
| sht.write(row_qm + idx, 1, cls) |
| num_content = len(clip_list) |
| totalnum_contents = totalnum_contents + num_content |
| sht.write(row_qm + idx, 2, num_content) |
| for rdcol, wtcol in zip(ScalSumQty_WtCols, cols_avg): |
| startcell = xlrd.cellname(rdrow_cls, rdcol + y) |
| endcell = xlrd.cellname(rdrow_cls + num_content - 1, rdcol + y) |
| formula = '=MAX(\'%s\'!%s:%s)' % (rdshtname, startcell, endcell) |
| sht.write(row_qm + idx, wtcol, formula) |
| |
| formula = '=SUM(\'%s\'!%s:%s)/%d' % (rdshtname, startcell, |
| endcell, num_content) |
| sht.write(row_qm + idx, wtcol + 2, formula) |
| |
| formula = '=MIN(\'%s\'!%s:%s)' % (rdshtname, startcell, endcell) |
| sht.write(row_qm + idx, wtcol + 4, formula) |
| |
| avgcell = xlrd.cellname(row_qm + idx, wtcol + 2) |
| formula = '= %s + _xlfn.STDEV.P(\'%s\'!%s:%s)'\ |
| % (avgcell, rdshtname, startcell, endcell) |
| sht.write(row_qm + idx, wtcol + 1, formula) |
| |
| formula = '= %s - _xlfn.STDEV.P(\'%s\'!%s:%s)'\ |
| % (avgcell, rdshtname, startcell, endcell) |
| sht.write(row_qm + idx, wtcol + 3, formula) |
| |
| #write total contents statistics |
| wtrow = row_qm + len(ClipDict) |
| sht.write(wtrow, 1, 'Total') |
| sht.write(wtrow, 2, totalnum_contents) |
| for rdcol, wtcol in zip(ScalSumQty_WtCols, cols_avg): |
| startcell = xlrd.cellname(ScalQty_startRow, rdcol + y) |
| endcell = xlrd.cellname(ScalQty_startRow + totalnum_contents - 1, |
| rdcol + y) |
| formula = '=MAX(\'%s\'!%s:%s)' % (rdshtname, startcell, endcell) |
| sht.write(wtrow, wtcol, formula) |
| |
| formula = '=SUM(\'%s\'!%s:%s)/%d'\ |
| % (rdshtname, startcell, endcell, totalnum_contents) |
| sht.write(wtrow, wtcol + 2, formula) |
| |
| formula = '=MIN(\'%s\'!%s:%s)' % (rdshtname, startcell, endcell) |
| sht.write(wtrow, wtcol + 4, formula) |
| |
| avgcell = xlrd.cellname(wtrow, wtcol + 2) |
| formula = '= %s + _xlfn.STDEV.P(\'%s\'!%s:%s)'\ |
| % (avgcell, rdshtname, startcell, endcell) |
| sht.write(wtrow, wtcol + 1, formula) |
| |
| formula = '= %s - _xlfn.STDEV.P(\'%s\'!%s:%s)'\ |
| % (avgcell, rdshtname, startcell, endcell) |
| sht.write(wtrow, wtcol + 3, formula) |
| |
| logger.info("finish average sheet for ratio:%2.2f." % ratio) |
| |
| def SaveScalingResultsToExcel(ScaleMethod, dnScalAlgos, upScalAlgos, clip_list, path_log): |
| logger.info("start saving scaling quality results to excel files.......") |
| if not os.path.exists(Path_ScalingResults): |
| os.makedirs(Path_ScalingResults) |
| |
| logger.info("start generating per content scaling quality excel file.......") |
| for clip in clip_list: |
| GeneratePerClipExcelFile(ScaleMethod, dnScalAlgos, upScalAlgos, clip, path_log) |
| |
| scaleRatios = DnScaleRatio |
| if (1.0 in scaleRatios): |
| scaleRatios.remove(1.0) |
| logger.info("start generating scaling quality summary excel file.......") |
| sumexcFile = GetScalingResultExcelFile(ScaleMethod, len(scaleRatios), len(dnScalAlgos)) |
| wb = xlsxwriter.Workbook(sumexcFile) |
| # to generate rows number of starting of each class: Rows_Class |
| global ClipDict, Rows_Class |
| ClipDict, Rows_Class = CalcRowsClassAndContentDict(ScalQty_startRow, clip_list) |
| sumShts = [] |
| for ratio in scaleRatios: |
| sht = GenerateSummarySheet(wb, ScaleMethod, dnScalAlgos, upScalAlgos, ratio, path_log) |
| sumShts.append(sht) |
| for ratio, sumsht in zip(scaleRatios, sumShts): |
| GenerateAverageSheet(wb, sumsht, dnScalAlgos, upScalAlgos, ratio) |
| |
| wb.close() |
| logger.info("finish saving scaling quality results to excel files.......") |
