CWG-F015: M-AVM: A Reference Software for Multi-layered Video Coding A software implementation of multilayer coding extensions that include: (i) basic multilayer coding capabilities, and (ii) allows extending the size of the decoded frame buffer (DPB). This software implementation is released for further multilayered/multiview video coding standardization development as a branch for AVM.
diff --git a/README.md b/README.md index 64caea8..82fd7da 100644 --- a/README.md +++ b/README.md
@@ -1,6 +1,23 @@ -# AVM +# Multiview-AVM (M-AVM) v1.0.0 -[TOC] +M-AVM is the multilayer/multiview extension of the AVM reference software. + +After building and compiling *aomenc* and *aomdec* binaries as instructed below, the following command line entries for *aomenc* will compress of multiple input views from separate raw video files. + +For example, two views stored in `Raw_video_view0.yuv` and `Raw_video_view1.yuv` files can be compressed using *aomenc* with the following command line arguments: +~~~ +aomenc <Raw_video_view0.yuv> <Raw_video_view1.yuv> --num-views=2 -o bitstream.avm [*additional command arguments for AVM] +~~~ +This will generate a single M-AVM bitstream in `bitstream.avm`. The following sample command can be used to decode using the *aomdec* binary. +~~~ +aomdec bitstream.avm --output=Recon_video.yuv --progress --output-bit-depth=8 --rawvideo +~~~ +This command will decode and output two files `Recon_video_view0.yuv` and `Recon_video_view1.yuv` for each view. + +The current version software allows up to encoding and decoding of 256 views. For more `N` views the encoder the following encoder command can be used: +~~~ +aomenc <Raw_video_view0.yuv> <Raw_video_view1.yuv> ... <Raw_video_view{N-1}.yuv> --num-views=N -o bitstream.avm [*additional command arguments for AVM] +~~~ ## Building
diff --git a/aom/aom_decoder.h b/aom/aom_decoder.h index 46846a9..1d7b731 100644 --- a/aom/aom_decoder.h +++ b/aom/aom_decoder.h
@@ -33,6 +33,7 @@ #include "aom/aom_codec.h" #include "aom/aom_frame_buffer.h" +#include "config/aom_config.h" /*!\brief Current ABI version number * @@ -75,7 +76,10 @@ unsigned int is_kf; /**< Current frame is a keyframe */ unsigned int number_spatial_layers; /**< Number of spatial layers */ unsigned int number_temporal_layers; /**< Number of temporal layers */ - unsigned int is_annexb; /**< Is Bitstream in Annex-B format */ +#if CONFIG_MULTIVIEW_CORE + unsigned int number_views; /**< Number of views */ +#endif + unsigned int is_annexb; /**< Is Bitstream in Annex-B format */ } aom_codec_stream_info_t; /* REQUIRED FUNCTIONS
diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h index fdba906..f83102d 100644 --- a/aom/aom_encoder.h +++ b/aom/aom_encoder.h
@@ -703,7 +703,13 @@ * height written in write_sequence_header(). */ unsigned int g_forced_max_frame_height; - +#if CONFIG_MULTIVIEW_CORE + /*!\brief Number of views + * + * + */ + unsigned int g_num_views; +#endif /*!\brief Bit-depth of the codec * * This value identifies the bit_depth of the codec,
diff --git a/aom/aom_frame_buffer.h b/aom/aom_frame_buffer.h index 7f5b6b9..12796b6 100644 --- a/aom/aom_frame_buffer.h +++ b/aom/aom_frame_buffer.h
@@ -22,17 +22,25 @@ #endif #include "aom/aom_integer.h" +#include "config/aom_config.h" /*!\brief The maximum number of work buffers used by libaom. * Support maximum 4 threads to decode video in parallel. * Each thread will use one work buffer. * TODO(hkuang): Add support to set number of worker threads dynamically. */ +#if CONFIG_MULTIVIEW_EXTENDED_DPB +#define AOM_MAXIMUM_WORK_BUFFERS 16 +#else #define AOM_MAXIMUM_WORK_BUFFERS 8 - +#endif /*!\brief The maximum number of reference buffers that a AV1 encoder may use. */ +#if CONFIG_MULTIVIEW_EXTENDED_DPB +#define AOM_MAXIMUM_REF_BUFFERS 16 +#else #define AOM_MAXIMUM_REF_BUFFERS 8 +#endif /*!\brief External frame buffer *
diff --git a/aom/aom_image.h b/aom/aom_image.h index bd8c045..4048264 100644 --- a/aom/aom_image.h +++ b/aom/aom_image.h
@@ -21,6 +21,7 @@ extern "C" { #endif +#include "config/aom_config.h" #include "aom/aom_integer.h" /*!\brief Current ABI version number @@ -210,7 +211,10 @@ int temporal_id; /**< Temporal layer Id of image */ int spatial_id; /**< Spatial layer Id of image */ - +#if CONFIG_MULTIVIEW_CORE + int view_id; /**< View layer id of image */ + int display_order_hint; /**< display order of image */ +#endif /*!\brief The following member may be set by the application to associate * data with this image. */
diff --git a/aom/src/aom_decoder.c b/aom/src/aom_decoder.c index 3f2db00..ff3756d 100644 --- a/aom/src/aom_decoder.c +++ b/aom/src/aom_decoder.c
@@ -66,7 +66,9 @@ /* Set default/unknown values */ si->w = 0; si->h = 0; - +#if CONFIG_MULTIVIEW_CORE + si->number_views = 1; +#endif res = iface->dec.peek_si(data, data_sz, si); } @@ -85,7 +87,9 @@ /* Set default/unknown values */ si->w = 0; si->h = 0; - +#if CONFIG_MULTIVIEW_CORE + si->number_views = 0; +#endif res = ctx->iface->dec.get_si(get_alg_priv(ctx), si); }
diff --git a/apps/aomdec.c b/apps/aomdec.c index 80f6620..74c7d21 100644 --- a/apps/aomdec.c +++ b/apps/aomdec.c
@@ -231,18 +231,32 @@ static int read_frame(struct AvxDecInputContext *input, uint8_t **buf, size_t *bytes_in_buffer, size_t *buffer_size) { +#if CONFIG_MULTIVIEW_CORE + switch (input->aom_input_ctx->file_type[0]) { +#else switch (input->aom_input_ctx->file_type) { +#endif #if CONFIG_WEBM_IO case FILE_TYPE_WEBM: return webm_read_frame(input->webm_ctx, buf, bytes_in_buffer, buffer_size); #endif case FILE_TYPE_RAW: +#if CONFIG_MULTIVIEW_CORE + return raw_read_frame(input->aom_input_ctx->file[0], buf, bytes_in_buffer, + buffer_size); +#else return raw_read_frame(input->aom_input_ctx->file, buf, bytes_in_buffer, buffer_size); +#endif case FILE_TYPE_IVF: +#if CONFIG_MULTIVIEW_CORE + return ivf_read_frame(input->aom_input_ctx->file[0], buf, bytes_in_buffer, + buffer_size, NULL); +#else return ivf_read_frame(input->aom_input_ctx->file, buf, bytes_in_buffer, buffer_size, NULL); +#endif case FILE_TYPE_OBU: return obudec_read_temporal_unit(input->obu_ctx, buf, bytes_in_buffer, buffer_size); @@ -256,7 +270,11 @@ aom_codec_stream_info_t si; memset(&si, 0, sizeof(si)); +#if CONFIG_MULTIVIEW_CORE + if (fread(buf, 1, 32, input->file[0]) == 32) { +#else if (fread(buf, 1, 32, input->file) == 32) { +#endif int i; if (mem_get_le32(buf) < 256 * 1024 * 1024) { @@ -274,10 +292,23 @@ } } } - +#if CONFIG_MULTIVIEW_CORE + rewind(input->file[0]); +#else rewind(input->file); +#endif return is_raw; } +#if CONFIG_MULTIVIEW_CORE +static void show_progress_multiview(int frame_in, int frame_out, int view_id, + int frame_out_per_view, uint64_t dx_time) { + fprintf(stderr, + "%d decoded frames/%d showed frames (view_id=%d, frame_per_view=%d) " + "in %" PRId64 " us (%.2f fps)\r", + frame_in, frame_out, view_id, frame_out_per_view, dx_time, + (double)frame_out * 1000000.0 / (double)dx_time); +} +#endif static void show_progress(int frame_in, int frame_out, uint64_t dx_time) { fprintf(stderr, @@ -513,6 +544,26 @@ } } +#if CONFIG_MULTIVIEW_CORE +static void open_multiple_output_files_perview( + const char *outfile_name, const struct AvxInputContext aom_input_ctx, + FILE **outfile) { + for (int view_id = 0; view_id < aom_input_ctx.num_views; view_id++) { + char output_filename[PATH_MAX]; + char extension[6]; + char suffix[16]; + sprintf(suffix, "_view%0d", view_id); + strcpy(output_filename, outfile_name); + char *ptr_extension = strrchr(output_filename, '.'); + strcpy(extension, ptr_extension); + if (ptr_extension != NULL) *ptr_extension = '\0'; + strcat(output_filename, suffix); + strcat(output_filename, extension); + outfile[view_id] = open_outfile(output_filename); + } +} +#endif + static int main_loop(int argc, const char **argv_) { aom_codec_ctx_t decoder; char *fn = NULL; @@ -553,7 +604,11 @@ const char *outfile_pattern = NULL; char outfile_name[PATH_MAX] = { 0 }; +#if CONFIG_MULTIVIEW_CORE + FILE *outfile[NUM_LAYERS_MAX] = { NULL }; +#else FILE *outfile = NULL; +#endif FILE *framestats_file = NULL; @@ -702,20 +757,41 @@ return EXIT_FAILURE; } #endif +#if CONFIG_MULTIVIEW_CORE + input.aom_input_ctx->filename[0] = fn; + input.aom_input_ctx->file[0] = infile; +#else input.aom_input_ctx->filename = fn; input.aom_input_ctx->file = infile; +#endif if (file_is_ivf(input.aom_input_ctx)) { +#if CONFIG_MULTIVIEW_CORE + input.aom_input_ctx->file_type[0] = FILE_TYPE_IVF; +#else input.aom_input_ctx->file_type = FILE_TYPE_IVF; +#endif is_ivf = 1; } #if CONFIG_WEBM_IO else if (file_is_webm(input.webm_ctx, input.aom_input_ctx)) +#if CONFIG_MULTIVIEW_CORE + input.aom_input_ctx->file_type[0] = FILE_TYPE_WEBM; +#else input.aom_input_ctx->file_type = FILE_TYPE_WEBM; #endif +#endif else if (file_is_obu(&obu_ctx)) +#if CONFIG_MULTIVIEW_CORE + input.aom_input_ctx->file_type[0] = FILE_TYPE_OBU; +#else input.aom_input_ctx->file_type = FILE_TYPE_OBU; +#endif else if (file_is_raw(input.aom_input_ctx)) +#if CONFIG_MULTIVIEW_CORE + input.aom_input_ctx->file_type[0] = FILE_TYPE_RAW; +#else input.aom_input_ctx->file_type = FILE_TYPE_RAW; +#endif else { fprintf(stderr, "Unrecognized input file type.\n"); #if !CONFIG_WEBM_IO @@ -727,7 +803,7 @@ outfile_pattern = outfile_pattern ? outfile_pattern : "-"; single_file = is_single_file(outfile_pattern); - +#if !CONFIG_MULTIVIEW_CORE if (!noblit && single_file) { generate_filename(outfile_pattern, outfile_name, PATH_MAX, aom_input_ctx.width, aom_input_ctx.height, 0); @@ -736,7 +812,7 @@ else outfile = open_outfile(outfile_name); } - +#endif if (use_y4m && !noblit) { if (!single_file) { fprintf(stderr, @@ -746,7 +822,11 @@ } #if CONFIG_WEBM_IO +#if CONFIG_MULTIVIEW_CORE + if (aom_input_ctx.file_type[0] == FILE_TYPE_WEBM) { +#else if (aom_input_ctx.file_type == FILE_TYPE_WEBM) { +#endif if (webm_guess_framerate(input.webm_ctx, input.aom_input_ctx)) { fprintf(stderr, "Failed to guess framerate -- error parsing " @@ -842,18 +922,32 @@ if (!read_frame(&input, &buf, &bytes_in_buffer, &buffer_size)) { frame_avail = 1; frame_in++; - aom_usec_timer_start(&timer); - if (aom_codec_decode(&decoder, buf, bytes_in_buffer, NULL)) { const char *detail = aom_codec_error_detail(&decoder); warn("Failed to decode frame %d: %s", frame_in, aom_codec_error(&decoder)); - if (detail) warn("Additional information: %s", detail); if (!keep_going) goto fail; } +#if CONFIG_MULTIVIEW_CORE + if (frame_in == 1) { + if (!noblit && single_file) { + generate_filename(outfile_pattern, outfile_name, PATH_MAX, + aom_input_ctx.width, aom_input_ctx.height, 0); + if (do_md5) + MD5Init(&md5_ctx); + else { + aom_codec_stream_info_t si; + aom_codec_get_stream_info(&decoder, &si); + aom_input_ctx.num_views = si.number_views; + open_multiple_output_files_perview(outfile_name, aom_input_ctx, + outfile); + } + } + } +#endif if (framestats_file) { int qp; if (AOM_CODEC_CONTROL_TYPECHECKED(&decoder, AOMD_GET_LAST_QUANTIZER, @@ -887,7 +981,15 @@ dx_time += aom_usec_timer_elapsed(&timer); got_data = 0; +#if CONFIG_MULTIVIEW_CORE while ((img = aom_codec_get_frame(&decoder, &iter))) { +#else + while ((img = aom_codec_get_frame(&decoder, &iter))) { +#endif +#if CONFIG_MULTIVIEW_CORE + const int view_id = img->view_id; + const int frame_per_view = frame_out / aom_input_ctx.num_views; +#endif ++frame_out; if (frame_in < frame_out) { // No OBUs for show_existing_frame. frame_in = frame_out; @@ -901,7 +1003,13 @@ } frames_corrupted += corrupted; +#if CONFIG_MULTIVIEW_CORE + if (progress) + show_progress_multiview(frame_in, frame_out, view_id, frame_per_view, + dx_time); +#else if (progress) show_progress(frame_in, frame_out, dx_time); +#endif if (do_verify) { if (check_decoded_frame_hash(&decoder, img, frame_out, @@ -976,7 +1084,12 @@ if (use_y4m) { char y4m_buf[Y4M_BUFFER_SIZE] = { 0 }; size_t len = 0; +#if CONFIG_MULTIVIEW_CORE + if (frame_out == 1 || + ((frame_out - 1) / aom_input_ctx.num_views) < 1) { +#else if (frame_out == 1) { +#endif // Y4M file header len = y4m_write_file_header( y4m_buf, sizeof(y4m_buf), aom_input_ctx.width, @@ -991,7 +1104,11 @@ if (do_md5) { MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len); } else { +#if CONFIG_MULTIVIEW_CORE + fputs(y4m_buf, outfile[view_id]); +#else fputs(y4m_buf, outfile); +#endif } } @@ -1001,8 +1118,13 @@ MD5Update(&md5_ctx, (md5byte *)y4m_buf, (unsigned int)len); y4m_update_image_md5(img, planes, &md5_ctx); } else { +#if CONFIG_MULTIVIEW_CORE + fputs(y4m_buf, outfile[view_id]); + y4m_write_image_file(img, planes, outfile[view_id]); +#else fputs(y4m_buf, outfile); y4m_write_image_file(img, planes, outfile); +#endif } } else { if (frame_out == 1) { @@ -1029,7 +1151,11 @@ if (do_md5) { raw_update_image_md5(img, planes, num_planes, &md5_ctx); } else { +#if CONFIG_MULTIVIEW_CORE + raw_write_image_file(img, planes, num_planes, outfile[view_id]); +#else raw_write_image_file(img, planes, num_planes, outfile); +#endif } } } else { @@ -1045,6 +1171,15 @@ MD5Final(md5_digest, &md5_ctx); print_md5(md5_digest, outfile_name); } else { +#if CONFIG_MULTIVIEW_CORE + outfile[0] = open_outfile(outfile_name); + if (use_y4m) { + y4m_write_image_file(img, planes, outfile[0]); + } else { + raw_write_image_file(img, planes, num_planes, outfile[0]); + } + fclose(outfile[0]); +#else outfile = open_outfile(outfile_name); if (use_y4m) { y4m_write_image_file(img, planes, outfile); @@ -1052,6 +1187,7 @@ raw_write_image_file(img, planes, num_planes, outfile); } fclose(outfile); +#endif } } } @@ -1083,19 +1219,36 @@ MD5Final(md5_digest, &md5_ctx); print_md5(md5_digest, outfile_name); } else { +#if CONFIG_MULTIVIEW_CORE + for (int view_id = 0; view_id < aom_input_ctx.num_views; view_id++) { + fclose(outfile[view_id]); + } +#else fclose(outfile); +#endif } } #if CONFIG_WEBM_IO +#if CONFIG_MULTIVIEW_CORE + if (input.aom_input_ctx->file_type[0] == FILE_TYPE_WEBM) +#else if (input.aom_input_ctx->file_type == FILE_TYPE_WEBM) +#endif webm_free(input.webm_ctx); #endif +#if CONFIG_MULTIVIEW_CORE + if (input.aom_input_ctx->file_type[0] == FILE_TYPE_OBU) +#else if (input.aom_input_ctx->file_type == FILE_TYPE_OBU) +#endif obudec_free(input.obu_ctx); +#if CONFIG_MULTIVIEW_CORE + if (input.aom_input_ctx->file_type[0] != FILE_TYPE_WEBM) free(buf); +#else if (input.aom_input_ctx->file_type != FILE_TYPE_WEBM) free(buf); - +#endif if (scaled_img) aom_img_free(scaled_img); if (img_shifted) aom_img_free(img_shifted);
diff --git a/apps/aomenc.c b/apps/aomenc.c index 3f980ba..45b22b8 100644 --- a/apps/aomenc.c +++ b/apps/aomenc.c
@@ -268,6 +268,9 @@ #if CONFIG_WEBM_IO &g_av1_codec_arg_defs.stereo_mode, #endif +#if CONFIG_MULTIVIEW_CORE + &g_av1_codec_arg_defs.num_views, +#endif &g_av1_codec_arg_defs.timebase, &g_av1_codec_arg_defs.framerate, &g_av1_codec_arg_defs.global_error_resilient, @@ -749,6 +752,9 @@ global->csp = AOM_CSP_UNKNOWN; global->show_psnr = 0; global->step_frames = 1; +#if CONFIG_MULTIVIEW_CORE + global->num_views = 1; +#endif int cfg_included = 0; init_config(&global->encoder_config); @@ -811,6 +817,10 @@ global->show_psnr = arg_parse_int(&arg); else global->show_psnr = 2; +#if CONFIG_MULTIVIEW_CORE + } else if (arg_match(&arg, &g_av1_codec_arg_defs.num_views, argi)) { + global->num_views = arg_parse_uint(&arg); +#endif } else if (arg_match(&arg, &g_av1_codec_arg_defs.recontest, argi)) { global->test_decode = arg_parse_enum_or_int(&arg); } else if (arg_match(&arg, &g_av1_codec_arg_defs.framerate, argi)) { @@ -852,6 +862,57 @@ static void open_input_file(struct AvxInputContext *input, aom_chroma_sample_position_t csp) { +#if CONFIG_MULTIVIEW_CORE + for (int i = 0; i < input->num_views; i++) { + /* Parse certain options from the input file, if possible */ + input->file[i] = strcmp(input->filename[i], "-") + ? fopen(input->filename[i], "rb") + : set_binary_mode(stdin); + + if (!input->file[i]) fatal("Failed to open input file"); + + if (!fseeko(input->file[i], 0, SEEK_END)) { + /* Input file is seekable. Figure out how long it is, so we can get + * progress info. + */ + input->length = ftello(input->file[i]); + rewind(input->file[i]); + } + + /* Default to 1:1 pixel aspect ratio. */ + input->pixel_aspect_ratio.numerator = 1; + input->pixel_aspect_ratio.denominator = 1; + + /* For RAW input sources, these bytes will applied on the first frame + * in read_frame(). + */ + input->detect[i].buf_read = + fread(input->detect[i].buf, 1, 4, input->file[i]); + input->detect[i].position = 0; + + if (input->detect[i].buf_read == 4 && file_is_y4m(input->detect[i].buf)) { + if (y4m_input_open(&input->y4m[i], input->file[i], input->detect[i].buf, + 4, csp, input->only_i420) >= 0) { + input->file_type[i] = FILE_TYPE_Y4M; + input->width = input->y4m[i].pic_w; + input->height = input->y4m[i].pic_h; + input->pixel_aspect_ratio.numerator = input->y4m[i].par_n; + input->pixel_aspect_ratio.denominator = input->y4m[i].par_d; + input->framerate.numerator = input->y4m[i].fps_n; + input->framerate.denominator = input->y4m[i].fps_d; + input->fmt = input->y4m[i].aom_fmt; + input->bit_depth = input->y4m[i].bit_depth; + input->color_range = input->y4m[i].color_range; + } else + fatal("Unsupported Y4M stream."); + } else if (input->detect[i].buf_read == 4 && + fourcc_is_ivf(input->detect[i].buf)) { + fatal("IVF is not supported as input."); + } else { + input->file_type[i] = FILE_TYPE_RAW; + } + } +#else /* Parse certain options from the input file, if possible */ input->file = strcmp(input->filename, "-") ? fopen(input->filename, "rb") : set_binary_mode(stdin); @@ -896,11 +957,19 @@ } else { input->file_type = FILE_TYPE_RAW; } +#endif } static void close_input_file(struct AvxInputContext *input) { +#if CONFIG_MULTIVIEW_CORE + for (int i = 0; i < input->num_views; i++) { + fclose(input->file[i]); + if (input->file_type[i] == FILE_TYPE_Y4M) y4m_input_close(&input->y4m[i]); + } +#else fclose(input->file); if (input->file_type == FILE_TYPE_Y4M) y4m_input_close(&input->y4m); +#endif } static struct stream_state *new_stream(struct AvxEncoderConfig *global, @@ -1166,6 +1235,10 @@ } else if (arg_match(&arg, &g_av1_codec_arg_defs.stereo_mode, argi)) { config->stereo_fmt = arg_parse_enum_or_int(&arg); #endif +#if CONFIG_MULTIVIEW_CORE + } else if (arg_match(&arg, &g_av1_codec_arg_defs.num_views, argi)) { + config->cfg.g_num_views = arg_parse_uint(&arg); +#endif } else if (arg_match(&arg, &g_av1_codec_arg_defs.timebase, argi)) { config->cfg.g_timebase = arg_parse_rational(&arg); validate_positive_rational(arg.name, &config->cfg.g_timebase); @@ -1428,12 +1501,24 @@ fprintf(stdout, "Codec : %s\n", aom_codec_iface_name(global->codec)); fprintf(stdout, "Executable : aomenc %s\n", ENV_BITS); +#if CONFIG_MULTIVIEW_CORE + fprintf(stdout, "Number of views : %d \n", cfg->g_num_views); + for (int i = 0; i < input->num_views; i++) { + fprintf(stdout, "Input file%2d : %s\n", i, + input->filename[i]); + } +#else fprintf(stdout, "Input file : %s\n", input->filename); +#endif fprintf(stdout, "Output file : %s\n", stream->config.out_fn); fprintf(stdout, "Input format : %s, %s, %dx%d, %3.1f FPS, %d bit\n", +#if CONFIG_MULTIVIEW_CORE + file_type_to_string(input->file_type[0]), +#else file_type_to_string(input->file_type), +#endif image_format_to_string(input->fmt), input->width, input->height, (double)global->framerate.num / (double)global->framerate.den, input->bit_depth); @@ -1441,6 +1526,18 @@ print_frames_to_code(stdout, stream, global); fprintf(stdout, "Operating bit depth : %d\n", cfg->g_bit_depth); fprintf(stdout, "Num of coding passes : %d\n", global->passes); +#if CONFIG_MULTIVIEW_CORE + fprintf(stdout, "Lag in frames : %d\n", + cfg->g_lag_in_frames / cfg->g_num_views); + if (cfg->kf_min_dist != cfg->kf_max_dist) { + fprintf(stdout, "Key frame distance : %d - %d\n", + cfg->kf_min_dist / cfg->g_num_views, + cfg->kf_max_dist / cfg->g_num_views); + } else { + fprintf(stdout, "Key frame distance : %d\n", + cfg->kf_min_dist / cfg->g_num_views); + } +#else fprintf(stdout, "Lag in frames : %d\n", cfg->g_lag_in_frames); if (cfg->kf_min_dist != cfg->kf_max_dist) { @@ -1449,6 +1546,7 @@ } else { fprintf(stdout, "Key frame distance : %d\n", cfg->kf_min_dist); } +#endif if (encoder_cfg->superblock_size != 0) { fprintf(stdout, "Super block size : %d\n", encoder_cfg->superblock_size); @@ -1811,7 +1909,6 @@ aom_codec_pts_t frame_start, next_frame_start; struct aom_codec_enc_cfg *cfg = &stream->config.cfg; struct aom_usec_timer timer; - frame_start = (cfg->g_timebase.den * (int64_t)(frames_in - 1) * global->framerate.den) / cfg->g_timebase.num / global->framerate.num; @@ -2129,8 +2226,25 @@ die("only support ivf output format while large-scale-tile=1\n"); } +#if CONFIG_MULTIVIEW_CORE + /* read filenames for each view */ + input.num_views = global.num_views; + for (int i = 0; i < input.num_views; i++) { + input.filename[i] = argv[i]; + printf("Filename inputted : %s \n", input.filename[i]); + } + FOREACH_STREAM(stream, streams) { + stream->config.cfg.g_num_views = input.num_views; + stream->config.cfg.g_lag_in_frames *= input.num_views; + stream->config.cfg.kf_min_dist *= input.num_views; + stream->config.cfg.kf_max_dist *= input.num_views; + if (stream->config.cfg.kf_min_dist == 1) stream->config.cfg.kf_min_dist = 0; + if (stream->config.cfg.kf_max_dist == 1) stream->config.cfg.kf_max_dist = 0; + } +#else /* Handle non-option arguments */ input.filename = argv[0]; +#endif FOREACH_STREAM(stream, streams) { if (stream->config.recon_fn != NULL) { @@ -2139,10 +2253,20 @@ } } +#if CONFIG_MULTIVIEW_CORE + /* check input files for each view */ + for (int i = 0; i < input.num_views; i++) { + if (!input.filename[i]) { + fprintf(stderr, "No input file specified!\n"); + usage_exit(); + } + } +#else if (!input.filename) { fprintf(stderr, "No input file specified!\n"); usage_exit(); } +#endif /* Decide if other chroma subsamplings than 4:2:0 are supported */ if (get_fourcc_by_aom_encoder(global.codec) == AV1_FOURCC) @@ -2221,6 +2345,37 @@ } break; case 2: +#if CONFIG_MULTIVIEW_CORE + for (int i = 0; i < input.num_views; i++) { + if (input.bit_depth < 12 && (input.fmt == AOM_IMG_FMT_I444 || + input.fmt == AOM_IMG_FMT_I44416)) { + stream->config.cfg.g_profile = 1; + profile_updated = 1; + } else if (input.bit_depth < 12 && + (input.fmt == AOM_IMG_FMT_I420 || + input.fmt == AOM_IMG_FMT_I42016)) { + stream->config.cfg.g_profile = 0; + profile_updated = 1; + } else if (input.bit_depth == 12 && + input.file_type[i] == FILE_TYPE_Y4M) { + // Note that here the input file values for chroma subsampling + // are used instead of those from the command line. + AOM_CODEC_CONTROL_TYPECHECKED(&stream->encoder, + AV1E_SET_CHROMA_SUBSAMPLING_X, + input.y4m[i].dst_c_dec_h >> 1); + AOM_CODEC_CONTROL_TYPECHECKED(&stream->encoder, + AV1E_SET_CHROMA_SUBSAMPLING_Y, + input.y4m[i].dst_c_dec_v >> 1); + } else if (input.bit_depth == 12 && + input.file_type[i] == FILE_TYPE_RAW) { + AOM_CODEC_CONTROL_TYPECHECKED(&stream->encoder, + AV1E_SET_CHROMA_SUBSAMPLING_X, + stream->chroma_subsampling_x); + AOM_CODEC_CONTROL_TYPECHECKED(&stream->encoder, + AV1E_SET_CHROMA_SUBSAMPLING_Y, + stream->chroma_subsampling_y); + } +#else if (input.bit_depth < 12 && (input.fmt == AOM_IMG_FMT_I444 || input.fmt == AOM_IMG_FMT_I44416)) { stream->config.cfg.g_profile = 1; @@ -2248,6 +2403,7 @@ AOM_CODEC_CONTROL_TYPECHECKED(&stream->encoder, AV1E_SET_CHROMA_SUBSAMPLING_Y, stream->chroma_subsampling_y); +#endif } break; default: break; @@ -2329,6 +2485,17 @@ stream->config.cfg.g_timebase.num = global.framerate.den; } +#if CONFIG_MULTIVIEW_CORE + for (int i = 0; i < input.num_views; i++) { + if (input.file_type[i] == FILE_TYPE_Y4M) + /*The Y4M reader does its own allocation. + Just initialize this here to avoid problems if we never read any + frames.*/ + memset(&raw, 0, sizeof(raw)); + else + aom_img_alloc(&raw, input.fmt, input.width, input.height, 32); + } +#else if (input.file_type == FILE_TYPE_Y4M) /*The Y4M reader does its own allocation. Just initialize this here to avoid problems if we never read any @@ -2336,7 +2503,7 @@ memset(&raw, 0, sizeof(raw)); else aom_img_alloc(&raw, input.fmt, input.width, input.height, 32); - +#endif FOREACH_STREAM(stream, streams) { stream->rate_hist = init_rate_histogram(&stream->config.cfg, &global.framerate); @@ -2362,8 +2529,14 @@ // that test frameworks should use, when they want deterministic output // from the container format). if (stream->config.write_webm && !stream->webm_ctx.debug) { +#if CONFIG_MULTIVIEW_CORE + const int i = 0; + encoder_settings = extract_encoder_settings( + aom_codec_version_str(), argv_, argc, input.filename[i]); +#else encoder_settings = extract_encoder_settings( aom_codec_version_str(), argv_, argc, input.filename); +#endif if (encoder_settings == NULL) { fprintf( stderr, @@ -2409,8 +2582,19 @@ // The limit iterator will stop returning frames after the N-th. StreamIter limit_stream; +#if CONFIG_MULTIVIEW_CORE + int limit = global.limit * input.num_views; + limit_stream_iter_init(&limit_stream, &step_stream, limit); + int view_id = 0; +#else limit_stream_iter_init(&limit_stream, &step_stream, global.limit); +#endif while (frame_avail || got_data) { +#if CONFIG_MULTIVIEW_CORE + view_id = seen_frames % input.num_views; + orig_stream.view_id = view_id; + limit_stream.view_id = view_id; +#endif frame_avail = read_stream_iter(&limit_stream, &raw); if (frame_avail) { seen_frames++; @@ -2432,6 +2616,9 @@ frame_to_encode = &raw; } assert(frame_to_encode->fmt & AOM_IMG_FMT_HIGHBITDEPTH); +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" frame_avail:%d seen_frames=%d \n", frame_avail, seen_frames); +#endif FOREACH_STREAM(stream, streams) { encode_frame(stream, &global, frame_avail ? frame_to_encode : NULL, seen_frames);
diff --git a/apps/aomenc.h b/apps/aomenc.h index f243891..57f7397 100644 --- a/apps/aomenc.h +++ b/apps/aomenc.h
@@ -40,6 +40,9 @@ int limit; int skip_frames; int step_frames; +#if CONFIG_MULTIVIEW_CORE + int num_views; +#endif int show_psnr; enum TestDecodeFatality test_decode; int have_framerate;
diff --git a/av1/arg_defs.c b/av1/arg_defs.c index 582d7a5..334a1e1 100644 --- a/av1/arg_defs.c +++ b/av1/arg_defs.c
@@ -213,6 +213,10 @@ .stereo_mode = ARG_DEF_ENUM(NULL, "stereo-mode", 1, "Stereo 3D video format", stereo_mode_enum), #endif +#if CONFIG_MULTIVIEW_CORE + .num_views = + ARG_DEF(NULL, "num-views", 1, "Number of views for multi-layered coding"), +#endif .timebase = ARG_DEF(NULL, "timebase", 1, "Output timestamp precision (fractional seconds)"), .global_error_resilient = ARG_DEF(NULL, "global-error-resilient", 1,
diff --git a/av1/arg_defs.h b/av1/arg_defs.h index 47ed6dc..be69dd7 100644 --- a/av1/arg_defs.h +++ b/av1/arg_defs.h
@@ -73,6 +73,9 @@ #if CONFIG_WEBM_IO arg_def_t stereo_mode; #endif +#if CONFIG_MULTIVIEW_CORE + arg_def_t num_views; +#endif arg_def_t timebase; arg_def_t global_error_resilient; arg_def_t lag_in_frames;
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index ca3d56e..be358ea 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -36,6 +36,10 @@ #include "aom_dsp/psnr.h" #include "aom_ports/aom_timer.h" +#if CONFIG_MULTIVIEW_CORE +#include "common/tools_common.h" +#endif + #define MAG_SIZE (4) struct av1_extracfg { @@ -664,7 +668,9 @@ RANGE_CHECK(cfg, g_timebase.den, 1, 1000000000); RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den); RANGE_CHECK_HI(cfg, g_profile, MAX_PROFILES - 1); - +#if CONFIG_MULTIVIEW_CORE + RANGE_CHECK(cfg, g_num_views, 1, NUM_LAYERS_MAX); +#endif RANGE_CHECK(cfg, g_bit_depth, AOM_BITS_8, AOM_BITS_12); RANGE_CHECK(cfg, g_input_bit_depth, AOM_BITS_8, AOM_BITS_12); @@ -1263,6 +1269,11 @@ frm_dim_cfg->render_width = extra_cfg->render_width; frm_dim_cfg->render_height = extra_cfg->render_height; +#if CONFIG_MULTIVIEW_CORE + // Set number of views + input_cfg->num_views = cfg->g_num_views; +#endif + // Set input video related configuration. input_cfg->input_bit_depth = cfg->g_input_bit_depth; // guess a frame rate if out of whack, use 30 @@ -1455,6 +1466,9 @@ } else { // auto-selected qp offset q_cfg->fixed_qp_offsets[i] = get_modeled_qp_offset( rc_cfg->qp, i, tool_cfg->bit_depth, q_cfg->q_based_qp_offsets); +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" qp-offset[%d]=%2.2f\n", i, q_cfg->fixed_qp_offsets[i]); +#endif } } else { q_cfg->fixed_qp_offsets[i] = -1.0; @@ -1859,7 +1873,6 @@ const SubGOPCfg *const subgop_cfg = gf_group->subgop_cfg; subgop_info->gf_interval = cpi->rc.baseline_gf_interval; subgop_info->frames_to_key = cpi->rc.frames_to_key; - // As key frame is not part of sub-gop configuration, // parameters are assigned separately. if (cpi->common.current_frame.frame_type == KEY_FRAME) { @@ -2857,23 +2870,38 @@ if (!cm->show_existing_frame) { // Get reference frame information int ref_poc[INTER_REFS_PER_FRAME]; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + int ref_view[INTER_REFS_PER_FRAME]; +#endif for (int ref_frame = 0; ref_frame < INTER_REFS_PER_FRAME; ++ref_frame) { const int ref_idx = ref_frame; const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame); +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + ref_view[ref_idx] = buf ? (int)buf->view_id : -1; + ref_poc[ref_idx] = buf ? (int)buf->display_order_hint : -1; +#else ref_poc[ref_idx] = buf ? (int)buf->absolute_poc : -1; ref_poc[ref_idx] = (ref_poc[ref_idx] == (int)cm->cur_frame->absolute_poc) ? -1 : ref_poc[ref_idx]; +#endif } if (cpi->b_calculate_psnr >= 1) { const bool use_hbd_psnr = (cpi->b_calculate_psnr == 2); fprintf(stdout, +#if CONFIG_MULTIVIEW_CORE + "POC:%6d, DOH:%3d, View:%2d [%s][Level:%d][Q:%3d]: %10" PRIu64 +#else "POC:%6d [%s][Level:%d][Q:%3d]: %10" PRIu64 +#endif " Bytes, " "%6.1fms, %2.4f dB(Y), %2.4f dB(U), " "%2.4f dB(V), " "%2.4f dB(Avg)", cm->cur_frame->absolute_poc, +#if CONFIG_MULTIVIEW_CORE + cm->cur_frame->display_order_hint, cm->cur_frame->view_id, +#endif frameType[cm->current_frame.frame_type], cm->cur_frame->pyramid_level, base_qindex, (uint64_t)frame_size, cx_time / 1000.0, use_hbd_psnr ? psnr.psnr_hbd[1] : psnr.psnr[1], @@ -2893,7 +2921,11 @@ fprintf(stdout, " ["); for (int ref_idx = 0; ref_idx < INTER_REFS_PER_FRAME; ++ref_idx) { +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + fprintf(stdout, "(%d,%d)", ref_poc[ref_idx], ref_view[ref_idx]); +#else fprintf(stdout, "%3d,", ref_poc[ref_idx]); +#endif } fprintf(stdout, "]\n"); } @@ -4260,11 +4292,14 @@ 0, // g_threads 0, // g_profile - 320, // g_w - 240, // g_h - 0, // g_limit - 0, // g_forced_max_frame_width - 0, // g_forced_max_frame_height + 320, // g_w + 240, // g_h + 0, // g_limit + 0, // g_forced_max_frame_width + 0, // g_forced_max_frame_height +#if CONFIG_MULTIVIEW_CORE + 1, // g_num_views +#endif AOM_BITS_8, // g_bit_depth 8, // g_input_bit_depth
diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c index f3a724c..3ed2131 100644 --- a/av1/av1_dx_iface.c +++ b/av1/av1_dx_iface.c
@@ -292,6 +292,9 @@ si->w = 0; si->h = 0; si->is_kf = 0; // is_kf indicates whether the current packet contains a RAP +#if CONFIG_MULTIVIEW_CORE + si->number_views = 0; +#endif ObuHeader obu_header; memset(&obu_header, 0, sizeof(obu_header)); @@ -336,6 +339,12 @@ status = parse_operating_points(&rb, reduced_still_picture_hdr, si); if (status != AOM_CODEC_OK) return status; +#if CONFIG_MULTIVIEW_CORE + si->number_views = + aom_rb_read_literal(&rb, 8) + 1; // TODO: @hegilmez redesign later + // or change where this is signaled +#endif + int num_bits_width = aom_rb_read_literal(&rb, 4) + 1; int num_bits_height = aom_rb_read_literal(&rb, 4) + 1; int max_frame_width = aom_rb_read_literal(&rb, num_bits_width) + 1; @@ -699,14 +708,26 @@ // show_existing_frame if (pbi->common.seq_params.order_hint_info.enable_order_hint && pbi->common.seq_params.enable_frame_output_order) { +#if !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST if (!pbi->common.show_existing_frame) decrease_ref_count(pbi->output_frames[0], pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } else { for (size_t j = 0; j < pbi->num_output_frames; j++) { decrease_ref_count(pbi->output_frames[j], pool); } } pbi->num_output_frames = 0; +#if CONFIG_MULTIVIEW_CORE + for (size_t j = 0; j < REF_FRAMES; j++) { + pbi->output_view_ids[j] = -1; + } +#endif +#if CONFIG_MULTIVIEW_CORE + for (size_t j = 0; j < REF_FRAMES; j++) { + pbi->display_order_hint_ids[j] = -1; + } +#endif unlock_buffer_pool(pool); for (size_t j = 0; j < ctx->num_grain_image_frame_buffers; j++) { pool->release_fb_cb(pool->cb_priv, &ctx->grain_image_frame_buffers[j]); @@ -899,6 +920,7 @@ AV1Decoder *const pbi = frame_worker_data->pbi; AV1_COMMON *const cm = &pbi->common; CommonTileParams *const tiles = &cm->tiles; + // Wait for the frame from worker thread. if (winterface->sync(worker)) { // Check if worker has received any frames. @@ -911,6 +933,11 @@ if (av1_get_raw_frame(frame_worker_data->pbi, *index, &sd, &grain_params) == 0) { RefCntBuffer *const output_frame_buf = pbi->output_frames[*index]; +#if CONFIG_MULTIVIEW_CORE + int view_id_ctx = pbi->output_view_ids[*index]; + ctx->img.view_id = view_id_ctx; + ctx->img.display_order_hint = pbi->display_order_hint_ids[*index]; +#endif ctx->last_show_frame = output_frame_buf; if (ctx->need_resync) return NULL; aom_img_remove_metadata(&ctx->img); @@ -969,6 +996,7 @@ img = &ctx->img; img->temporal_id = cm->temporal_layer_id; img->spatial_id = cm->spatial_layer_id; + if (pbi->skip_film_grain) grain_params->apply_grain = 0; aom_image_t *res = add_grain_if_needed(ctx, img, &ctx->image_with_grain, grain_params);
diff --git a/av1/common/av1_common_int.h b/av1/common/av1_common_int.h index 6c2cb97..4e1be1b 100644 --- a/av1/common/av1_common_int.h +++ b/av1/common/av1_common_int.h
@@ -242,6 +242,10 @@ unsigned int order_hint; int ref_order_hints[INTER_REFS_PER_FRAME]; int ref_display_order_hint[INTER_REFS_PER_FRAME]; +#if CONFIG_MULTIVIEW_CORE + int view_id; + int ref_view_ids[INTER_REFS_PER_FRAME]; +#endif // These variables are used only in encoder and compare the absolute // display order hint to compute the relative distance and overcome @@ -251,6 +255,9 @@ unsigned int absolute_poc; // Frame's level within the hierarchical structure unsigned int pyramid_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + unsigned int temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST #if CONFIG_IMPROVED_GLOBAL_MOTION // How many ref frames did this frame use? @@ -315,8 +322,14 @@ typedef struct { FRAME_TYPE frame_type; int pyr_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + int temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST int disp_order; int base_qindex; +#if CONFIG_MULTIVIEW_CORE + int view_id; +#endif } RefFrameMapPair; #endif // CONFIG_PRIMARY_REF_FRAME_OPT @@ -562,6 +575,9 @@ // Operating point info. int operating_points_cnt_minus_1; int operating_point_idc[MAX_NUM_OPERATING_POINTS]; +#if CONFIG_MULTIVIEW_CORE + int num_views; // for multiview support +#endif int timing_info_present; aom_timing_info_t timing_info; uint8_t decoder_model_info_present_flag; @@ -591,9 +607,15 @@ unsigned int display_order_hint; // Frame's level within the hierarchical structure unsigned int pyramid_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + unsigned int temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST unsigned int absolute_poc; unsigned int key_frame_number; unsigned int frame_number; +#if CONFIG_MULTIVIEW_CORE + int view_id; +#endif SkipModeInfo skip_mode_info; int refresh_frame_flags; // Which ref frames are overwritten by this frame } CurrentFrame; @@ -1741,6 +1763,17 @@ */ int spatial_layer_id; +#if CONFIG_MULTIVIEW_CORE + /*! + * Number of multiple layers + */ + unsigned int number_layers; + /*! + * Multi-layer ID of this frame + * (in the range 0 ... (number_layers - 1)). + */ + int layer_id; +#endif /*! * Weights for IBP of directional modes. */ @@ -1772,6 +1805,11 @@ FILE *fDecCoeffLog; #endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + FILE *fEncMultiviewLog; + FILE *fDecMultiviewLog; +#endif + /*! * Flag to indicate if current frame has backward ref frame */ @@ -1960,6 +1998,19 @@ ++rhs_ptr->ref_count; } +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +static INLINE void set_frame_buffer_invalid(RefCntBuffer **lhs_ptr) { + RefCntBuffer *const old_ptr = *lhs_ptr; + if (old_ptr != NULL) { + assert(old_ptr->ref_count > 0); + // One less reference to the buffer at 'old_ptr', so decrease ref count. + --old_ptr->ref_count; + } + + *lhs_ptr = NULL; +} +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + static INLINE int frame_is_intra_only(const AV1_COMMON *const cm) { return cm->current_frame.frame_type == KEY_FRAME || cm->current_frame.frame_type == INTRA_ONLY_FRAME; @@ -1995,6 +2046,156 @@ return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL; } +#if CONFIG_MULTIVIEW_DEBUG +static INLINE void debug_print_multiview_curr_frame( + const AV1_COMMON *const cm) { + const CurrentFrame *const cf = &cm->current_frame; + const char frame_type_str[4][25] = { "KEY_FRAME", "INTER_FRAME", + "INTRA_ONLY_FRAME", "S_FRAME" }; + const char ref_mode_str[4][50] = { "SINGLE_REFERENCE", "COMPOUND_REFERENCE", + "REFERENCE_MODE_SELECT", + "REFERENCE_MODES" }; + printf("frame_type=%s,", frame_type_str[cf->frame_type]); + // printf("reference_mode=%s,", ref_mode_str[cf->reference_mode]); + printf("order_hint=%2d, ", cf->order_hint); + printf("display_order_hint=%2d, ", cf->display_order_hint); + // printf("pyramid_level=%2d, ", cf->pyramid_level); // encoder only + // printf("absolute_poc=%2d, ", cf->absolute_poc); // encoder only + // printf("key_frame_number=%2d, ", cf->key_frame_number); + // printf("frame_number=%2d, ", cf->frame_number); + printf("view_id=%2d, ", cf->view_id); + printf("refresh_frame_flags=%2d,", cf->refresh_frame_flags); + printf("num_total_refs=%2d,", cm->ref_frames_info.num_total_refs); + printf("num_cur_refs=%2d,", cm->ref_frames_info.num_cur_refs); + printf("num_past_refs=%2d,", cm->ref_frames_info.num_past_refs); + printf("num_future_refs=%2d", cm->ref_frames_info.num_future_refs); + + printf("\n"); +} + +static INLINE void debug_print_buffer_state(const AV1_COMMON *const cm) { + printf(" [ "); + for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { + RefCntBuffer *const buf = cm->ref_frame_map[ref_idx]; + if (buf == NULL) + printf("%d:(**,**,**) ", ref_idx); + else + printf("%d:(%d,%d,%d) ", ref_idx, buf->view_id, buf->pyramid_level, + buf->display_order_hint); + } + printf("]\n"); +} + +static INLINE void debug_print_multiview_buf_refs(const AV1_COMMON *const cm) { + const RefCntBuffer *const cf = cm->cur_frame; + MV_REFERENCE_FRAME ref_frame; + printf("(View,OH,DOH):(%2d,%2d,%2d) ", cf->view_id, cf->order_hint, + cf->display_order_hint); + printf(" num_ref_frames/ref_frames_info.num_total_refs=%d/%d ", + cf->num_ref_frames, cm->ref_frames_info.num_total_refs); + printf("["); + for (ref_frame = 0; ref_frame < INTER_REFS_PER_FRAME; ++ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + printf("%2d:", map_idx); + printf("(%2d,", cf->ref_view_ids[ref_frame]); + printf("%2d,", cf->ref_order_hints[ref_frame]); + printf("%2d) ", cf->ref_display_order_hint[ref_frame]); + } + printf("]\n"); +} + +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES +static INLINE void logfile_multiview_curr_frame(const AV1_COMMON *const cm, + FILE *const file) { + const CurrentFrame *const cf = &cm->current_frame; + const char frame_type_str[4][25] = { "KEY_FRAME", "INTER_FRAME", + "INTRA_ONLY_FRAME", "S_FRAME" }; + const char ref_mode_str[4][50] = { "SINGLE_REFERENCE", "COMPOUND_REFERENCE", + "REFERENCE_MODE_SELECT", + "REFERENCE_MODES" }; + fprintf(file, "frame_type=%s,", frame_type_str[cf->frame_type]); + // fprintf(file,"reference_mode=%s,", ref_mode_str[cf->reference_mode]); + fprintf(file, "order_hint=%2d, ", cf->order_hint); + fprintf(file, "display_order_hint=%2d, ", cf->display_order_hint); + // fprintf(file,"pyramid_level=%2d, ", cf->pyramid_level); // encoder only + // fprintf(file,"absolute_poc=%2d, ", cf->absolute_poc); // encoder only + // fprintf(file,"key_frame_number=%2d, ", cf->key_frame_number); + // fprintf(file,"frame_number=%2d, ", cf->frame_number); + fprintf(file, "view_id=%2d, ", cf->view_id); + fprintf(file, "refresh_frame_flags=%2d,", cf->refresh_frame_flags); + fprintf(file, "num_total_refs=%2d,", cm->ref_frames_info.num_total_refs); + fprintf(file, "num_cur_refs=%2d,", cm->ref_frames_info.num_cur_refs); + fprintf(file, "num_past_refs=%2d,", cm->ref_frames_info.num_past_refs); + fprintf(file, "num_future_refs=%2d", cm->ref_frames_info.num_future_refs); + fprintf(file, "\n"); +} + +static INLINE void logfile_multiview_buf_refs(const AV1_COMMON *const cm, + FILE *const file) { + const RefCntBuffer *const cf = cm->cur_frame; + MV_REFERENCE_FRAME ref_frame; + fprintf(file, "(View,OH,DOH):(%2d,%2d,%2d) ", cf->view_id, cf->order_hint, + cf->display_order_hint); + fprintf(file, " num_ref_frames/ref_frames_info.num_total_refs=%d/%d ", + cf->num_ref_frames, cm->ref_frames_info.num_total_refs); + fprintf(file, "["); + for (ref_frame = 0; ref_frame < INTER_REFS_PER_FRAME; ++ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + fprintf(file, "%2d:", map_idx); + fprintf(file, "(%2d,", cf->ref_view_ids[ref_frame]); + fprintf(file, "%2d,", cf->ref_order_hints[ref_frame]); + fprintf(file, "%2d) ", cf->ref_display_order_hint[ref_frame]); + } + fprintf(file, "]\n"); +} + +static INLINE void logfile_buffer_state(const AV1_COMMON *const cm, + FILE *const file) { + fprintf(file, " [ "); + for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { + RefCntBuffer *const buf = cm->ref_frame_map[ref_idx]; + if (buf == NULL) + fprintf(file, "%d:(**,**,**) ", ref_idx); + else + fprintf(file, "%d:(%d,%d,%d) ", ref_idx, buf->view_id, buf->order_hint, + buf->display_order_hint); + } + fprintf(file, "]\n"); +} + +static INLINE void logfile_primary_ref_info(const AV1_COMMON *const cm, + FILE *const file) { + const int idx = cm->features.primary_ref_frame; + const int map_idx = get_ref_frame_map_idx(cm, idx); + const RefCntBuffer *const ref_buf = cm->ref_frame_map[map_idx]; + if (ref_buf == NULL) + fprintf(file, + " Primary Reference Info. : [idx,map_idx]:[%2d,%2d] --> " + "(N/A,N/A,N/A) \n ", + idx, map_idx); + else + fprintf(file, + " Primary Reference Info. : [idx,map_idx]:[%2d,%2d] --> " + "(%2d,%2d,%3d) \n ", + idx, map_idx, ref_buf->view_id, ref_buf->order_hint, + ref_buf->display_order_hint); +} + +// static INLINE void logfile_show_existing_frame(const AV1_COMMON *const cm, +// FILE *const file) { +// int show_existing_frame = cm->show_existing_frame; +// if (show_existing_frame) +// fprintf(file," Show existing frame : [idx,map_idx]:[%2d,%2d] --> +// (%2d,%2d,%3d) \n ", idx, map_idx, ref_buf->view_id, ref_buf->order_hint, +// ref_buf->display_order_hint ); +// +// RefCntBuffer *const frame_to_show = +// cm->ref_frame_map[cpi->existing_fb_idx_to_show]; +// } +#endif + +#endif + // Both const and non-const versions of this function are provided so that it // can be used with a const AV1_COMMON if needed. static INLINE const struct scale_factors *get_ref_scale_factors_const(
diff --git a/av1/common/enums.h b/av1/common/enums.h index 1238896..43b4cce 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h
@@ -1164,7 +1164,13 @@ #define WARP_EXTEND_CTX 3 #endif // CONFIG_OPTIMIZE_CTX_TIP_WARP +#if CONFIG_MULTIVIEW_EXTENDED_DPB +#define EXTENDED_INTER_REFS_PER_FRAME 8 +#define INTER_REFS_PER_FRAME (EXTENDED_INTER_REFS_PER_FRAME + 7) +#define EXTENDED_INTER_REFS_CONTEXT 6 +#else #define INTER_REFS_PER_FRAME 7 +#endif #define REF_FRAMES (INTER_REFS_PER_FRAME + 1) // NOTE: A limited number of unidirectional reference pairs can be signalled for // compound prediction. The use of skip mode, on the other hand, makes it @@ -1195,7 +1201,11 @@ #define NONE_FRAME INVALID_IDX #define AOM_REFFRAME_ALL ((1 << INTER_REFS_PER_FRAME) - 1) +#if CONFIG_MULTIVIEW_EXTENDED_DPB +#define REF_FRAMES_LOG2 4 +#else #define REF_FRAMES_LOG2 3 +#endif #define REFRESH_FRAME_ALL ((1 << REF_FRAMES) - 1) // REF_FRAMES for the cm->ref_frame_map array, 1 scratch frame for the new @@ -1218,7 +1228,7 @@ // Note: It includes single and compound references. So, it can take values from // NONE_FRAME to (MODE_CTX_REF_FRAMES - 1). Hence, it is not defined as an enum. -typedef int8_t MV_REFERENCE_FRAME; +typedef int16_t MV_REFERENCE_FRAME; #define MAX_LR_FLEX_SWITCHABLE_BITS 4
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c index b830dca..363e113 100644 --- a/av1/common/mvref_common.c +++ b/av1/common/mvref_common.c
@@ -3751,6 +3751,13 @@ cm->cur_frame->display_order_hint = cm->current_frame.display_order_hint; cm->cur_frame->absolute_poc = cm->current_frame.absolute_poc; cm->cur_frame->pyramid_level = cm->current_frame.pyramid_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + cm->cur_frame->temporal_layer_id = cm->current_frame.temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + +#if CONFIG_MULTIVIEW_CORE + cm->cur_frame->view_id = cm->current_frame.view_id; +#endif MV_REFERENCE_FRAME ref_frame; for (ref_frame = 0; ref_frame < INTER_REFS_PER_FRAME; ++ref_frame) { @@ -3759,9 +3766,15 @@ cm->cur_frame->ref_order_hints[ref_frame] = buf->order_hint; cm->cur_frame->ref_display_order_hint[ref_frame] = buf->display_order_hint; +#if CONFIG_MULTIVIEW_CORE + cm->cur_frame->ref_view_ids[ref_frame] = buf->view_id; +#endif } else { cm->cur_frame->ref_order_hints[ref_frame] = -1; cm->cur_frame->ref_display_order_hint[ref_frame] = -1; +#if CONFIG_MULTIVIEW_CORE + cm->cur_frame->ref_view_ids[ref_frame] = -1; +#endif } } }
diff --git a/av1/common/mvref_common.h b/av1/common/mvref_common.h index daf62b2..991ed96 100644 --- a/av1/common/mvref_common.h +++ b/av1/common/mvref_common.h
@@ -329,10 +329,10 @@ // (2, 2), (2, 3), ..., (2, n-1), // ... // (n-1, n-1) -static INLINE int8_t single2comb(int n, const int8_t *const rf) { +static INLINE int16_t single2comb(int n, const int16_t *const rf) { assert(rf[0] < n && rf[1] < n); - int8_t rfr[2] = { rf[0], rf[1] }; + int16_t rfr[2] = { rf[0], rf[1] }; if (rf[1] < rf[0]) { rfr[0] = rf[1]; rfr[1] = rf[0]; @@ -347,7 +347,7 @@ // Converts a combined index in [0, n*(n+1)/2] to a pair of single // ref indices (rf) each in [0, n-1]. See comment above for order // of the combined indexing. -static INLINE void comb2single(int n, int8_t combindex, int8_t *rf) { +static INLINE void comb2single(int n, int16_t combindex, int16_t *rf) { assert(combindex < n * (n + 1) / 2); int i = n, j = n; rf[0] = 0; @@ -402,7 +402,7 @@ } #endif // CONFIG_SAME_REF_COMPOUND -static INLINE int8_t av1_ref_frame_type(const MV_REFERENCE_FRAME *const rf) { +static INLINE int16_t av1_ref_frame_type(const MV_REFERENCE_FRAME *const rf) { if (!is_inter_ref_frame(rf[0])) { // Intra or invalid return rf[0]; @@ -465,13 +465,13 @@ static INLINE int16_t av1_mode_context_pristine( const int16_t *const mode_context, const MV_REFERENCE_FRAME *const rf) { - const int8_t ref_frame = av1_ref_frame_type(rf); + const int16_t ref_frame = av1_ref_frame_type(rf); return mode_context[ref_frame]; } static INLINE int16_t av1_mode_context_analyzer( const int16_t *const mode_context, const MV_REFERENCE_FRAME *const rf) { - const int8_t ref_frame = av1_ref_frame_type(rf); + const int16_t ref_frame = av1_ref_frame_type(rf); if (!is_inter_ref_frame(rf[1])) return mode_context[ref_frame];
diff --git a/av1/common/pred_common.c b/av1/common/pred_common.c index 008b930..47e220d 100644 --- a/av1/common/pred_common.c +++ b/av1/common/pred_common.c
@@ -42,10 +42,19 @@ // Checks to see if a particular reference frame is already in the reference // frame map -static int is_in_ref_score(RefScoreData *map, int disp_order, int score, - int n_frames) { +static int is_in_ref_score(RefScoreData *map, int disp_order, +#if CONFIG_MULTIVIEW_CORE + int view_id, +#endif + int score, int n_frames) { for (int i = 0; i < n_frames; i++) { +#if CONFIG_MULTIVIEW_CORE + if (disp_order == map[i].disp_order && view_id == map[i].view_id && + score == map[i].score) + return 1; +#else if (disp_order == map[i].disp_order && score == map[i].score) return 1; +#endif } return 0; } @@ -147,24 +156,42 @@ if (cur_ref.disp_order == -1) continue; max_disp = AOMMAX(max_disp, cur_ref.disp_order); } - // Compute a score for each reference buffer for (int i = 0; i < REF_FRAMES; i++) { // Get reference frame buffer RefFrameMapPair cur_ref = ref_frame_map_pairs[i]; if (cur_ref.disp_order == -1) continue; const int ref_disp = cur_ref.disp_order; +#if CONFIG_MULTIVIEW_CORE + const int ref_view_id = cur_ref.view_id; +#endif // In error resilient mode, ref mapping must be independent of the // base_qindex to ensure decoding independency const int ref_base_qindex = cur_ref.base_qindex; +#if CONFIG_MULTIVIEW_CORE + const int layer_diff = cm->current_frame.view_id - cur_ref.view_id; + const int disp_diff = cur_frame_disp - ref_disp + layer_diff; +#else const int disp_diff = cur_frame_disp - ref_disp; +#endif int tdist = abs(disp_diff); +#if CONFIG_MULTIVIEW_CORE + // if (tdist == 0 && layer_diff != 0) { + // tdist = abs(layer_diff); + // disp_diff += layer_diff; + // } +#endif const int score = max_disp > cur_frame_disp ? ((tdist << DIST_WEIGHT_BITS) + ref_base_qindex) : temp_dist_score_lookup[AOMMIN(tdist, DECAY_DIST_CAP)] + AOMMAX(tdist - DECAY_DIST_CAP, 0) + ref_base_qindex; - if (is_in_ref_score(scores, ref_disp, score, n_ranked)) continue; + if (is_in_ref_score(scores, ref_disp, +#if CONFIG_MULTIVIEW_CORE + ref_view_id, +#endif + score, n_ranked)) + continue; scores[n_ranked].index = i; scores[n_ranked].score = score; @@ -200,7 +227,12 @@ // Fill any slots that are empty (should only happen for the first 7 frames) for (int i = 0; i < REF_FRAMES; i++) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (cm->remapped_ref_idx[i] == INVALID_IDX) + cm->remapped_ref_idx[i] = scores[0].index; +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST if (cm->remapped_ref_idx[i] == INVALID_IDX) cm->remapped_ref_idx[i] = 0; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } return n_ranked; } @@ -210,6 +242,9 @@ int idx; int disp_order; int base_qindex; +#if CONFIG_MULTIVIEW_CORE + int view_id; +#endif } PrimaryRefCand; // Check if one reference frame is better based on its distance to the current @@ -244,10 +279,18 @@ const RefFrameMapPair *ref_frame_map_pairs = cm->ref_frame_map_pairs; const int cur_frame_disp = cm->current_frame.display_order_hint; +#if CONFIG_MULTIVIEW_SEPARATE_DPB + const int curr_view_id = cm->current_frame.view_id; +#endif int i; +#if CONFIG_MULTIVIEW_CORE + PrimaryRefCand cand_lower_qp = { -1, -1, -1, -1 }; + PrimaryRefCand cand_higher_qp = { -1, -1, INT32_MAX, -1 }; +#else PrimaryRefCand cand_lower_qp = { -1, -1, -1 }; PrimaryRefCand cand_higher_qp = { -1, -1, INT32_MAX }; +#endif const OrderHintInfo *oh = &cm->seq_params.order_hint_info; for (i = 0; i < n_refs; i++) { @@ -255,7 +298,20 @@ RefFrameMapPair cur_ref = ref_frame_map_pairs[get_ref_frame_map_idx(cm, i)]; if (cur_ref.disp_order == -1) continue; if (cur_ref.frame_type != INTER_FRAME) continue; - +#if CONFIG_MULTIVIEW_SEPARATE_DPB + const int ref_view_id = cur_ref.view_id; + if (ref_view_id > curr_view_id) continue; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + const CurrentFrame *const cf = &cm->current_frame; + if (ref_view_id > curr_view_id) { + printf("Breaking view dependency: \n"); + printf("(index=%d): (View,Level,OH,DOH):(%d,%d,%d,%d) \n", i, cf->view_id, + cf->pyramid_level, cf->order_hint, cf->display_order_hint); + printf("Current reference-buffer ="); + debug_print_buffer_state(cm); + } +#endif +#endif const int ref_base_qindex = cur_ref.base_qindex; if (ref_base_qindex > cm->quant_params.base_qindex) { @@ -266,6 +322,9 @@ cand_higher_qp.idx = i; cand_higher_qp.base_qindex = ref_base_qindex; cand_higher_qp.disp_order = cur_ref.disp_order; +#if CONFIG_MULTIVIEW_CORE + cand_higher_qp.view_id = cur_ref.view_id; +#endif } } else { if ((ref_base_qindex > cand_lower_qp.base_qindex) || @@ -275,6 +334,9 @@ cand_lower_qp.idx = i; cand_lower_qp.base_qindex = ref_base_qindex; cand_lower_qp.disp_order = cur_ref.disp_order; +#if CONFIG_MULTIVIEW_CORE + cand_lower_qp.view_id = cur_ref.view_id; +#endif } } }
diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h index e9a138d..ed39464 100644 --- a/av1/common/pred_common.h +++ b/av1/common/pred_common.h
@@ -25,6 +25,9 @@ #if !CONFIG_PRIMARY_REF_FRAME_OPT typedef struct { int pyr_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + int temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST int disp_order; int base_qindex; } RefFrameMapPair; @@ -42,10 +45,31 @@ // Get reference frame buffer const RefCntBuffer *const buf = cm->ref_frame_map[map_idx]; if (ref_frame_map_pairs[map_idx].disp_order == -1) continue; - if (buf == NULL) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + // If the temporal_layer_id of the reference frame is greater than + // the temporal_layer_id of the current frame, the reference frame + // is not included into the list of ref_frame_map_pairs[]. + if (buf == NULL || +#if CONFIG_MULTIVIEW_SEPARATE_DPB + buf->view_id > cm->current_frame.view_id || +#endif + buf->temporal_layer_id > cm->current_frame.temporal_layer_id) { +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (buf == NULL +#if CONFIG_MULTIVIEW_SEPARATE_DPB + || buf->view_id > cm->current_frame.view_id +#endif + ) { +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST ref_frame_map_pairs[map_idx].disp_order = -1; ref_frame_map_pairs[map_idx].pyr_level = -1; ref_frame_map_pairs[map_idx].base_qindex = -1; +#if CONFIG_MULTIVIEW_SEPARATE_DPB + ref_frame_map_pairs[map_idx].view_id = -1; +#endif +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + ref_frame_map_pairs[map_idx].temporal_layer_id = -1; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST continue; } else if (buf->ref_count > 1) { // Once the keyframe is coded, the slots in ref_frame_map will all @@ -59,12 +83,24 @@ ref_frame_map_pairs[idx2].disp_order = -1; ref_frame_map_pairs[idx2].pyr_level = -1; ref_frame_map_pairs[idx2].base_qindex = -1; +#if CONFIG_MULTIVIEW_SEPARATE_DPB + ref_frame_map_pairs[idx2].view_id = -1; +#endif +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + ref_frame_map_pairs[map_idx].temporal_layer_id = -1; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } } } ref_frame_map_pairs[map_idx].disp_order = (int)buf->display_order_hint; ref_frame_map_pairs[map_idx].pyr_level = buf->pyramid_level; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + ref_frame_map_pairs[map_idx].temporal_layer_id = buf->temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST ref_frame_map_pairs[map_idx].base_qindex = buf->base_qindex; +#if CONFIG_MULTIVIEW_SEPARATE_DPB + ref_frame_map_pairs[map_idx].view_id = buf->view_id; +#endif #if CONFIG_PRIMARY_REF_FRAME_OPT ref_frame_map_pairs[map_idx].frame_type = buf->frame_type; #endif // CONFIG_PRIMARY_REF_FRAME_OPT @@ -84,6 +120,9 @@ int disp_order; // Quality of the reference frame int base_qindex; +#if CONFIG_MULTIVIEW_CORE + int view_id; +#endif } RefScoreData; /*!\endcond */ @@ -495,8 +534,13 @@ MV_REFERENCE_FRAME ref, int num_total_refs) { assert((ref + 1) < num_total_refs); +#if CONFIG_MULTIVIEW_EXTENDED_DPB + return xd->tile_ctx->single_ref_cdf[av1_get_ref_pred_context( + xd, ref, num_total_refs)][clamp(ref, 0, EXTENDED_INTER_REFS_CONTEXT - 1)]; +#else return xd->tile_ctx ->single_ref_cdf[av1_get_ref_pred_context(xd, ref, num_total_refs)][ref]; +#endif } // This function checks whether the previously coded reference frame is on the @@ -525,13 +569,27 @@ assert(bit_type < COMPREF_BIT_TYPES); assert(IMPLIES(n_bits == 0, ref < RANKED_REF0_TO_PRUNE - 1)); #endif // CONFIG_SAME_REF_COMPOUND - return n_bits == 0 ? xd->tile_ctx->comp_ref0_cdf[av1_get_ref_pred_context( - xd, ref, num_total_refs)][ref] - : xd->tile_ctx->comp_ref1_cdf[av1_get_ref_pred_context( -#if CONFIG_SAME_REF_COMPOUND - xd, ref, num_total_refs)][bit_type][ref]; + return n_bits == 0 + ? xd->tile_ctx->comp_ref0_cdf[av1_get_ref_pred_context( +#if CONFIG_MULTIVIEW_EXTENDED_DPB + xd, ref, + num_total_refs)][clamp(ref, 0, + EXTENDED_INTER_REFS_CONTEXT - 1)] #else - xd, ref, num_total_refs)][bit_type][ref - 1]; + xd, ref, num_total_refs)][ref] +#endif + : xd->tile_ctx + ->comp_ref1_cdf[av1_get_ref_pred_context( +#if CONFIG_SAME_REF_COMPOUND +#if CONFIG_MULTIVIEW_EXTENDED_DPB + xd, ref, num_total_refs)][bit_type] + [clamp(ref, 0, + EXTENDED_INTER_REFS_CONTEXT - 1)]; +#else + xd, ref, num_total_refs)][bit_type][ref]; +#endif +#else + xd, ref, num_total_refs)][bit_type][ref - 1]; #endif // CONFIG_SAME_REF_COMPOUND }
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 21953cf..7737700 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -3854,8 +3854,9 @@ tip_frame_buf, cm->width, cm->height, seq_params->subsampling_x, seq_params->subsampling_y, AOM_DEC_BORDER_IN_PIXELS, cm->features.byte_alignment, NULL, NULL, NULL, false)) { - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate frame buffer"); + aom_internal_error( + &cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer: setup_tip_frame_size 0"); } if (tip_frame_buf) { @@ -3877,8 +3878,9 @@ tip_frame_buf, cm->width, cm->height, seq_params->subsampling_x, seq_params->subsampling_y, AOM_DEC_BORDER_IN_PIXELS, cm->features.byte_alignment, NULL, NULL, NULL, false)) { - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate frame buffer"); + aom_internal_error( + &cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer: setup_tip_frame_size 1"); } if (tip_frame_buf) { @@ -3908,7 +3910,7 @@ pool->get_fb_cb, pool->cb_priv, false)) { unlock_buffer_pool(pool); aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate frame buffer"); + "Failed to allocate frame buffer: setup_buffer_pool"); } unlock_buffer_pool(pool); @@ -6695,7 +6697,11 @@ BufferPool *const pool = cm->buffer_pool; for (int i = 0; i < REF_FRAMES; i++) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + set_ref_count_zero(cm->ref_frame_map[i], pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST decrease_ref_count(cm->ref_frame_map[i], pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST cm->ref_frame_map[i] = NULL; } } @@ -7087,9 +7093,25 @@ current_frame->order_hint = aom_rb_read_literal( rb, seq_params->order_hint_info.order_hint_bits_minus_1 + 1); +#if CONFIG_MULTIVIEW_CORE + current_frame->view_id = aom_rb_read_literal(rb, 8); + cm->layer_id = current_frame->view_id; +#endif + current_frame->display_order_hint = get_disp_order_hint(cm); current_frame->frame_number = current_frame->order_hint; +#if CONFIG_MULTIVIEW_CORE + current_frame->frame_number = + current_frame->order_hint * cm->number_layers + current_frame->view_id; +#endif + +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- decode bitstream: show_frame=%d, frame_number=%d -- ", + cm->show_existing_frame, current_frame->frame_number); + debug_print_multiview_curr_frame(cm); +#endif + if (!features->error_resilient_mode && !frame_is_intra_only(cm)) { #if CONFIG_PRIMARY_REF_FRAME_OPT signal_primary_ref_frame = aom_rb_read_literal(rb, 1); @@ -7231,7 +7253,11 @@ if (buf == NULL || order_hint != buf->order_hint) { if (buf != NULL) { lock_buffer_pool(pool); +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + set_ref_count_zero(buf, pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST decrease_ref_count(buf, pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST unlock_buffer_pool(pool); cm->ref_frame_map[ref_idx] = NULL; } @@ -7250,10 +7276,15 @@ seq_params->subsampling_y, AOM_BORDER_IN_PIXELS, features->byte_alignment, &buf->raw_frame_buffer, pool->get_fb_cb, pool->cb_priv, false)) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + set_ref_count_zero(buf, pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST decrease_ref_count(buf, pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST unlock_buffer_pool(pool); - aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate frame buffer"); + aom_internal_error( + &cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate frame buffer: read_uncompressed_header"); } unlock_buffer_pool(pool); // According to the specification, valid bitstreams are required to @@ -7301,6 +7332,9 @@ #endif // CONFIG_LF_SUB_PU if (current_frame->frame_type == KEY_FRAME) { cm->current_frame.pyramid_level = 1; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + cm->current_frame.temporal_layer_id = cm->temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST features->tip_frame_mode = TIP_FRAME_DISABLED; setup_frame_size(cm, frame_size_override_flag, rb); @@ -7338,6 +7372,9 @@ cm->cur_frame->num_ref_frames = 0; #endif // CONFIG_IMPROVED_GLOBAL_MOTION } else { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + cm->current_frame.temporal_layer_id = cm->temporal_layer_id; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST features->allow_ref_frame_mvs = 0; features->tip_frame_mode = TIP_FRAME_DISABLED; if (current_frame->frame_type == INTRA_ONLY_FRAME) { @@ -7481,6 +7518,12 @@ (int)ref_frame_map_pairs[ref].disp_order) #endif // CONFIG_PRIMARY_REF_FRAME_OPT : 1; +#if CONFIG_MULTIVIEW_CORE + if (scores[i].distance == 0 && + current_frame->view_id != cm->ref_frame_map_pairs[ref].view_id) { + scores[i].distance = 1; + } +#endif cm->ref_frames_info.ref_frame_distance[i] = scores[i].distance; } av1_get_past_future_cur_ref_lists(cm, scores);
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c index 1e557d0..2d47da1 100644 --- a/av1/decoder/decoder.c +++ b/av1/decoder/decoder.c
@@ -248,6 +248,10 @@ cm->fDecCoeffLog = fopen("DecCoeffLog.txt", "wt"); #endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + cm->fDecMultiviewLog = fopen("/tmp/DecMultilayerLog.txt", "wt"); +#endif + return pbi; } @@ -339,6 +343,12 @@ } #endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + if (pbi->common.fDecMultiviewLog != NULL) { + fclose(pbi->common.fDecMultiviewLog); + } +#endif + aom_free(pbi); } @@ -453,9 +463,14 @@ AV1_COMMON *const cm = &pbi->common; BufferPool *const pool = cm->buffer_pool; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + lock_buffer_pool(pool); + set_ref_count_zero(cm->cur_frame, pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST cm->cur_frame->buf.corrupted = 1; lock_buffer_pool(pool); decrease_ref_count(cm->cur_frame, pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST unlock_buffer_pool(pool); cm->cur_frame = NULL; } @@ -472,6 +487,9 @@ AV1_COMMON *const cm = &pbi->common; RefCntBuffer *trigger_frame = NULL; RefCntBuffer *output_candidate = NULL; +#if CONFIG_MULTIVIEW_CORE + const unsigned int num_layers = cm->number_layers; +#endif // Determine if the triggering frame is the current frame or a frame // already stored in the refrence buffer. @@ -483,12 +501,24 @@ output_candidate = trigger_frame; for (int i = 0; i < REF_FRAMES; i++) { if (is_frame_eligible_for_output(cm->ref_frame_map[i]) && +#if CONFIG_MULTIVIEW_CORE + (cm->ref_frame_map[i]->display_order_hint * num_layers + + cm->ref_frame_map[i]->view_id) < + (output_candidate->display_order_hint * num_layers + + output_candidate->view_id)) { +#else cm->ref_frame_map[i]->display_order_hint < output_candidate->display_order_hint) { +#endif output_candidate = cm->ref_frame_map[i]; } } if (output_candidate != trigger_frame) { +#if CONFIG_MULTIVIEW_CORE + pbi->output_view_ids[pbi->num_output_frames] = output_candidate->view_id; + pbi->display_order_hint_ids[pbi->num_output_frames] = + output_candidate->display_order_hint; +#endif pbi->output_frames[pbi->num_output_frames++] = output_candidate; output_candidate->frame_output_done = 1; #if CONFIG_BITSTREAM_DEBUG @@ -502,6 +532,11 @@ #endif // CONFIG_OUTPUT_FRAME_BASED_ON_ORDER_HINT_ENHANCEMENT // Add the output triggering frame into the output queue. +#if CONFIG_MULTIVIEW_CORE + pbi->output_view_ids[pbi->num_output_frames] = trigger_frame->view_id; + pbi->display_order_hint_ids[pbi->num_output_frames] = + trigger_frame->display_order_hint; +#endif pbi->output_frames[pbi->num_output_frames++] = trigger_frame; trigger_frame->frame_output_done = 1; #if CONFIG_BITSTREAM_DEBUG @@ -513,15 +548,32 @@ if (trigger_frame->display_order_hint != cm->cur_frame->display_order_hint) mismatch_move_frame_idx_r(0); #endif // CONFIG_MISMATCH_DEBUG - // Add the next frames (showable_frame == 1) into the output queue. int successive_output = 1; for (int k = 1; k <= REF_FRAMES && successive_output > 0; k++) { +#if CONFIG_MULTIVIEW_CORE + unsigned int next_disp_order = + (trigger_frame->display_order_hint * num_layers + + trigger_frame->view_id) + + k; +#else unsigned int next_disp_order = trigger_frame->display_order_hint + k; +#endif successive_output = 0; for (int i = 0; i < REF_FRAMES; i++) { if (is_frame_eligible_for_output(cm->ref_frame_map[i]) && +#if CONFIG_MULTIVIEW_CORE + (cm->ref_frame_map[i]->display_order_hint * num_layers + + cm->ref_frame_map[i]->view_id) == next_disp_order) { +#else cm->ref_frame_map[i]->display_order_hint == next_disp_order) { +#endif +#if CONFIG_MULTIVIEW_CORE + pbi->output_view_ids[pbi->num_output_frames] = + cm->ref_frame_map[i]->view_id; + pbi->display_order_hint_ids[pbi->num_output_frames] = + cm->ref_frame_map[i]->display_order_hint; +#endif pbi->output_frames[pbi->num_output_frames++] = cm->ref_frame_map[i]; cm->ref_frame_map[i]->frame_output_done = 1; successive_output++; @@ -550,7 +602,9 @@ if (frame_decoded) { lock_buffer_pool(pool); #if CONFIG_OUTPUT_FRAME_BASED_ON_ORDER_HINT_ENHANCEMENT - if (cm->seq_params.enable_frame_output_order) pbi->num_output_frames = 0; + if (cm->seq_params.enable_frame_output_order) { + pbi->num_output_frames = 0; + } #endif // CONFIG_OUTPUT_FRAME_BASED_ON_ORDER_HINT_ENHANCEMENT // In ext-tile decoding, the camera frame header is only decoded once. So, // we don't update the references here. @@ -565,9 +619,22 @@ is_frame_eligible_for_output(cm->ref_frame_map[ref_index])) output_frame_buffers(pbi, ref_index); #endif // CONFIG_OUTPUT_FRAME_BASED_ON_ORDER_HINT_ENHANCEMENT +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + set_ref_count_zero(cm->ref_frame_map[ref_index], pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST decrease_ref_count(cm->ref_frame_map[ref_index], pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (cm->current_frame.frame_type == KEY_FRAME && ref_index > 0) { + cm->ref_frame_map[ref_index] = NULL; + } else { + cm->ref_frame_map[ref_index] = cm->cur_frame; + //++cm->cur_frame->ref_count; + } +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST cm->ref_frame_map[ref_index] = cm->cur_frame; ++cm->cur_frame->ref_count; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } ++ref_index; } @@ -588,31 +655,50 @@ // We can't store the new frame anywhere, so drop it and return an // error cm->cur_frame->buf.corrupted = 1; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + set_ref_count_zero(cm->cur_frame, pool); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST decrease_ref_count(cm->cur_frame, pool); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST cm->error.error_code = AOM_CODEC_UNSUP_BITSTREAM; } else { +#if CONFIG_MULTIVIEW_CORE + pbi->output_view_ids[pbi->num_output_frames] = cm->cur_frame->view_id; + pbi->display_order_hint_ids[pbi->num_output_frames] = + cm->cur_frame->display_order_hint; +#endif pbi->output_frames[pbi->num_output_frames] = cm->cur_frame; pbi->num_output_frames++; } } else { // Replace any existing output frame +#if !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST assert(pbi->num_output_frames == 0 || pbi->num_output_frames == 1); if (pbi->num_output_frames > 0) { decrease_ref_count(pbi->output_frames[0], pool); } +#endif // !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +#if CONFIG_MULTIVIEW_CORE + pbi->output_view_ids[0] = cm->cur_frame->view_id; + pbi->display_order_hint_ids[0] = cm->cur_frame->display_order_hint; +#endif pbi->output_frames[0] = cm->cur_frame; pbi->num_output_frames = 1; } - } else { + } +#if !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + else { decrease_ref_count(cm->cur_frame, pool); } - +#endif // !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST unlock_buffer_pool(pool); } else { +#if !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST // Nothing was decoded, so just drop this frame buffer lock_buffer_pool(pool); decrease_ref_count(cm->cur_frame, pool); unlock_buffer_pool(pool); +#endif // !CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } cm->cur_frame = NULL; @@ -642,9 +728,17 @@ // thing to do here. const int last_frame = get_closest_pastcur_ref_index(cm); RefCntBuffer *ref_buf = get_ref_frame_buf(cm, last_frame); +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (ref_buf != NULL) printf("\nref_buf->buf.corrupted = 1\n"); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST if (ref_buf != NULL) ref_buf->buf.corrupted = 1; +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } - +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (cm->cur_frame != NULL) { + release_current_frame(pbi); + } +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST if (assign_cur_frame_new_fb(cm) == NULL) { cm->error.error_code = AOM_CODEC_MEM_ERROR; return 1; @@ -675,6 +769,11 @@ int frame_decoded = aom_decode_frame_from_obus(pbi, source, source + size, psource); +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (frame_decoded == 0) { + release_current_frame(pbi); + } +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST #if CONFIG_INSPECTION if (cm->features.tip_frame_mode == TIP_FRAME_AS_OUTPUT) { if (pbi->inspect_tip_cb != NULL) {
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h index 6fa9179..6968bd8 100644 --- a/av1/decoder/decoder.h +++ b/av1/decoder/decoder.h
@@ -290,6 +290,11 @@ size_t output_frames_offset; // Use only for single layer size_t num_output_frames; // How many frames are queued up so far? +#if CONFIG_MULTIVIEW_CORE + int output_view_ids[REF_FRAMES]; + int display_order_hint_ids[REF_FRAMES]; +#endif + // In order to properly support random-access decoding, we need // to behave slightly differently for the very first frame we decode. // So we track whether this is the first frame or not. @@ -427,6 +432,28 @@ } } +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +static INLINE void set_ref_count_zero(RefCntBuffer *const buf, + BufferPool *const pool) { + if (buf != NULL) { + buf->ref_count = 0; + // Reference counts should never become negative. If this assertion fails, + // there is a bug in our reference count management. + assert(buf->ref_count >= 0); + // A worker may only get a free framebuffer index when calling get_free_fb. + // But the raw frame buffer is not set up until we finish decoding header. + // So if any error happens during decoding header, frame_bufs[idx] will not + // have a valid raw frame buffer. + if (buf->ref_count == 0 && buf->raw_frame_buffer.data) { + pool->release_fb_cb(pool->cb_priv, &buf->raw_frame_buffer); + buf->raw_frame_buffer.data = NULL; + buf->raw_frame_buffer.size = 0; + buf->raw_frame_buffer.priv = NULL; + } + } +} +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + #if CONFIG_OUTPUT_FRAME_BASED_ON_ORDER_HINT_ENHANCEMENT // Check whether the frame is ready to output or not. static INLINE bool is_frame_eligible_for_output(RefCntBuffer *const buf) {
diff --git a/av1/decoder/obu.c b/av1/decoder/obu.c index 28e0be8..18b5191 100644 --- a/av1/decoder/obu.c +++ b/av1/decoder/obu.c
@@ -239,6 +239,12 @@ return 0; } +#if CONFIG_MULTIVIEW_CORE + cm->number_layers = + aom_rb_read_literal(rb, 8) + + 1; // TODO: (@hegilmez) redesign later or change where this is signaled +#endif + av1_read_sequence_header(cm, rb, seq_params); av1_read_color_config(rb, seq_params, &cm->error); @@ -1084,6 +1090,14 @@ data += payload_size; } +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + FILE *const logfile = cm->fDecMultiviewLog; + logfile_multiview_curr_frame(cm, logfile); + logfile_multiview_buf_refs(cm, logfile); + logfile_buffer_state(cm, logfile); + // logfile_primary_ref_info(cm, logfile); +#endif + if (cm->error.error_code != AOM_CODEC_OK) return -1; return frame_decoding_finished; }
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c index 6ec0f69..6475565 100644 --- a/av1/encoder/av1_quantize.c +++ b/av1/encoder/av1_quantize.c
@@ -523,11 +523,12 @@ *v_dc_delta_q = 0; *u_ac_delta_q = 0; *v_ac_delta_q = 0; - +#if !CONFIG_MULTIVIEW_CORE if (frame_is_intra_only(cm)) { *y_dc_delta_q = 0; *u_dc_delta_q = *v_dc_delta_q = -4; } +#endif } void av1_set_quantizer(AV1_COMMON *const cm, int min_qmlevel, int max_qmlevel,
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index d493ab1..a5b3a61 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -2153,7 +2153,7 @@ const MB_MODE_INFO *mbmi #endif // CONFIG_SEP_COMP_DRL ) { - const int8_t ref_frame_type = av1_ref_frame_type(ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(ref_frame); #if CONFIG_SEP_COMP_DRL const CANDIDATE_MV *curr_ref_mv_stack = has_second_drl(mbmi) ? mbmi_ext_frame->ref_mv_stack[ref_idx] @@ -4696,7 +4696,6 @@ aom_wb_write_literal( wb, quant_params->base_qindex, bit_depth == AOM_BITS_8 ? QINDEX_BITS_UNEXT : QINDEX_BITS); - write_delta_q(wb, quant_params->y_dc_delta_q); if (num_planes > 1) { int diff_uv_delta = @@ -5719,6 +5718,15 @@ wb, current_frame->order_hint, seq_params->order_hint_info.order_hint_bits_minus_1 + 1); +#if CONFIG_MULTIVIEW_CORE + aom_wb_write_literal(wb, current_frame->view_id, 8); +#endif + +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- write bitstream: view_id=%d --- ", current_frame->view_id); + debug_print_multiview_curr_frame(cm); +#endif + if (!features->error_resilient_mode && !frame_is_intra_only(cm)) { #if CONFIG_PRIMARY_REF_FRAME_OPT aom_wb_write_literal(wb, cpi->signal_primary_ref_frame, 1); @@ -5801,19 +5809,43 @@ if (features->error_resilient_mode && seq_params->order_hint_info.enable_order_hint) { for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST_FIX_ERROR_RES + int idx = ref_idx; + if (cm->ref_frame_map[ref_idx] == NULL) { + idx = 0; + // idx = cm->remapped_ref_idx[0]; + } + aom_wb_write_literal( + wb, cm->ref_frame_map[idx]->order_hint, + seq_params->order_hint_info.order_hint_bits_minus_1 + 1); + +#else aom_wb_write_literal( wb, cm->ref_frame_map[ref_idx]->order_hint, seq_params->order_hint_info.order_hint_bits_minus_1 + 1); +#endif } } // Write all ref frame base_qindex if error_resilient_mode == 1. This is // required by reference mapping. if (features->error_resilient_mode) { for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST_FIX_ERROR_RES + int idx = ref_idx; + if (cm->ref_frame_map[ref_idx] == NULL) { + idx = 0; + // idx = cm->remapped_ref_idx[0]; + } + aom_wb_write_literal(wb, cm->ref_frame_map[idx]->base_qindex, + cm->seq_params.bit_depth == AOM_BITS_8 + ? QINDEX_BITS_UNEXT + : QINDEX_BITS); +#else aom_wb_write_literal(wb, cm->ref_frame_map[ref_idx]->base_qindex, cm->seq_params.bit_depth == AOM_BITS_8 ? QINDEX_BITS_UNEXT : QINDEX_BITS); +#endif } } } @@ -6427,6 +6459,12 @@ } } } +#if CONFIG_MULTIVIEW_CORE + aom_wb_write_literal( + &wb, seq_params->num_views - 1, + 8); // TODO: (@hegilmez) redesign later or change where this is signaled +#endif + write_sequence_header(seq_params, &wb); write_color_config(seq_params, &wb); @@ -6973,6 +7011,9 @@ uint32_t obu_header_size = 0; uint32_t obu_payload_size = 0; FrameHeaderInfo fh_info = { NULL, 0, 0 }; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + // printf("\ncm->temporal_layer_id: %d\n", cm->temporal_layer_id); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST const uint8_t obu_extension_header = cm->temporal_layer_id << 5 | cm->spatial_layer_id << 3 | 0;
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index c9e4843..fee0718 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -159,6 +159,9 @@ } static void update_rc_counts(AV1_COMP *cpi) { +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("*** update_counters \n"); +#endif update_keyframe_counters(cpi); update_frames_till_gf_update(cpi); update_gf_group_index(cpi); @@ -254,8 +257,14 @@ GF_GROUP gf_group = cpi->gf_group; // The current display index stored has not yet been updated. We must add // The order offset to get the correct value here. +#if CONFIG_MULTIVIEW_CORE + const int cur_frame_disp = + (cpi->common.current_frame.frame_number + order_offset) / + cpi->common.number_layers; +#else const int cur_frame_disp = cpi->common.current_frame.frame_number + order_offset; +#endif const SubGOPStepCfg *step_gop_cfg = get_subgop_step(&gf_group, gf_group.index); @@ -618,13 +627,77 @@ return 1; } -static int get_free_ref_map_index(RefFrameMapPair ref_map_pairs[REF_FRAMES]) { +static int get_free_ref_map_index(RefFrameMapPair ref_map_pairs[REF_FRAMES] +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + , + int cur_temporal_layer_id +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + , + int current_view_id, int num_views +#endif +) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER || \ + CONFIG_MULTIVIEW_SEPARATE_DPB + int start_idx = 0; + int end_idx = REF_FRAMES - 1; +#endif + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + // Allocate ref map slot(s) to each temporal layer + // to avoid the case that a higher layer frame + // flushes out a lower layer frame from a ref map slot + + if (cur_temporal_layer_id == 1) { + start_idx = 0; + end_idx = 1; + } else if (cur_temporal_layer_id == 2) { + start_idx = 2; + end_idx = 2; + } else if (cur_temporal_layer_id == 3) { + start_idx = 3; + end_idx = 4; + } else if (cur_temporal_layer_id == 4) { + start_idx = 5; + end_idx = 6; + } else if (cur_temporal_layer_id == 5) { + start_idx = 7; + end_idx = 7; + } +#endif + +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int idx_offset = REF_FRAMES / num_views; + if (current_view_id == 0) { + start_idx = 0; + end_idx = idx_offset; + } else if (current_view_id > 0) { + start_idx = idx_offset + 1; + end_idx = REF_FRAMES - 1; + } else // current_view_id == -1 + { + start_idx = 0; + end_idx = REF_FRAMES - 1; + } +#endif + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER || \ + CONFIG_MULTIVIEW_SEPARATE_DPB + for (int idx = start_idx; idx <= end_idx; idx++) +#else for (int idx = 0; idx < REF_FRAMES; ++idx) +#endif if (ref_map_pairs[idx].disp_order == -1) return idx; return INVALID_IDX; } static int get_refresh_idx(int update_arf, int refresh_level, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + int cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int current_view_id, int num_views, +#endif int cur_frame_disp, RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]) { int arf_count = 0; @@ -637,7 +710,56 @@ int oldest_ref_level_order = INT32_MAX; int oldest_ref_level_idx = -1; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER || \ + CONFIG_MULTIVIEW_SEPARATE_DPB + int start_idx = 0; + int end_idx = REF_FRAMES - 1; +#endif + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + // Allocate ref map slot(s) to each temporal layer + // to avoid the case that a higher layer frame + // flushes out a lower layer frame from a ref map slot + + if (cur_temporal_layer_id == 1) { + start_idx = 0; + end_idx = 1; + } else if (cur_temporal_layer_id == 2) { + start_idx = 2; + end_idx = 2; + } else if (cur_temporal_layer_id == 3) { + start_idx = 3; + end_idx = 4; + } else if (cur_temporal_layer_id == 4) { + start_idx = 5; + end_idx = 6; + } else if (cur_temporal_layer_id == 5) { + start_idx = 7; + end_idx = 7; + } +#endif + +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int idx_offset = REF_FRAMES / num_views; + if (current_view_id == 0) { + start_idx = 0; + end_idx = idx_offset; + } else if (current_view_id > 0) { + start_idx = idx_offset + 1; + end_idx = REF_FRAMES - 1; + } else // current_view_id == -1 + { + start_idx = 0; + end_idx = REF_FRAMES - 1; + } +#endif + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER || \ + CONFIG_MULTIVIEW_SEPARATE_DPB + for (int map_idx = start_idx; map_idx <= end_idx; map_idx++) { +#else for (int map_idx = 0; map_idx < REF_FRAMES; map_idx++) { +#endif RefFrameMapPair ref_pair = ref_frame_map_pairs[map_idx]; if (ref_pair.disp_order == -1) continue; const int frame_order = ref_pair.disp_order; @@ -677,14 +799,48 @@ if (update_arf && arf_count > 2) return oldest_arf_idx; if (oldest_idx >= 0) return oldest_idx; if (oldest_arf_idx >= 0) return oldest_arf_idx; +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + // Allocate ref map slot(s) to each temporal layer + // to avoid the case that a higher layer frame + // flushes out a lower layer frame from a ref map slot + if (cur_temporal_layer_id == 1) { + return 1; + } else if (cur_temporal_layer_id == 2) { + return 2; + } else if (cur_temporal_layer_id == 3) { + return 4; + } else if (cur_temporal_layer_id == 4) { + return 5; + } else if (cur_temporal_layer_id == 5) { + return 7; + } +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int idx_refresh = 1; + if (current_view_id == 0) { + return idx_refresh; + } else if (current_view_id > 0) { + idx_refresh += idx_offset; + return start_idx; + } else // current_view_id == -1 + { + // do nothing + } +#endif assert(0 && "No valid refresh index found"); return -1; } static int get_refresh_frame_flags_subgop_cfg( - const AV1_COMP *const cpi, int gf_index, int cur_disp_order, - RefFrameMapPair ref_frame_map_pairs[REF_FRAMES], int refresh_mask, - int free_fb_index) { + const AV1_COMP *const cpi, int gf_index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + int cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int current_view_id, int num_views, +#endif + int cur_disp_order, RefFrameMapPair ref_frame_map_pairs[REF_FRAMES], + int refresh_mask, int free_fb_index) { const SubGOPStepCfg *step_gop_cfg = get_subgop_step(&cpi->gf_group, gf_index); assert(step_gop_cfg != NULL); const int pyr_level = step_gop_cfg->pyr_level; @@ -704,14 +860,33 @@ const int update_arf = type_code == FRAME_TYPE_OOO_FILTERED && pyr_level == 1; const int refresh_idx = get_refresh_idx(update_arf, refresh_level, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + current_view_id, num_views, +#endif cur_disp_order, ref_frame_map_pairs); +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + const CurrentFrame *const cf = &cpi->common.current_frame; // cm->cur_frame; + printf("In - get_refresh_idx: (View,Level,OH,DOH):(%d,%d,%d,%d) \n", + cf->view_id, cf->pyramid_level, cf->order_hint, + cf->display_order_hint); + printf("Out - get_refresh_idx: %d \n", refresh_idx); +#endif return 1 << refresh_idx; } int av1_get_refresh_frame_flags( const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params, - FRAME_UPDATE_TYPE frame_update_type, int gf_index, int cur_disp_order, - RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]) { + FRAME_UPDATE_TYPE frame_update_type, int gf_index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + int cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int current_view_id, int num_views, +#endif + int cur_disp_order, RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]) { // Switch frames and shown key-frames overwrite all reference slots if ((frame_params->frame_type == KEY_FRAME && !cpi->no_show_fwd_kf) || frame_params->frame_type == S_FRAME) { @@ -737,12 +912,33 @@ } // Search for the open slot to store the current frame. - int free_fb_index = get_free_ref_map_index(ref_frame_map_pairs); - + int free_fb_index = get_free_ref_map_index(ref_frame_map_pairs +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + , + cur_temporal_layer_id +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + , + current_view_id, num_views +#endif + ); +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + const CurrentFrame *const cf = &cpi->common.current_frame; // cm->cur_frame; + printf("In - get_free_ref_map_index: (View,Level,OH,DOH):(%d,%d,%d,%d) \n", + cf->view_id, cf->pyramid_level, cf->order_hint, + cf->display_order_hint); + printf("Out - get_free_ref_map_index: %d \n", free_fb_index); +#endif if (use_subgop_cfg(&cpi->gf_group, gf_index)) { const int mask = get_refresh_frame_flags_subgop_cfg( - cpi, gf_index, cur_disp_order, ref_frame_map_pairs, refresh_mask, - free_fb_index); + cpi, gf_index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + current_view_id, num_views, +#endif + cur_disp_order, ref_frame_map_pairs, refresh_mask, free_fb_index); return mask; } @@ -759,8 +955,20 @@ } const int update_arf = frame_update_type == ARF_UPDATE; - const int refresh_idx = - get_refresh_idx(update_arf, -1, cur_disp_order, ref_frame_map_pairs); + const int refresh_idx = get_refresh_idx(update_arf, -1, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + current_view_id, num_views, +#endif + cur_disp_order, ref_frame_map_pairs); +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("In - get_refresh_idx: (View,Level,OH,DOH):(%d,%d,%d,%d) \n", + cf->view_id, cf->pyramid_level, cf->order_hint, + cf->display_order_hint); + printf("Out - get_refresh_idx: %d \n", refresh_idx); +#endif return 1 << refresh_idx; } @@ -1077,7 +1285,15 @@ frame_update_type != KFFLT_OVERLAY_UPDATE && frame_update_type != INTNL_OVERLAY_UPDATE) { frame_params.frame_type = KEY_FRAME; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set1 frame type to KEY_FRAME: "); + debug_print_multiview_curr_frame(cm); +#endif } else { +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set1 frame type to INTER_FRAME: "); + debug_print_multiview_curr_frame(cm); +#endif frame_params.frame_type = INTER_FRAME; } } else if (is_stat_consumption_stage(cpi)) { @@ -1111,8 +1327,63 @@ av1_configure_buffer_updates(cpi, frame_update_type); const int order_offset = gf_group->arf_src_offset[gf_group->index]; + +#if CONFIG_MULTIVIEW_CORE + int cur_frame_disp = cpi->common.current_frame.frame_number + order_offset; +#else const int cur_frame_disp = cpi->common.current_frame.frame_number + order_offset; +#endif + +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("encode_startegy: cur_frame_disp=%2d, order_offset[%d]=%2d\n", + cur_frame_disp, gf_group->index, order_offset); +#endif + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + // Here, ff temporal_layer_id is set to a non-zero value (pry_level), + // temporal_layer_id is siganled in obu extension, + // and affect reference list construction in both encoder and decoder. + // Otherwise (if temporal_layer_id is set to 0), temporal_layer_id is + // not signaled and does not change the reference frame list construction. +#if CONFIG_MULTIVIEW_CORE + cm->current_frame.view_id = cur_frame_disp % cm->number_layers; + cm->current_frame.order_hint = cur_frame_disp / cm->number_layers; + cm->current_frame.display_order_hint = cur_frame_disp / cm->number_layers; + cur_frame_disp = cur_frame_disp / cm->number_layers; +#else + cm->current_frame.order_hint = cur_frame_disp; + cm->current_frame.display_order_hint = cur_frame_disp; +#endif + cm->current_frame.pyramid_level = + get_true_pyr_level(cpi->gf_group.layer_depth[cpi->gf_group.index], + cur_frame_disp, cpi->gf_group.max_layer_depth); + +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cm->temporal_layer_id = cm->current_frame.pyramid_level; + cm->current_frame.temporal_layer_id = cm->temporal_layer_id; +#else + cm->temporal_layer_id = 0; + cm->current_frame.temporal_layer_id = cm->temporal_layer_id; +#endif +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + const CurrentFrame *const cf = &cm->current_frame; + printf( + "Encode startegy before init: " + "(View,Level,OH,DOH,order_offset):(%d,%d,%d,%d,%d) \n", + cf->view_id, cf->pyramid_level, cf->order_hint, cf->display_order_hint, + order_offset); + printf("Encoder side ref-buffer before init ="); + debug_print_buffer_state(cm); +#endif + +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- encode_strategy: "); + debug_print_multiview_curr_frame(cm); +#endif + #if CONFIG_PRIMARY_REF_FRAME_OPT init_ref_map_pair(&cpi->common, cm->ref_frame_map_pairs, gf_group->update_type[gf_group->index] == KF_UPDATE); @@ -1167,6 +1438,15 @@ frame_params.refresh_frame_flags = av1_get_refresh_frame_flags( cpi, &frame_params, frame_update_type, cpi->gf_group.index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cm->seq_params.explicit_ref_frame_map + ? 0 + : cm->current_frame.temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + cm->seq_params.explicit_ref_frame_map ? -1 : cm->current_frame.view_id, + cm->number_layers, +#endif #if CONFIG_PRIMARY_REF_FRAME_OPT cur_frame_disp, cm->ref_frame_map_pairs); #else @@ -1183,6 +1463,21 @@ if (frame_order == cur_frame_disp) frame_params.existing_fb_idx_to_show = frame; } +#if CONFIG_MULTIVIEW_SEPARATE_DPB + if (frame_params.existing_fb_idx_to_show < 0) { + frame_params.show_existing_frame = 0; + } +#endif +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + if (frame_params.existing_fb_idx_to_show < 0) { + printf( + "Warning (show_existing_frame): existing_fb_idx_to_show=%d -- " + "(View,Level,OH,DOH):(%d,%d,%d,%d) \n", + frame_params.existing_fb_idx_to_show, cf->view_id, + cf->pyramid_level, cf->order_hint, cf->display_order_hint); + frame_params.show_existing_frame = 0; + } +#endif } }
diff --git a/av1/encoder/encode_strategy.h b/av1/encoder/encode_strategy.h index 414548a..ca05a24 100644 --- a/av1/encoder/encode_strategy.h +++ b/av1/encoder/encode_strategy.h
@@ -70,8 +70,14 @@ int av1_get_refresh_frame_flags( const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params, - FRAME_UPDATE_TYPE frame_update_type, int gf_index, int cur_frame_disp, - RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]); + FRAME_UPDATE_TYPE frame_update_type, int gf_index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + int cur_temporal_layer_id, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + int current_view_id, int num_views, +#endif + int cur_frame_disp, RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]); int av1_get_refresh_ref_frame_map(AV1_COMMON *cm, int refresh_frame_flags);
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index c217375..6747513 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -1737,7 +1737,6 @@ // Indicates whether or not to use a default reduced set for ext-tx // rather than the potential full set of 16 transforms features->reduced_tx_set_used = cpi->oxcf.txfm_cfg.reduced_tx_type_set; - // Make sure segment_id is no larger than last_active_segid. if (cm->seg.enabled && cm->seg.update_map) { const int mi_rows = cm->mi_params.mi_rows; @@ -1823,3 +1822,23 @@ encode_frame_internal(cpi); } } + +#if CONFIG_MULTIVIEW_DEBUG +void debug_print_buf_refs_enc(const AV1_COMMON *const cm) { + const RefCntBuffer *const cf = cm->cur_frame; + MV_REFERENCE_FRAME ref_frame; + printf("(View,OH,DOH):(%2d,%2d,%2d) ", cf->view_id, cf->order_hint, + cf->display_order_hint); + printf(" num_ref_frames/ref_frames_info.num_total_refs=%d/%d ", + cf->num_ref_frames, cm->ref_frames_info.num_total_refs); + printf("["); + for (ref_frame = 0; ref_frame < INTER_REFS_PER_FRAME; ++ref_frame) { + const int map_idx = get_ref_frame_map_idx(cm, ref_frame); + printf("%2d:", map_idx); + printf("(%2d,", cf->ref_view_ids[ref_frame]); + printf("%2d,", cf->ref_order_hints[ref_frame]); + printf("%2d) ", cf->ref_display_order_hint[ref_frame]); + } + printf("]\n"); +} +#endif
diff --git a/av1/encoder/encodeframe.h b/av1/encoder/encodeframe.h index 598190d..cfbfc3e 100644 --- a/av1/encoder/encodeframe.h +++ b/av1/encoder/encodeframe.h
@@ -39,7 +39,9 @@ const CHROMA_REF_INFO *chroma_ref_info); void av1_encode_frame(struct AV1_COMP *cpi); - +#if CONFIG_MULTIVIEW_DEBUG +void debug_print_buf_refs_enc(const AV1_COMMON *const cm); +#endif void av1_alloc_tile_data(struct AV1_COMP *cpi); void av1_init_tile_data(struct AV1_COMP *cpi); void av1_encode_tile(struct AV1_COMP *cpi, struct ThreadData *td, int tile_row,
diff --git a/av1/encoder/encodemv.c b/av1/encoder/encodemv.c index 6684c2a..c6f2157 100644 --- a/av1/encoder/encodemv.c +++ b/av1/encoder/encodemv.c
@@ -1129,7 +1129,7 @@ const MB_MODE_INFO *mbmi #endif // CONFIG_SEP_COMP_DRL ) { - const int8_t ref_frame_type = av1_ref_frame_type(ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(ref_frame); #if CONFIG_SEP_COMP_DRL const CANDIDATE_MV *curr_ref_mv_stack = has_second_drl(mbmi) ? mbmi_ext->ref_mv_stack[ref_frame[ref_idx]]
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 7d2b8c7..35410d4 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -362,6 +362,10 @@ seq->still_picture = (tool_cfg->force_video_mode == 0) && (oxcf->input_cfg.limit == 1); +#if CONFIG_MULTIVIEW_CORE + assert(oxcf->input_cfg.num_views > 0); + seq->still_picture &= oxcf->input_cfg.num_views == 1; +#endif seq->reduced_still_picture_hdr = seq->still_picture; seq->reduced_still_picture_hdr &= !tool_cfg->full_still_picture_hdr; seq->force_screen_content_tools = 2; @@ -387,6 +391,10 @@ seq->num_same_ref_compound = SAME_REF_COMPOUND_PRUNE; #endif // CONFIG_SAME_REF_COMPOUND +#if CONFIG_MULTIVIEW_CORE + seq->num_views = oxcf->input_cfg.num_views; +#endif + seq->max_frame_width = frm_dim_cfg->forced_max_frame_width ? frm_dim_cfg->forced_max_frame_width : frm_dim_cfg->width; @@ -611,6 +619,10 @@ cm->spatial_layer_id = 0; cm->temporal_layer_id = 0; +#if CONFIG_MULTIVIEW_CORE + cm->number_layers = oxcf->input_cfg.num_views; +#endif + // change includes all joint functionality av1_change_config(cpi, oxcf); @@ -1068,6 +1080,9 @@ #if DEBUG_EXTQUANT cm->fEncCoeffLog = fopen("EncCoeffLog.txt", "wt"); #endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + cm->fEncMultiviewLog = fopen("/tmp/EncMultilayerLog.txt", "wt"); +#endif cm->error.setjmp = 1; cpi->lap_enabled = num_lap_buffers > 0; @@ -1103,6 +1118,10 @@ } cpi->frames_left = cpi->oxcf.input_cfg.limit; +#if CONFIG_MULTIVIEW_CORE + cpi->frames_left *= cm->number_layers; + assert(cm->number_layers > 0); +#endif av1_rc_init(&cpi->oxcf, 0, &cpi->rc); @@ -1561,6 +1580,13 @@ fclose(cpi->common.fEncCoeffLog); } #endif + +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + if (cpi->common.fEncMultiviewLog != NULL) { + fclose(cpi->common.fEncMultiviewLog); + } +#endif + aom_free(cpi->subgop_config_str); aom_free(cpi->subgop_config_path); aom_free(cpi); @@ -3575,6 +3601,20 @@ cpi->last_encoded_frame_order_hint = cm->current_frame.display_order_hint; #endif // CONFIG_PRIMARY_REF_FRAME_OPT +#if CONFIG_MULTIVIEW_DEBUG +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("Encoder - "); + debug_print_multiview_buf_refs(cm); + debug_print_multiview_curr_frame(cm); +#endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + FILE *const logfile = cm->fEncMultiviewLog; + logfile_multiview_curr_frame(cm, logfile); + logfile_multiview_buf_refs(cm, logfile); + logfile_buffer_state(cm, logfile); +#endif +#endif + return AOM_CODEC_OK; } @@ -3911,13 +3951,16 @@ realloc_and_scale_source(cpi, cm->cur_frame->buf.y_crop_width, cm->cur_frame->buf.y_crop_height); } - // current_frame->frame_number is incremented already for // keyframe overlays. if (!av1_check_keyframe_overlay(cpi->gf_group.index, &cpi->gf_group, - cpi->rc.frames_since_key)) + cpi->rc.frames_since_key)) { ++current_frame->frame_number; - +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("Frame number increment 1 : %d --> %d \n", + current_frame->frame_number - 1, current_frame->frame_number); +#endif + } return AOM_CODEC_OK; } @@ -4151,6 +4194,11 @@ ++current_frame->frame_number; } +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("Frame number increment 2 : %d --> %d \n", + current_frame->frame_number - 1, current_frame->frame_number); +#endif + return AOM_CODEC_OK; } @@ -4188,16 +4236,74 @@ current_frame->order_hint = current_frame->frame_number + frame_params->order_offset; current_frame->display_order_hint = current_frame->order_hint; +#if CONFIG_MULTIVIEW_CORE + current_frame->absolute_poc = + current_frame->key_frame_number + current_frame->display_order_hint; + current_frame->view_id = + current_frame->display_order_hint % cm->number_layers; + current_frame->order_hint = current_frame->order_hint / cm->number_layers; + current_frame->display_order_hint = + current_frame->display_order_hint / cm->number_layers; +#endif + current_frame->pyramid_level = get_true_pyr_level( cpi->gf_group.layer_depth[cpi->gf_group.index], current_frame->display_order_hint, cpi->gf_group.max_layer_depth); +#if !CONFIG_MULTIVIEW_CORE current_frame->absolute_poc = current_frame->key_frame_number + current_frame->display_order_hint; +#endif +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + cm->temporal_layer_id = current_frame->pyramid_level; + current_frame->temporal_layer_id = cm->temporal_layer_id; +#else + cm->temporal_layer_id = 0; + current_frame->temporal_layer_id = cm->temporal_layer_id; +#endif + + const int order_offset = cpi->gf_group.arf_src_offset[cpi->gf_group.index]; +#if CONFIG_MULTIVIEW_CORE + const int cur_frame_disp = + (cpi->common.current_frame.frame_number + order_offset) / + cm->number_layers; +#else + const int cur_frame_disp = + cpi->common.current_frame.frame_number + order_offset; +#endif + +#if CONFIG_PRIMARY_REF_FRAME_OPT + init_ref_map_pair( + &cpi->common, cm->ref_frame_map_pairs, + cpi->gf_group.update_type[cpi->gf_group.index] == KF_UPDATE); +#else + RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]; + init_ref_map_pair( + &cpi->common, ref_frame_map_pairs, + cpi->gf_group.update_type[cpi->gf_group.index] == KF_UPDATE); +#endif // CONFIG_PRIMARY_REF_FRAME_OPT +#if CONFIG_PRIMARY_REF_FRAME_OPT + if (cm->seq_params.explicit_ref_frame_map) + av1_get_ref_frames_enc(cm, cur_frame_disp, cm->ref_frame_map_pairs); + else + av1_get_ref_frames(cm, cur_frame_disp, cm->ref_frame_map_pairs); +#else + if (cm->seq_params.explicit_ref_frame_map) + av1_get_ref_frames_enc(cm, cur_frame_disp, ref_frame_map_pairs); + else + av1_get_ref_frames(cm, cur_frame_disp, ref_frame_map_pairs); +#endif // CONFIG_PRIMARY_REF_FRAME_OPT +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST current_frame->order_hint %= (1 << (cm->seq_params.order_hint_info.order_hint_bits_minus_1 + 1)); +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- encoder: "); + debug_print_multiview_curr_frame(cm); +#endif + if (is_stat_generation_stage(cpi)) { av1_first_pass(cpi, frame_input->ts_duration); } else { @@ -4208,6 +4314,22 @@ aom_bitstream_queue_set_frame_write(cm->current_frame.order_hint * 2 + cm->show_frame); #endif // CONFIG_BITSTREAM_DEBUG +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + cm->show_existing_frame = frame_params->show_existing_frame; + cpi->existing_fb_idx_to_show = frame_params->existing_fb_idx_to_show; + if (cm->show_existing_frame != 0 && cpi->existing_fb_idx_to_show < 0) { + FILE *const logfile = cm->fEncMultiviewLog; + fprintf(logfile, + " -- Encoder show existing frame may fail (flag=%d, " + "fb_idx_to_show=%d): \n ", + cm->show_existing_frame, cpi->existing_fb_idx_to_show); + logfile_multiview_curr_frame(cm, logfile); + logfile_multiview_buf_refs(cm, logfile); + logfile_buffer_state(cm, logfile); + logfile_primary_ref_info(cm, logfile); + fprintf(logfile, " -- End show existing frame --- \n "); + } +#endif if (encode_frame_to_data_rate(cpi, &frame_results->size, dest) != AOM_CODEC_OK) { return AOM_CODEC_ERROR;
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index de205b5..1d9f860 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -731,6 +731,10 @@ unsigned int chroma_subsampling_x; // Indicates the chrome subsampling y value. unsigned int chroma_subsampling_y; +#if CONFIG_MULTIVIEW_CORE + // Indicates number of input views. + unsigned int num_views; +#endif } InputCfg; typedef struct {
diff --git a/av1/encoder/encoder_utils.c b/av1/encoder/encoder_utils.c index cfd5564..bc51bd5 100644 --- a/av1/encoder/encoder_utils.c +++ b/av1/encoder/encoder_utils.c
@@ -1106,6 +1106,48 @@ (encode_show_existing_frame(cm) || cm->show_existing_frame)) { RefCntBuffer *const frame_to_show = cm->ref_frame_map[cpi->existing_fb_idx_to_show]; +#if CONFIG_MULTIVIEW_DEBUG + const CurrentFrame *const cf = &cpi->common.current_frame; +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + FILE *const logfile = cm->fEncMultiviewLog; +#endif + if (frame_to_show == NULL || cpi->existing_fb_idx_to_show < 0) { +#if CONFIG_MULTIVIEW_DEBUG_PROMPT + printf( + "Frame to show - does not exist (index=%d): " + "(View,Level,OH,DOH):(%d,%d,%d,%d) \n", + cpi->existing_fb_idx_to_show, cf->view_id, cf->pyramid_level, + cf->order_hint, cf->display_order_hint); + fflush(stdout); +#endif +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + fprintf(logfile, + "Frame to show - does not exist (index=%d): " + "(View,Level,OH,DOH):(%d,%d,%d,%d) \n", + cpi->existing_fb_idx_to_show, cf->view_id, cf->pyramid_level, + cf->order_hint, cf->display_order_hint); +#endif + } else { + // printf("Frame to show - exists (index=%d): + // (View,Level,OH,DOH):(%d,%d,%d,%d) \n", cpi->existing_fb_idx_to_show, + // cf->view_id, cf->pyramid_level, cf->order_hint, + // cf->display_order_hint); + +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES + // fprintf(logfile, "Frame to show - exists (index=%d): + // (View,Level,OH,DOH):(%d,%d,%d,%d) \n", cpi->existing_fb_idx_to_show, + // cf->view_id, cf->pyramid_level, cf->order_hint, + // cf->display_order_hint); +#endif + } +#if CONFIG_MULTIVIEW_DEBUG_LOGFILES +// logfile_multiview_curr_frame(cm, logfile); +// logfile_multiview_buf_refs(cm, logfile); +// logfile_buffer_state(cm, logfile); +// logfile_primary_ref_info(cm, logfile); +// fflush(logfile); +#endif +#endif if (frame_to_show == NULL) { aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
diff --git a/av1/encoder/encoder_utils.h b/av1/encoder/encoder_utils.h index 6432bb4..64adb9a 100644 --- a/av1/encoder/encoder_utils.h +++ b/av1/encoder/encoder_utils.h
@@ -1145,7 +1145,14 @@ // All buffers are refreshed for shown keyframes and S-frames. for (int ref_frame = 0; ref_frame < REF_FRAMES; ref_frame++) { if (((cm->current_frame.refresh_frame_flags >> ref_frame) & 1) == 1) { +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + if (cm->current_frame.frame_type == KEY_FRAME && ref_frame > 0) + set_frame_buffer_invalid(&cm->ref_frame_map[ref_frame]); + else + assign_frame_buffer_p(&cm->ref_frame_map[ref_frame], cm->cur_frame); +#else // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST assign_frame_buffer_p(&cm->ref_frame_map[ref_frame], cm->cur_frame); +#endif // CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST } } }
diff --git a/av1/encoder/gop_structure.c b/av1/encoder/gop_structure.c index 8bf8234..85b79e0 100644 --- a/av1/encoder/gop_structure.c +++ b/av1/encoder/gop_structure.c
@@ -32,57 +32,128 @@ GF_GROUP *const gf_group, RATE_CONTROL *rc, FRAME_INFO *frame_info, int start, int end, int *cur_frame_idx, int *frame_ind, - int layer_depth) { - const int num_frames_to_process = end - start; + int layer_depth +#if CONFIG_MULTIVIEW_CORE + , + const int num_views +#endif +) { + const int num_frames_to_process = end - start; // Either we are at the last level of the pyramid, or we don't have enough // frames between 'l' and 'r' to create one more level. if (layer_depth > gf_group->max_layer_depth_allowed || num_frames_to_process < 3) { // Leaf nodes. while (start < end) { + gf_group->max_layer_depth = + AOMMAX(gf_group->max_layer_depth, layer_depth); gf_group->update_type[*frame_ind] = LF_UPDATE; gf_group->arf_src_offset[*frame_ind] = 0; +#if CONFIG_MULTIVIEW_CORE + gf_group->cur_frame_idx[*frame_ind] = num_views * (*cur_frame_idx); +#else gf_group->cur_frame_idx[*frame_ind] = *cur_frame_idx; +#endif gf_group->layer_depth[*frame_ind] = MAX_ARF_LAYERS; gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost( twopass, rc, frame_info, start, end - start, 0, NULL, NULL); - gf_group->max_layer_depth = - AOMMAX(gf_group->max_layer_depth, layer_depth); ++(*frame_ind); ++(*cur_frame_idx); ++start; +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + const int prev_frame_ind = (*frame_ind) - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[*frame_ind] = LF_UPDATE; + gf_group->arf_src_offset[*frame_ind] = 0; + gf_group->cur_frame_idx[*frame_ind] = + gf_group->cur_frame_idx[prev_frame_ind] + 1; + gf_group->layer_depth[*frame_ind] = MAX_ARF_LAYERS; + gf_group->arf_boost[*frame_ind] = gf_group->arf_boost[prev_frame_ind]; + ++(*frame_ind); + } +#endif } } else { const int m = (start + end - 1) / 2; // Internal ARF. gf_group->update_type[*frame_ind] = INTNL_ARF_UPDATE; +#if CONFIG_MULTIVIEW_CORE + gf_group->arf_src_offset[*frame_ind] = num_views * (m - start); + gf_group->cur_frame_idx[*frame_ind] = num_views * (*cur_frame_idx); +#else gf_group->arf_src_offset[*frame_ind] = m - start; gf_group->cur_frame_idx[*frame_ind] = *cur_frame_idx; +#endif gf_group->layer_depth[*frame_ind] = layer_depth; // Get the boost factor for intermediate ARF frames. gf_group->arf_boost[*frame_ind] = av1_calc_arf_boost( twopass, rc, frame_info, m, end - m, m - start, NULL, NULL); ++(*frame_ind); +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + // Internal ARF per-view. + const int prev_frame_ind = (*frame_ind) - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[*frame_ind] = INTNL_ARF_UPDATE; + gf_group->arf_src_offset[*frame_ind] = + gf_group->arf_src_offset[prev_frame_ind] + 1; + gf_group->cur_frame_idx[*frame_ind] = + gf_group->cur_frame_idx[prev_frame_ind]; + gf_group->layer_depth[*frame_ind] = layer_depth; + gf_group->arf_boost[*frame_ind] = gf_group->arf_boost[prev_frame_ind]; + ++(*frame_ind); + } +#endif // Frames displayed before this internal ARF. set_multi_layer_params(twopass, gf_group, rc, frame_info, start, m, - cur_frame_idx, frame_ind, layer_depth + 1); + cur_frame_idx, frame_ind, layer_depth + 1 +#if CONFIG_MULTIVIEW_CORE + , + num_views +#endif + ); // Overlay for internal ARF. gf_group->update_type[*frame_ind] = INTNL_OVERLAY_UPDATE; gf_group->arf_src_offset[*frame_ind] = 0; +#if CONFIG_MULTIVIEW_CORE + gf_group->cur_frame_idx[*frame_ind] = num_views * (*cur_frame_idx); +#else gf_group->cur_frame_idx[*frame_ind] = *cur_frame_idx; +#endif gf_group->arf_boost[*frame_ind] = 0; gf_group->layer_depth[*frame_ind] = layer_depth; ++(*frame_ind); ++(*cur_frame_idx); +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + // Overlay for internal ARF per-view. + const int prev_frame_ind = (*frame_ind) - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[*frame_ind] = INTNL_OVERLAY_UPDATE; + gf_group->arf_src_offset[*frame_ind] = + gf_group->arf_src_offset[prev_frame_ind]; + gf_group->cur_frame_idx[*frame_ind] = + gf_group->cur_frame_idx[prev_frame_ind] + 1; + gf_group->arf_boost[*frame_ind] = 0; + gf_group->layer_depth[*frame_ind] = layer_depth; + ++(*frame_ind); + } +#endif // Frames displayed after this internal ARF. set_multi_layer_params(twopass, gf_group, rc, frame_info, m + 1, end, - cur_frame_idx, frame_ind, layer_depth + 1); + cur_frame_idx, frame_ind, layer_depth + 1 +#if CONFIG_MULTIVIEW_CORE + , + num_views +#endif + ); } } @@ -124,7 +195,12 @@ int *cur_frame_idx, int *frame_index, int is_ld_map_first_gop) { int last_shown_frame = 0; int min_pyr_level = MAX_ARF_LAYERS; - +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf( + " ml-subgop: subgop_cfg->num_steps=%d " + "(cur_frame_idx=%d,frame_index=%d)\n", + subgop_cfg->num_steps, *cur_frame_idx, *frame_index); +#endif for (int idx = 0; idx < subgop_cfg->num_steps; ++idx) { const SubGOPStepCfg *frame = &subgop_cfg->step[idx]; @@ -219,12 +295,23 @@ return NULL; } +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT +static const char frame_update_type[FRAME_UPDATE_TYPES][50] = { + "KF_UPDATE", "LF_UPDATE", "GF_UPDATE", + "ARF_UPDATE", "OVERLAY_UPDATE", "INTNL_OVERLAY_UPDATE", + "INTNL_ARF_UPDATE", "KFFLT_UPDATE", "KFFLT_OVERLAY_UPDATE", +}; +#endif + static int construct_multi_layer_gf_structure( AV1_COMP *cpi, TWO_PASS *twopass, GF_GROUP *const gf_group, RATE_CONTROL *rc, FRAME_INFO *const frame_info, int gf_interval, FRAME_UPDATE_TYPE first_frame_update_type) { int frame_index = 0; int cur_frame_index = 0; +#if CONFIG_MULTIVIEW_CORE + const int num_views = cpi->common.number_layers; +#endif assert(first_frame_update_type == KF_UPDATE || first_frame_update_type == ARF_UPDATE || @@ -240,7 +327,12 @@ subgop_cfg_set, gf_interval, rc->frames_to_key <= gf_interval + 2, first_frame_update_type == KF_UPDATE, use_altref, &is_ld_map); int is_ld_map_first_gop = is_ld_map && first_frame_update_type == KF_UPDATE; - +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" 1: frame_index=%d, cur_frame_index=%d\n", frame_index, + cur_frame_index); + printf("-- first_frame_update_type = %s\n", + frame_update_type[first_frame_update_type]); +#endif if (first_frame_update_type == KF_UPDATE && cpi->oxcf.kf_cfg.enable_keyframe_filtering > 1) { gf_group->has_overlay_for_key_frame = 1; @@ -266,9 +358,26 @@ gf_group->layer_depth[frame_index] = 0; gf_group->max_layer_depth = 0; ++frame_index; - ++cur_frame_index; + cur_frame_index++; +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + // arf update + const int prev_frame_ind = frame_index - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[frame_index] = LF_UPDATE; + gf_group->arf_src_offset[frame_index] = 0; + gf_group->cur_frame_idx[frame_index] = + gf_group->cur_frame_idx[prev_frame_ind] + 1; + gf_group->layer_depth[frame_index] = 0; + gf_group->max_layer_depth = 0; + ++frame_index; + } +#endif } - +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" 2: frame_index=%d, cur_frame_index=%d\n", frame_index, + cur_frame_index); +#endif if (subgop_cfg) { gf_group->subgop_cfg = subgop_cfg; gf_group->is_user_specified = 1; @@ -277,43 +386,117 @@ &frame_index, is_ld_map_first_gop); frame_index++; } else { +#if CONFIG_MULTIVIEW_CORE + if (num_views == 1) { + if (first_frame_update_type == KF_UPDATE) gf_interval++; + } +#else if (first_frame_update_type == KF_UPDATE) gf_interval++; +#endif // ALTREF. if (use_altref) { gf_group->update_type[frame_index] = ARF_UPDATE; +#if CONFIG_MULTIVIEW_CORE + gf_group->arf_src_offset[frame_index] = + num_views * (gf_interval - cur_frame_index - 1); + gf_group->cur_frame_idx[frame_index] = num_views * cur_frame_index; +#else gf_group->arf_src_offset[frame_index] = gf_interval - cur_frame_index - 1; gf_group->cur_frame_idx[frame_index] = cur_frame_index; +#endif gf_group->layer_depth[frame_index] = 1; gf_group->arf_boost[frame_index] = cpi->rc.gfu_boost; gf_group->max_layer_depth = 1; gf_group->arf_index = frame_index; ++frame_index; +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + const int prev_frame_ind = frame_index - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[frame_index] = ARF_UPDATE; + gf_group->arf_src_offset[frame_index] = + gf_group->arf_src_offset[prev_frame_ind] + 1; + gf_group->cur_frame_idx[frame_index] = + gf_group->cur_frame_idx[prev_frame_ind]; + + gf_group->layer_depth[frame_index] = 1; + gf_group->arf_boost[frame_index] = cpi->rc.gfu_boost; + gf_group->max_layer_depth = 1; + gf_group->arf_index = frame_index; + ++frame_index; + } +#endif } else { gf_group->arf_index = -1; } +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" frame_index=%d, cur_frame_index=%d\n", frame_index, + cur_frame_index); +#endif set_multi_layer_params(twopass, gf_group, rc, frame_info, cur_frame_index, gf_interval - 1, &cur_frame_index, &frame_index, - use_altref + 1); + use_altref + 1 +#if CONFIG_MULTIVIEW_CORE + , + num_views +#endif + ); if (use_altref) { - gf_group->update_type[frame_index] = OVERLAY_UPDATE; + gf_group->update_type[frame_index] = + INTNL_OVERLAY_UPDATE; // OVERLAY_UPDATE; gf_group->arf_src_offset[frame_index] = 0; +#if CONFIG_MULTIVIEW_CORE + gf_group->cur_frame_idx[frame_index] = num_views * cur_frame_index; +#else gf_group->cur_frame_idx[frame_index] = cur_frame_index; +#endif gf_group->layer_depth[frame_index] = MAX_ARF_LAYERS; gf_group->arf_boost[frame_index] = NORMAL_BOOST; ++frame_index; +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + const int prev_frame_ind = frame_index - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[frame_index] = + (i == (num_views - 1)) ? OVERLAY_UPDATE : INTNL_OVERLAY_UPDATE; + gf_group->arf_src_offset[frame_index] = 0; + gf_group->cur_frame_idx[frame_index] = + gf_group->cur_frame_idx[prev_frame_ind] + 1; + gf_group->layer_depth[frame_index] = MAX_ARF_LAYERS; + gf_group->arf_boost[frame_index] = NORMAL_BOOST; + ++frame_index; + } +#endif } else { for (; cur_frame_index < gf_interval; ++cur_frame_index) { gf_group->update_type[frame_index] = LF_UPDATE; gf_group->arf_src_offset[frame_index] = 0; +#if CONFIG_MULTIVIEW_CORE + gf_group->cur_frame_idx[frame_index] = cur_frame_index * num_views; +#else gf_group->cur_frame_idx[frame_index] = cur_frame_index; +#endif gf_group->layer_depth[frame_index] = MAX_ARF_LAYERS; gf_group->arf_boost[frame_index] = NORMAL_BOOST; gf_group->max_layer_depth = AOMMAX(gf_group->max_layer_depth, 2); ++frame_index; +#if CONFIG_MULTIVIEW_CORE + for (int i = 1; i < num_views; i++) { + const int prev_frame_ind = frame_index - 1; + assert(prev_frame_ind >= 0); + gf_group->update_type[frame_index] = LF_UPDATE; + gf_group->arf_src_offset[frame_index] = 1; + gf_group->cur_frame_idx[frame_index] = + gf_group->cur_frame_idx[prev_frame_ind]; + gf_group->layer_depth[frame_index] = MAX_ARF_LAYERS; + gf_group->arf_boost[frame_index] = NORMAL_BOOST; + gf_group->max_layer_depth = AOMMAX(gf_group->max_layer_depth, 2); + ++frame_index; + } +#endif } } } - return frame_index; }
diff --git a/av1/encoder/mv_prec.c b/av1/encoder/mv_prec.c index 61db6e6..24d679b 100644 --- a/av1/encoder/mv_prec.c +++ b/av1/encoder/mv_prec.c
@@ -33,7 +33,7 @@ has_second_ref(mbmi))); const MV_REFERENCE_FRAME *ref_frames = mbmi->ref_frame; - const int8_t ref_frame_type = av1_ref_frame_type(ref_frames); + const int16_t ref_frame_type = av1_ref_frame_type(ref_frames); #if CONFIG_SEP_COMP_DRL const CANDIDATE_MV *curr_ref_mv_stack = has_second_drl(mbmi) ? mbmi_ext_frame->ref_mv_stack[ref_idx]
diff --git a/av1/encoder/partition_search.c b/av1/encoder/partition_search.c index c264fc1..7ccf373 100644 --- a/av1/encoder/partition_search.c +++ b/av1/encoder/partition_search.c
@@ -5669,10 +5669,10 @@ #endif // CONFIG_EXTENDED_SDP ) { #if CONFIG_EXTENDED_SDP - const int ref_type = av1_ref_frame_type( + const int16_t ref_type = av1_ref_frame_type( pc_tree->none[pc_tree->region_type]->mic.ref_frame); #else - const int ref_type = av1_ref_frame_type(pc_tree->none->mic.ref_frame); + const int16_t ref_type = av1_ref_frame_type(pc_tree->none->mic.ref_frame); #endif // CONFIG_EXTENDED_SDP av1_update_picked_ref_frames_mask(x, ref_type, bsize, cm->mib_size, mi_row, mi_col);
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c index 1ec40a0..6040b5a 100644 --- a/av1/encoder/pass2_strategy.c +++ b/av1/encoder/pass2_strategy.c
@@ -2236,6 +2236,10 @@ break; } +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" frames_to_key=%d, frames_since_key=%d \n", frames_to_key, + frames_since_key); +#endif // Step on to the next frame. ++frames_to_key; ++frames_since_key; @@ -2430,7 +2434,6 @@ const FIRSTPASS_STATS first_frame = *this_frame; FIRSTPASS_STATS next_frame; av1_zero(next_frame); - rc->frames_since_key = 0; // Use arfs if possible. rc->use_arf_in_this_kf_group = is_altref_enabled( @@ -2796,8 +2799,16 @@ if (cpi->no_show_fwd_kf) { assert(update_type == ARF_UPDATE || update_type == KFFLT_UPDATE); frame_params->frame_type = KEY_FRAME; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set2 frame type to KEY_FRAME: "); + debug_print_multiview_curr_frame(&cpi->common); +#endif } else { frame_params->frame_type = INTER_FRAME; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set2 frame type to INTER_FRAME: "); + debug_print_multiview_curr_frame(&cpi->common); +#endif } if (frame_params->frame_type != KEY_FRAME) @@ -2835,12 +2846,20 @@ rc->active_worst_quality = oxcf->rc_cfg.qp; } +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" frames_to_key= %d, current_frame.view_id=%d\n", rc->frames_to_key, + cpi->common.current_frame.view_id); +#endif // Keyframe and section processing. if (rc->frames_to_key <= 0) { assert(rc->frames_to_key >= -1); FIRSTPASS_STATS this_frame_copy; this_frame_copy = this_frame; frame_params->frame_type = KEY_FRAME; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set3 frame type to KEY_FRAME: "); + debug_print_multiview_curr_frame(&cpi->common); +#endif // Define next KF group and assign bits to it. find_next_key_frame(cpi, &this_frame); this_frame = this_frame_copy; @@ -2935,6 +2954,10 @@ if (cpi->no_show_fwd_kf) { assert(update_type == ARF_UPDATE || update_type == KFFLT_UPDATE); frame_params->frame_type = KEY_FRAME; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf("--- set4 frame type to KEY_FRAME: "); + debug_print_multiview_curr_frame(&cpi->common); +#endif } else { frame_params->frame_type = rc->frames_since_key == 0 ? KEY_FRAME : INTER_FRAME; @@ -3042,6 +3065,9 @@ // Update the active best quality pyramid. if (!rc->is_src_frame_alt_ref) { const int pyramid_level = cpi->gf_group.layer_depth[cpi->gf_group.index]; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" pyramid level = %d \n", pyramid_level); +#endif int i; for (i = pyramid_level; i <= MAX_ARF_LAYERS; ++i) { rc->active_best_quality[i] = cpi->common.quant_params.base_qindex;
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index 268611c..94dc7f4 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -1043,13 +1043,14 @@ oxcf->q_cfg.use_fixed_qp_offsets == 2); assert(oxcf->rc_cfg.mode == AOM_Q); const FRAME_UPDATE_TYPE update_type = gf_group->update_type[gf_index]; - int offset_idx = -1; if (update_type == KF_UPDATE) { +#if !CONFIG_MULTIVIEW_CORE if (rc->frames_to_key <= 1) { // Image / intra-only coding: ignore offsets. return qp; } +#endif offset_idx = 0; } else if (update_type == ARF_UPDATE || update_type == GF_UPDATE || update_type == INTNL_ARF_UPDATE || update_type == LF_UPDATE ||
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c index 71a1671..5cb8c63 100644 --- a/av1/encoder/rd.c +++ b/av1/encoder/rd.c
@@ -576,7 +576,11 @@ #endif // CONFIG_OPTIMIZE_CTX_TIP_WARP for (i = 0; i < REF_CONTEXTS; ++i) { +#if CONFIG_MULTIVIEW_EXTENDED_DPB + for (j = 0; j < EXTENDED_INTER_REFS_CONTEXT; ++j) { +#else for (j = 0; j < INTER_REFS_PER_FRAME - 1; ++j) { +#endif av1_cost_tokens_from_cdf(mode_costs->single_ref_cost[i][j], fc->single_ref_cdf[i][j], NULL); } @@ -584,7 +588,11 @@ for (i = 0; i < REF_CONTEXTS; ++i) { #if CONFIG_SAME_REF_COMPOUND +#if CONFIG_MULTIVIEW_EXTENDED_DPB + for (j = 0; j < EXTENDED_INTER_REFS_CONTEXT; ++j) { +#else for (j = 0; j < INTER_REFS_PER_FRAME - 1; ++j) { +#endif #else for (j = 0; j < INTER_REFS_PER_FRAME - 2; ++j) { #endif // CONFIG_SAME_REF_COMPOUND @@ -595,7 +603,11 @@ for (i = 0; i < REF_CONTEXTS; ++i) { for (j = 0; j < COMPREF_BIT_TYPES; j++) { #if CONFIG_SAME_REF_COMPOUND +#if CONFIG_MULTIVIEW_EXTENDED_DPB + for (int k = 0; k < EXTENDED_INTER_REFS_CONTEXT; ++k) { +#else for (int k = 0; k < INTER_REFS_PER_FRAME - 1; ++k) { +#endif #else for (int k = 0; k < INTER_REFS_PER_FRAME - 2; ++k) { #endif // CONFIG_SAME_REF_COMPOUND @@ -910,6 +922,7 @@ int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) { int64_t rdmult = av1_compute_rd_mult_based_on_qindex(cpi, qindex); +#if !CONFIG_MULTIVIEW_CORE if (is_stat_consumption_stage(cpi) && (cpi->common.current_frame.frame_type != KEY_FRAME)) { const GF_GROUP *const gf_group = &cpi->gf_group; @@ -919,6 +932,7 @@ rdmult = (rdmult * rd_layer_depth_factor[layer_depth]) >> 7; rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7); } +#endif return (int)rdmult; }
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 4e85399..bfcb4b7 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -879,7 +879,7 @@ if (!(this_mode == GLOBALMV || this_mode == NEARMV)) { return 0; } - const uint8_t ref_frame_type = av1_ref_frame_type(ref_frames); + const uint16_t ref_frame_type = av1_ref_frame_type(ref_frames); const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; const int ref_mv_count = mbmi_ext->ref_mv_count[ref_frame_type]; if (ref_mv_count > 1) { @@ -3217,11 +3217,11 @@ assert(single_mode == NEARMV); const int ref_mv_offset = ref_mv_idx; #if CONFIG_SEP_COMP_DRL - const int8_t ref_frame_type = has_second_drl_by_mode(this_mode, ref_frame) - ? ref_frame[ref_idx] - : av1_ref_frame_type(ref_frame); + const int16_t ref_frame_type = has_second_drl_by_mode(this_mode, ref_frame) + ? ref_frame[ref_idx] + : av1_ref_frame_type(ref_frame); #else - const int8_t ref_frame_type = av1_ref_frame_type(ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(ref_frame); #endif if (ref_frame_type > NONE_FRAME && ref_mv_offset < mbmi_ext->ref_mv_count[ref_frame_type]) { @@ -3328,7 +3328,7 @@ if (!ret) return 0; const PREDICTION_MODE single_mode = get_single_mode(this_mode, i); if (single_mode == NEWMV) { - const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const uint16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); #if CONFIG_SEP_COMP_DRL if (has_second_drl(mbmi)) cur_mv[i] = @@ -3501,7 +3501,7 @@ } #endif - const int8_t ref_frame_type = av1_ref_frame_type(ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(ref_frame); int ref_mv_count = ref_frame_type > NONE_FRAME ? mbmi_ext->ref_mv_count[ref_frame_type] : 0; @@ -3539,7 +3539,7 @@ #if !CONFIG_CWG_E099_DRL_WRL_SIMPLIFY const AV1_COMMON *const cm = &cpi->common; const SPEED_FEATURES *const sf = &cpi->sf; - const int8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); #if CONFIG_SEP_COMP_DRL if (sf->inter_sf.reduce_inter_modes && (ref_mv_idx[0] > 0 || ref_mv_idx[1] > 0)) { @@ -3774,7 +3774,7 @@ mbmi->ref_mv_idx > 0) { #endif const int is_comp_pred = has_second_ref(mbmi); - const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const uint16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); int_mv this_refmv[2]; this_refmv[0].as_int = 0; this_refmv[1].as_int = 0; @@ -7138,7 +7138,7 @@ #endif // CONFIG_MORPH_PRED #if !CONFIG_SKIP_MODE_ENHANCEMENT - const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const uint16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); if (x->mbmi_ext->ref_mv_count[ref_frame_type] == UINT8_MAX) { if (x->mbmi_ext->ref_mv_count[ref_frame] == UINT8_MAX || x->mbmi_ext->ref_mv_count[second_ref_frame] == UINT8_MAX) { @@ -7242,7 +7242,7 @@ setup_buffer_ref_mvs_inter(cpi, x, rf_idx, bsize, yv12_mb); } - const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const uint16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); av1_find_mv_refs(cm, xd, mbmi, ref_frame_type, mbmi_ext->ref_mv_count, xd->ref_mv_stack, xd->weight, NULL, mbmi_ext->global_mvs @@ -7587,7 +7587,7 @@ #if CONFIG_IBC_SR_EXT mbmi->use_intrabc[xd->tree_type == CHROMA_PART] = 0; #endif // CONFIG_IBC_SR_EXT - const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); + const int16_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame); if (x->mbmi_ext->ref_mv_count[ref_frame_type] == UINT8_MAX) { if (x->mbmi_ext->ref_mv_count[ref_frame] == UINT8_MAX || x->mbmi_ext->ref_mv_count[second_ref_frame] == UINT8_MAX) { @@ -8551,9 +8551,9 @@ return 1; } #if CONFIG_SAME_REF_COMPOUND - const uint8_t ref_type = av1_ref_frame_type(ref_frame); + const uint16_t ref_type = av1_ref_frame_type(ref_frame); #else - const int ref_type = av1_ref_frame_type(ref_frame); + const int16_t ref_type = av1_ref_frame_type(ref_frame); #endif // CONFIG_SAME_REF_COMPOUND if (prune_ref_frame(cpi, x, ref_type)) return 1; @@ -9805,10 +9805,17 @@ // Note that the 0-th element will contain a cut-off that is later used // to determine if we should skip a compound mode. +#if CONFIG_MULTIVIEW_EXTENDED_DPB + int64_t ref_frame_rd[SINGLE_REF_FRAMES] = { + INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, + INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, + INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX + }; +#else int64_t ref_frame_rd[SINGLE_REF_FRAMES] = { INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX }; - +#endif // Prepared stats used later to check if we could skip intra mode eval. int64_t inter_cost = -1; int64_t intra_cost = -1;
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c index 3c9044a..907af1d 100644 --- a/av1/encoder/tpl_model.c +++ b/av1/encoder/tpl_model.c
@@ -1016,7 +1016,6 @@ int anc_frame_offset = gf_group->cur_frame_idx[cur_frame_idx]; int process_frame_count = 0; const int gop_length = get_gop_length(gf_group); - for (gf_index = cur_frame_idx; gf_index < gop_length; ++gf_index) { TplDepFrame *tpl_frame = &tpl_data->tpl_frame[gf_index]; FRAME_UPDATE_TYPE frame_update_type = gf_group->update_type[gf_index]; @@ -1077,9 +1076,15 @@ av1_get_ref_frames_enc(cm, true_disp, ref_frame_map_pairs); else av1_get_ref_frames(cm, true_disp, ref_frame_map_pairs); - int refresh_mask = - av1_get_refresh_frame_flags(cpi, &frame_params, frame_update_type, - gf_index, true_disp, ref_frame_map_pairs); + int refresh_mask = av1_get_refresh_frame_flags( + cpi, &frame_params, frame_update_type, gf_index, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + 0, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + -1, cm->number_layers, +#endif + true_disp, ref_frame_map_pairs); int refresh_frame_map_index = av1_get_refresh_ref_frame_map(cm, refresh_mask); @@ -1125,7 +1130,6 @@ MAX_TPL_EXTEND, cpi->rc.frames_to_key - cpi->rc.baseline_gf_interval); int frame_display_index = gf_group->cur_frame_idx[gop_length - 1] + gf_group->arf_src_offset[gop_length - 1] + 1; - for (; gf_index < MAX_TPL_FRAME_IDX && extend_frame_count < extend_frame_length; ++gf_index) { @@ -1162,6 +1166,9 @@ gf_group->update_type[gf_index] = LF_UPDATE; gf_group->q_val[gf_index] = *pframe_qindex; +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" init_gop_frames_for_tpl: pframe_qindex=%2d\n", *pframe_qindex); +#endif const int true_disp = (int)(tpl_frame->frame_display_index) - @@ -1176,6 +1183,12 @@ // frame_update_type. int refresh_mask = av1_get_refresh_frame_flags(cpi, &frame_params, frame_update_type, -1, +#if CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + 0, +#endif +#if CONFIG_MULTIVIEW_SEPARATE_DPB + -1, cm->number_layers, +#endif true_disp, ref_frame_map_pairs); int refresh_frame_map_index = av1_get_refresh_ref_frame_map(cm, refresh_mask); @@ -1255,7 +1268,10 @@ gf_group->q_val[gf_index] = av1_rc_pick_q_and_bounds(cpi, &cpi->rc, cm->width, cm->height, gf_index, &bottom_index, &top_index); - +#if CONFIG_MULTIVIEW_CORE && CONFIG_MULTIVIEW_DEBUG_PROMPT + printf(" av1_tpl_setup_stats: q_val[%d]=%2d\n", gf_index, + gf_group->q_val[gf_index]); +#endif cm->current_frame.frame_type = INTER_FRAME; }
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake index 5b57270..6c925e2 100644 --- a/build/cmake/aom_config_defaults.cmake +++ b/build/cmake/aom_config_defaults.cmake
@@ -141,6 +141,38 @@ set_aom_config_var(CONFIG_ZERO_OFFSET_BITUPSHIFT 1 "Use zero offset for non-normative bit upshift") +set_aom_config_var(CONFIG_MULTIVIEW_CORE 0 + "Core changes in the codebase for multiview coding support.") +set_aom_config_var(CONFIG_MULTIVIEW_SEPARATE_DPB 0 + "separate dpb support for multiview.") +set_aom_config_var(CONFIG_MULTIVIEW_EXTENDED_DPB 0 + "extended dpb support for multiview.") + +set_aom_config_var(CONFIG_MULTIVIEW_DEBUG 0 + "Debug functionality macro for multiview coding.") +set_aom_config_var(CONFIG_MULTIVIEW_DEBUG_PROMPT 0 + "Debug macro for prompting on display.") +set_aom_config_var(CONFIG_MULTIVIEW_DEBUG_LOGFILES 0 + "Debug create logfiles for encode and decode.") + +set_aom_config_var( + CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST + 0 + "Enable temporal scalability using the implicit reference frame list construction" +) +set_aom_config_var( + CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_ENCODER + 0 + "Enable temporal scalability encoding with ref map slot allocation and frame drop" +) +# Bugfix for error resilience case for +# CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST +set_aom_config_var( + CONFIG_MULTILAYER_TEMPORAL_SCALABILITY_REFLIST_FIX_ERROR_RES + 0 + "Enable temporal scalability using the implicit reference frame list construction" +) + # AV2 experiment flags. set_aom_config_var(CONFIG_IMPROVEIDTX 1 "AV2 enable improved identity transform coding.")
diff --git a/build/cmake/aom_experiment_deps.cmake b/build/cmake/aom_experiment_deps.cmake index 04db6a8..3d82a4f 100644 --- a/build/cmake/aom_experiment_deps.cmake +++ b/build/cmake/aom_experiment_deps.cmake
@@ -56,6 +56,11 @@ change_config_and_warn(CONFIG_ENABLE_MHCCP 0 !CONFIG_EXT_RECUR_PARTITIONS) endif() + # CONFIG_MULTIVIEW_SEPARATE_DPB is depdendent on CONFIG_MULTIVIEW_CORE + if(NOT CONFIG_MULTIVIEW_CORE AND CONFIG_MULTIVIEW_SEPARATE_DPB) + change_config_and_warn(CONFIG_MULTIVIEW_SEPARATE_DPB 0 + !CONFIG_MULTIVIEW_CORE) + endif() # CONFIG_CB1TO4_SPLIT is dependent on CONFIG_EXT_RECUR_PARTITIONS. if(NOT CONFIG_EXT_RECUR_PARTITIONS AND CONFIG_CB1TO4_SPLIT) change_config_and_warn(CONFIG_CB1TO4_SPLIT 0 !CONFIG_EXT_RECUR_PARTITIONS)
diff --git a/common/ivfdec.c b/common/ivfdec.c index 7eee07f..6437d6e 100644 --- a/common/ivfdec.c +++ b/common/ivfdec.c
@@ -32,8 +32,11 @@ int file_is_ivf(struct AvxInputContext *input_ctx) { char raw_hdr[32]; int is_ivf = 0; - +#if CONFIG_MULTIVIEW_CORE + if (fread(raw_hdr, 1, 32, input_ctx->file[0]) == 32) { +#else if (fread(raw_hdr, 1, 32, input_ctx->file) == 32) { +#endif if (memcmp(IVF_SIGNATURE, raw_hdr, 4) == 0) { is_ivf = 1; @@ -54,10 +57,19 @@ } if (!is_ivf) { +#if CONFIG_MULTIVIEW_CORE + rewind(input_ctx->file[0]); + input_ctx->detect[0].buf_read = 0; +#else rewind(input_ctx->file); input_ctx->detect.buf_read = 0; +#endif } else { +#if CONFIG_MULTIVIEW_CORE + input_ctx->detect[0].position = 4; +#else input_ctx->detect.position = 4; +#endif } return is_ivf; }
diff --git a/common/obudec.c b/common/obudec.c index 4826f92..1ac8079 100644 --- a/common/obudec.c +++ b/common/obudec.c
@@ -276,7 +276,11 @@ struct AvxInputContext *avx_ctx = obu_ctx->avx_ctx; uint8_t detect_buf[OBU_DETECTION_SIZE] = { 0 }; const int is_annexb = obu_ctx->is_annexb; +#if CONFIG_MULTIVIEW_CORE + FILE *f = avx_ctx->file[0]; +#else FILE *f = avx_ctx->file; +#endif size_t payload_length = 0; ObuHeader obu_header; memset(&obu_header, 0, sizeof(obu_header)); @@ -369,7 +373,11 @@ int obudec_read_temporal_unit(struct ObuDecInputContext *obu_ctx, uint8_t **buffer, size_t *bytes_read, size_t *buffer_size) { +#if CONFIG_MULTIVIEW_CORE + FILE *f = obu_ctx->avx_ctx->file[0]; +#else FILE *f = obu_ctx->avx_ctx->file; +#endif if (!f) return -1; *buffer_size = 0;
diff --git a/common/stream_iter.c b/common/stream_iter.c index 078be62..fe1230d 100644 --- a/common/stream_iter.c +++ b/common/stream_iter.c
@@ -18,15 +18,31 @@ #include "common/y4minput.h" static int copy_reader(StreamIter *iter, aom_image_t *img) { +#if CONFIG_MULTIVIEW_CORE + const int view_id = iter->view_id; +#endif struct AvxInputContext *input_ctx = iter->input.avx; +#if CONFIG_MULTIVIEW_CORE + FILE *f = input_ctx->file[view_id]; + y4m_input *y4m = &input_ctx->y4m[view_id]; +#else FILE *f = input_ctx->file; y4m_input *y4m = &input_ctx->y4m; +#endif int shortread = 0; +#if CONFIG_MULTIVIEW_CORE + if (input_ctx->file_type[view_id] == FILE_TYPE_Y4M) { +#else if (input_ctx->file_type == FILE_TYPE_Y4M) { +#endif if (y4m_input_fetch_frame(y4m, f, img) < 1) return 0; } else { +#if CONFIG_MULTIVIEW_CORE + shortread = read_yuv_frame(input_ctx, img, view_id); +#else shortread = read_yuv_frame(input_ctx, img); +#endif } return !shortread; } @@ -34,6 +50,9 @@ void copy_stream_iter_init(StreamIter *iter, struct AvxInputContext *input) { iter->input.avx = input; iter->reader = copy_reader; +#if CONFIG_MULTIVIEW_CORE + iter->view_id = 0; +#endif } static int skip_reader(StreamIter *iter, aom_image_t *raw) { @@ -96,6 +115,9 @@ assert(limit >= 0); iter->input.stream = input; iter->current = 0; +#if CONFIG_MULTIVIEW_CORE + iter->view_id = 0; +#endif iter->n = limit; iter->reader = limit_reader; }
diff --git a/common/stream_iter.h b/common/stream_iter.h index 73781c7..751f356 100644 --- a/common/stream_iter.h +++ b/common/stream_iter.h
@@ -36,6 +36,9 @@ // iterator, do not use either field. int current; int n; +#if CONFIG_MULTIVIEW_CORE + int view_id; +#endif // Pointer to the function that performs the read. Returns whether a frame // is available. If a frame is returned, it is written into *raw. int (*reader)(struct StreamIter *iter, aom_image_t *raw);
diff --git a/common/tools_common.c b/common/tools_common.c index 0e4db53..a80a5af 100644 --- a/common/tools_common.c +++ b/common/tools_common.c
@@ -77,10 +77,17 @@ if (detail) printf(" %s\n", detail); exit(EXIT_FAILURE); } - +#if CONFIG_MULTIVIEW_CORE +int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame, + int view_id) { + assert(view_id >= 0); + FILE *f = input_ctx->file[view_id]; + struct FileTypeDetectionBuffer *detect = &input_ctx->detect[view_id]; +#else int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame) { FILE *f = input_ctx->file; struct FileTypeDetectionBuffer *detect = &input_ctx->detect; +#endif int plane = 0; int shortread = 0; const int bytespp = (yuv_frame->fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 2 : 1;
diff --git a/common/tools_common.h b/common/tools_common.h index 34368dc..d34af53 100644 --- a/common/tools_common.h +++ b/common/tools_common.h
@@ -71,6 +71,10 @@ #define AV1_FOURCC 0x31305641 +#if CONFIG_MULTIVIEW_CORE +#define NUM_LAYERS_MAX 256 // maximum number of layers supported +#endif + enum VideoFileType { FILE_TYPE_OBU, FILE_TYPE_RAW, @@ -101,11 +105,22 @@ }; struct AvxInputContext { +#if CONFIG_MULTIVIEW_CORE + const char *filename[NUM_LAYERS_MAX]; + FILE *file[NUM_LAYERS_MAX]; + int num_views; +#else const char *filename; FILE *file; +#endif int64_t length; +#if CONFIG_MULTIVIEW_CORE + struct FileTypeDetectionBuffer detect[NUM_LAYERS_MAX]; + enum VideoFileType file_type[NUM_LAYERS_MAX]; +#else struct FileTypeDetectionBuffer detect; enum VideoFileType file_type; +#endif uint32_t width; uint32_t height; struct AvxRational pixel_aspect_ratio; @@ -115,8 +130,12 @@ uint32_t fourcc; struct AvxRational framerate; #if CONFIG_AV1_ENCODER +#if CONFIG_MULTIVIEW_CORE + y4m_input y4m[NUM_LAYERS_MAX]; +#else y4m_input y4m; #endif +#endif aom_color_range_t color_range; }; @@ -163,7 +182,12 @@ // If the interface is unknown, returns 0. uint32_t get_fourcc_by_aom_decoder(aom_codec_iface_t *iface); +#if CONFIG_MULTIVIEW_CORE +int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame, + int view_id); +#else int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame); +#endif void aom_img_write(const aom_image_t *img, FILE *file); int aom_img_read(aom_image_t *img, FILE *file);
diff --git a/common/video_reader.c b/common/video_reader.c index 2cb653e..546e171 100644 --- a/common/video_reader.c +++ b/common/video_reader.c
@@ -42,24 +42,41 @@ return NULL; // Can't allocate AvxVideoReader } +#if CONFIG_MULTIVIEW_CORE + reader->input_ctx.filename[0] = filename; + reader->input_ctx.file[0] = file; +#else reader->input_ctx.filename = filename; reader->input_ctx.file = file; +#endif reader->obu_ctx.avx_ctx = &reader->input_ctx; if (file_is_ivf(&reader->input_ctx)) { +#if CONFIG_MULTIVIEW_CORE + reader->input_ctx.file_type[0] = FILE_TYPE_IVF; +#else reader->input_ctx.file_type = FILE_TYPE_IVF; +#endif reader->info.codec_fourcc = reader->input_ctx.fourcc; reader->info.frame_width = reader->input_ctx.width; reader->info.frame_height = reader->input_ctx.height; #if CONFIG_WEBM_IO } else if (file_is_webm(&reader->webm_ctx, &reader->input_ctx)) { +#if CONFIG_MULTIVIEW_CORE + reader->input_ctx.file_type[0] = FILE_TYPE_WEBM; +#else reader->input_ctx.file_type = FILE_TYPE_WEBM; +#endif reader->info.codec_fourcc = reader->input_ctx.fourcc; reader->info.frame_width = reader->input_ctx.width; reader->info.frame_height = reader->input_ctx.height; #endif } else if (file_is_obu(&reader->obu_ctx)) { +#if CONFIG_MULTIVIEW_CORE + reader->input_ctx.file_type[0] = FILE_TYPE_OBU; +#else reader->input_ctx.file_type = FILE_TYPE_OBU; +#endif // assume AV1 reader->info.codec_fourcc = AV1_FOURCC; reader->info.is_annexb = reader->obu_ctx.is_annexb; @@ -74,8 +91,13 @@ void aom_video_reader_close(AvxVideoReader *reader) { if (reader) { +#if CONFIG_MULTIVIEW_CORE + fclose(reader->input_ctx.file[0]); + if (reader->input_ctx.file_type[0] == FILE_TYPE_OBU) { +#else fclose(reader->input_ctx.file); if (reader->input_ctx.file_type == FILE_TYPE_OBU) { +#endif obudec_free(&reader->obu_ctx); } free(reader->buffer); @@ -84,16 +106,29 @@ } int aom_video_reader_read_frame(AvxVideoReader *reader) { +#if CONFIG_MULTIVIEW_CORE + if (reader->input_ctx.file_type[0] == FILE_TYPE_IVF) { + return !ivf_read_frame(reader->input_ctx.file[0], &reader->buffer, +#else if (reader->input_ctx.file_type == FILE_TYPE_IVF) { return !ivf_read_frame(reader->input_ctx.file, &reader->buffer, +#endif &reader->frame_size, &reader->buffer_size, &reader->pts); +#if CONFIG_MULTIVIEW_CORE + } else if (reader->input_ctx.file_type[0] == FILE_TYPE_OBU) { +#else } else if (reader->input_ctx.file_type == FILE_TYPE_OBU) { +#endif return !obudec_read_temporal_unit(&reader->obu_ctx, &reader->buffer, &reader->frame_size, &reader->buffer_size); #if CONFIG_WEBM_IO +#if CONFIG_MULTIVIEW_CORE + } else if (reader->input_ctx.file_type[0] == FILE_TYPE_WEBM) { +#else } else if (reader->input_ctx.file_type == FILE_TYPE_WEBM) { +#endif return !webm_read_frame(&reader->webm_ctx, &reader->buffer, &reader->frame_size, &reader->buffer_size); #endif @@ -115,7 +150,11 @@ } FILE *aom_video_reader_get_file(AvxVideoReader *reader) { +#if CONFIG_MULTIVIEW_CORE + return reader->input_ctx.file[0]; +#else return reader->input_ctx.file; +#endif } const AvxVideoInfo *aom_video_reader_get_info(AvxVideoReader *reader) {
diff --git a/common/webmdec.cc b/common/webmdec.cc index b3601d4..ae96f24 100644 --- a/common/webmdec.cc +++ b/common/webmdec.cc
@@ -56,7 +56,11 @@ void rewind_and_reset(struct WebmInputContext *const webm_ctx, struct AvxInputContext *const aom_ctx) { +#if CONFIG_MULTIVIEW_CORE + rewind(aom_ctx->file[0]); +#else rewind(aom_ctx->file); +#endif reset(webm_ctx); } @@ -64,7 +68,12 @@ int file_is_webm(struct WebmInputContext *webm_ctx, struct AvxInputContext *aom_ctx) { +#if CONFIG_MULTIVIEW_CORE + mkvparser::MkvReader *const reader = + new mkvparser::MkvReader(aom_ctx->file[0]); +#else mkvparser::MkvReader *const reader = new mkvparser::MkvReader(aom_ctx->file); +#endif webm_ctx->reader = reader; webm_ctx->reached_eos = 0;
diff --git a/examples/scalable_decoder.c b/examples/scalable_decoder.c index 5e95732..117fbd4 100644 --- a/examples/scalable_decoder.c +++ b/examples/scalable_decoder.c
@@ -110,8 +110,13 @@ if (!(inputfile = fopen(argv[1], "rb"))) die("Failed to open %s for read.", argv[1]); +#if CONFIG_MULTIVIEW_CORE + obu_ctx.avx_ctx->file[0] = inputfile; + obu_ctx.avx_ctx->filename[0] = argv[1]; +#else obu_ctx.avx_ctx->file = inputfile; obu_ctx.avx_ctx->filename = argv[1]; +#endif aom_codec_iface_t *decoder = get_aom_decoder_by_index(0); printf("Using %s\n", aom_codec_iface_name(decoder));
diff --git a/test/webm_video_source.h b/test/webm_video_source.h index 0f4755a..1f330d2 100644 --- a/test/webm_video_source.h +++ b/test/webm_video_source.h
@@ -32,7 +32,11 @@ frame_number_(0), end_of_file_(false) {} virtual ~WebMVideoSource() { +#if CONFIG_MULTIVIEW_CORE + if (aom_ctx_->file[0] != NULL) fclose(aom_ctx_->file[0]); +#else if (aom_ctx_->file != NULL) fclose(aom_ctx_->file); +#endif webm_free(webm_ctx_); delete aom_ctx_; delete webm_ctx_; @@ -41,9 +45,15 @@ virtual void Init() {} virtual void Begin() { +#if CONFIG_MULTIVIEW_CORE + aom_ctx_->file[0] = OpenTestDataFile(file_name_); + ASSERT_TRUE(aom_ctx_->file[0] != NULL) + << "Input file open failed. Filename: " << file_name_; +#else aom_ctx_->file = OpenTestDataFile(file_name_); ASSERT_TRUE(aom_ctx_->file != NULL) << "Input file open failed. Filename: " << file_name_; +#endif ASSERT_EQ(file_is_webm(webm_ctx_, aom_ctx_), 1) << "file is not WebM"; @@ -56,7 +66,11 @@ } void FillFrame() { +#if CONFIG_MULTIVIEW_CORE + ASSERT_TRUE(aom_ctx_->file[0] != NULL); +#else ASSERT_TRUE(aom_ctx_->file != NULL); +#endif const int status = webm_read_frame(webm_ctx_, &buf_, &frame_sz_, &buf_sz_); ASSERT_GE(status, 0) << "webm_read_frame failed"; if (status == 1) { @@ -65,7 +79,11 @@ } void SeekToNextKeyFrame() { +#if CONFIG_MULTIVIEW_CORE + ASSERT_TRUE(aom_ctx_->file[0] != NULL); +#else ASSERT_TRUE(aom_ctx_->file != NULL); +#endif do { const int status = webm_read_frame(webm_ctx_, &buf_, &frame_sz_, &buf_sz_);
diff --git a/tools/dump_obu.cc b/tools/dump_obu.cc index 755e5f1..e6956bf 100644 --- a/tools/dump_obu.cc +++ b/tools/dump_obu.cc
@@ -64,10 +64,18 @@ } bool ReadTemporalUnit(InputContext *ctx, size_t *unit_size) { +#if CONFIG_MULTIVIEW_CORE + const VideoFileType file_type = ctx->avx_ctx->file_type[0]; +#else const VideoFileType file_type = ctx->avx_ctx->file_type; +#endif switch (file_type) { case FILE_TYPE_IVF: { +#if CONFIG_MULTIVIEW_CORE + if (ivf_read_frame(ctx->avx_ctx->file[0], &ctx->unit_buffer, unit_size, +#else if (ivf_read_frame(ctx->avx_ctx->file, &ctx->unit_buffer, unit_size, +#endif &ctx->unit_buffer_size, NULL)) { return false; } @@ -130,8 +138,13 @@ #endif input_ctx.Init(); +#if CONFIG_MULTIVIEW_CORE + avx_ctx.file[0] = input_file.get(); + avx_ctx.file_type[0] = GetFileType(&input_ctx); +#else avx_ctx.file = input_file.get(); avx_ctx.file_type = GetFileType(&input_ctx); +#endif // Note: the reader utilities will realloc the buffer using realloc() etc. // Can't have nice things like unique_ptr wrappers with that type of