ref: 9e73bb8f42673674c6ddf1e87e72f2c7edaec52e
parent: 60cc1b8b9eab94545d9a41dca25b123da924442b
author: Derek Buitenhuis <derek.buitenhuis@gmail.com>
date: Tue Oct 2 09:30:32 EDT 2018
ref_mvs: Check malloc in av1_init_ref_mv_common and propagate error Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
--- a/src/decode.c
+++ b/src/decode.c
@@ -2405,13 +2405,14 @@
if ((f->frame_hdr.frame_type & 1) || f->frame_hdr.allow_intrabc) {f->mvs = f->mvs_ref->data;
const int order_hint_n_bits = f->seq_hdr.order_hint * f->seq_hdr.order_hint_n_bits;
- av1_init_ref_mv_common(f->libaom_cm, f->bw >> 1, f->bh >> 1,
- f->b4_stride, f->seq_hdr.sb128,
- f->mvs, f->ref_mvs, f->cur.p.poc, f->refpoc,
- f->refrefpoc, f->frame_hdr.gmv,
- f->frame_hdr.hp, f->frame_hdr.force_integer_mv,
- f->frame_hdr.use_ref_frame_mvs,
- order_hint_n_bits);
+ const int ret = av1_init_ref_mv_common(f->libaom_cm, f->bw >> 1, f->bh >> 1,
+ f->b4_stride, f->seq_hdr.sb128,
+ f->mvs, f->ref_mvs, f->cur.p.poc, f->refpoc,
+ f->refrefpoc, f->frame_hdr.gmv,
+ f->frame_hdr.hp, f->frame_hdr.force_integer_mv,
+ f->frame_hdr.use_ref_frame_mvs,
+ order_hint_n_bits);
+ if (ret < 0) return -ENOMEM;
if (c->n_fc == 1 && f->frame_hdr.use_ref_frame_mvs)
av1_init_ref_mv_tile_row(f->libaom_cm, 0, f->bw, 0, f->bh);
}
--- a/src/ref_mvs.c
+++ b/src/ref_mvs.c
@@ -48,6 +48,7 @@
#include "config.h"
#include <assert.h>
+#include <errno.h>
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
@@ -3468,25 +3469,26 @@
}
}
-void av1_init_ref_mv_common(AV1_COMMON *cm,
- const int w8, const int h8,
- const ptrdiff_t stride,
- const int allow_sb128,
- MV_REF *cur,
- MV_REF *ref_mvs[7],
- const unsigned cur_poc,
- const unsigned ref_poc[7],
- const unsigned ref_ref_poc[7][7],
- const WarpedMotionParams gmv[7],
- const int allow_hp,
- const int force_int_mv,
- const int allow_ref_frame_mvs,
- const int order_hint)
+int av1_init_ref_mv_common(AV1_COMMON *cm,
+ const int w8, const int h8,
+ const ptrdiff_t stride,
+ const int allow_sb128,
+ MV_REF *cur,
+ MV_REF *ref_mvs[7],
+ const unsigned cur_poc,
+ const unsigned ref_poc[7],
+ const unsigned ref_ref_poc[7][7],
+ const WarpedMotionParams gmv[7],
+ const int allow_hp,
+ const int force_int_mv,
+ const int allow_ref_frame_mvs,
+ const int order_hint)
{ if (cm->mi_cols != (w8 << 1) || cm->mi_rows != (h8 << 1)) {const int align_h = (h8 + 15) & ~15;
if (cm->tpl_mvs) free(cm->tpl_mvs);
cm->tpl_mvs = malloc(sizeof(*cm->tpl_mvs) * (stride >> 1) * align_h);
+ if (!cm->tpl_mvs) return -ENOMEM;
for (int i = 0; i < 7; i++)
cm->frame_refs[i].idx = i;
cm->mi_cols = w8 << 1;
@@ -3528,6 +3530,8 @@
cm->ref_frame_sign_bias[1 + i] = get_relative_dist(cm, ref_poc, cur_poc) > 0;
}
av1_setup_motion_field(cm);
+
+ return 0;
}
void av1_init_ref_mv_tile_row(AV1_COMMON *cm,
--- a/src/ref_mvs.h
+++ b/src/ref_mvs.h
@@ -36,18 +36,18 @@
void av1_free_ref_mv_common(AV1_COMMON *cm);
// call once per frame
-void av1_init_ref_mv_common(AV1_COMMON *cm,
- int w8, int h8,
- ptrdiff_t stride,
- int allow_sb128,
- refmvs *cur,
- refmvs *ref_mvs[7],
- unsigned cur_poc,
- const unsigned ref_poc[7],
- const unsigned ref_ref_poc[7][7],
- const WarpedMotionParams gmv[7],
- int allow_hp, int force_int_mv,
- int allow_ref_frame_mvs, int order_hint);
+int av1_init_ref_mv_common(AV1_COMMON *cm,
+ int w8, int h8,
+ ptrdiff_t stride,
+ int allow_sb128,
+ refmvs *cur,
+ refmvs *ref_mvs[7],
+ unsigned cur_poc,
+ const unsigned ref_poc[7],
+ const unsigned ref_ref_poc[7][7],
+ const WarpedMotionParams gmv[7],
+ int allow_hp, int force_int_mv,
+ int allow_ref_frame_mvs, int order_hint);
// call for start of each sbrow per tile
void av1_init_ref_mv_tile_row(AV1_COMMON *cm,