shithub: dav1d

Info  •  Files  •  Log  •  Branches

loopfilter: limit filter width to the frame edge

Fixes ubsan index-out-of-bounds error in loop_filter_v_sb128y_c() with
clusterfuzz-testcase-minimized-dav1d_fuzzer-5691087507685376. Credits to
oss-fuzz.

--- a/src/lf_apply_tmpl.c

+++ b/src/lf_apply_tmpl.c

@@ -235,7 +235,8 @@

              x < f->sb128w; x++, a++)

         {

             uint16_t (*const y_vmask)[2] = lflvl[x].filter_y[1][starty4];

-            for (unsigned mask = 1, i = 0; i < 32; mask <<= 1, i++) {

+            const unsigned w = imin(32, (f->w4 >> sbl2) - x);

+            for (unsigned mask = 1, i = 0; i < w; mask <<= 1, i++) {

                 const int sidx = mask >= 0x10000U;

                 const unsigned smask = mask >> (sidx << 4);

                 const int idx = 2 * !!(y_vmask[2][sidx] & smask) +

@@ -247,8 +248,9 @@

             }

             if (f->cur.p.p.layout != DAV1D_PIXEL_LAYOUT_I400) {

+                const unsigned cw = (w + ss_hor) >> ss_hor;

                 uint16_t (*const uv_vmask)[2] = lflvl[x].filter_uv[1][starty4 >> ss_ver];

-                for (unsigned uv_mask = 1, i = 0; i < (32U >> ss_hor); uv_mask <<= 1, i++) {

+                for (unsigned uv_mask = 1, i = 0; i < cw; uv_mask <<= 1, i++) {

                     const int sidx = uv_mask >= hmax;

                     const unsigned smask = uv_mask >> (sidx << (4 - ss_hor));

                     const int idx = !!(uv_vmask[1][sidx] & smask);