shithub: libvpx

Info  •  Files  •  Log

Merge "vpx_dsp/loopfilter.c: fix int sanitizer warnings"

--- a/vpx_dsp/loopfilter.c

+++ b/vpx_dsp/loopfilter.c

@@ -81,11 +81,11 @@

                            uint8_t *op0, uint8_t *oq0, uint8_t *oq1) {

   int8_t filter1, filter2;

-  const int8_t ps1 = (int8_t)*op1 ^ 0x80;

-  const int8_t ps0 = (int8_t)*op0 ^ 0x80;

-  const int8_t qs0 = (int8_t)*oq0 ^ 0x80;

-  const int8_t qs1 = (int8_t)*oq1 ^ 0x80;

-  const uint8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1);

+  const int8_t ps1 = (int8_t)(*op1 ^ 0x80);

+  const int8_t ps0 = (int8_t)(*op0 ^ 0x80);

+  const int8_t qs0 = (int8_t)(*oq0 ^ 0x80);

+  const int8_t qs1 = (int8_t)(*oq1 ^ 0x80);

+  const int8_t hev = hev_mask(thresh, *op1, *op0, *oq0, *oq1);

   // add outer taps if we have high edge variance

   int8_t filter = signed_char_clamp(ps1 - qs1) & hev;

@@ -99,14 +99,14 @@

   filter1 = signed_char_clamp(filter + 4) >> 3;

   filter2 = signed_char_clamp(filter + 3) >> 3;

-  *oq0 = signed_char_clamp(qs0 - filter1) ^ 0x80;

-  *op0 = signed_char_clamp(ps0 + filter2) ^ 0x80;

+  *oq0 = (uint8_t)(signed_char_clamp(qs0 - filter1) ^ 0x80);

+  *op0 = (uint8_t)(signed_char_clamp(ps0 + filter2) ^ 0x80);

   // outer tap adjustments

   filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev;

-  *oq1 = signed_char_clamp(qs1 - filter) ^ 0x80;

-  *op1 = signed_char_clamp(ps1 + filter) ^ 0x80;

+  *oq1 = (uint8_t)(signed_char_clamp(qs1 - filter) ^ 0x80);

+  *op1 = (uint8_t)(signed_char_clamp(ps1 + filter) ^ 0x80);

 }

 void vpx_lpf_horizontal_4_c(uint8_t *s, int pitch, const uint8_t *blimit,

@@ -423,7 +423,7 @@

   const int16_t ps0 = (int16_t)*op0 - (0x80 << shift);

   const int16_t qs0 = (int16_t)*oq0 - (0x80 << shift);

   const int16_t qs1 = (int16_t)*oq1 - (0x80 << shift);

-  const uint16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd);

+  const int16_t hev = highbd_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd);

   // Add outer taps if we have high edge variance.

   int16_t filter = signed_char_clamp_high(ps1 - qs1, bd) & hev;