ref: 766d875b9da5c088d62fa50c37194c6e0dc0861a
parent: 35e98a70ba65d82f0b94df3f5e453a3a9f0d6365
author: Luc Trudeau <luc@trud.ca>
date: Mon May 28 11:04:57 EDT 2018
VSX version of vpx_mbpost_proc_ip Low bit depth version only. Passes the VpxMbPostProcAcrossIpTest. VpxMbPostProcAcrossIpTest Speed Test (POWER8 Model 2.1) C time = 188.5ms (±0.2ms), VSX time = 65.2ms (±0.1ms) [2.9x] Change-Id: I1cf72365d94a9d7f1e9323925a87a30e3bd5cfe2
--- a/test/pp_filter_test.cc
+++ b/test/pp_filter_test.cc
@@ -164,11 +164,21 @@
}
class VpxMbPostProcAcrossIpTest
- : public ::testing::TestWithParam<VpxMbPostProcAcrossIpFunc> {+ : public AbstractBench,
+ public ::testing::TestWithParam<VpxMbPostProcAcrossIpFunc> {public:
+ VpxMbPostProcAcrossIpTest()
+ : rows(16), cols(16), mbPostProcAcrossIp(GetParam()),
+ src(Buffer<uint8_t>(rows, cols, 8, 8, 17, 8)) {} virtual void TearDown() { libvpx_test::ClearSystemState(); }protected:
+ const int rows;
+ const int cols;
+ const VpxMbPostProcAcrossIpFunc mbPostProcAcrossIp;
+ Buffer<uint8_t> src;
+ void run();
+
void SetCols(unsigned char *s, int rows, int cols, int src_width) { for (int r = 0; r < rows; r++) { for (int c = 0; c < cols; c++) {@@ -197,11 +207,11 @@
}
};
-TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) {- const int rows = 16;
- const int cols = 16;
+void VpxMbPostProcAcrossIpTest::run() {+ mbPostProcAcrossIp(src.TopLeftPixel(), src.stride(), rows, cols, q2mbl(0));
+}
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
+TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) {ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
@@ -215,15 +225,11 @@
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckMediumFilterOutput) {- const int rows = 16;
- const int cols = 16;
-
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
- static const unsigned char kExpectedOutput[cols] = {+ static const unsigned char kExpectedOutput[] = {2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 13
};
@@ -232,15 +238,11 @@
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckHighFilterOutput) {- const int rows = 16;
- const int cols = 16;
-
- Buffer<uint8_t> src = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(src.Init());
src.SetPadding(10);
SetCols(src.TopLeftPixel(), rows, cols, src.stride());
- static const unsigned char kExpectedOutput[cols] = {+ static const unsigned char kExpectedOutput[] = {2, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13
};
@@ -254,9 +256,6 @@
}
TEST_P(VpxMbPostProcAcrossIpTest, CheckCvsAssembly) {- const int rows = 16;
- const int cols = 16;
-
Buffer<uint8_t> c_mem = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
ASSERT_TRUE(c_mem.Init());
Buffer<uint8_t> asm_mem = Buffer<uint8_t>(cols, rows, 8, 8, 17, 8);
@@ -279,6 +278,16 @@
}
}
+TEST_P(VpxMbPostProcAcrossIpTest, DISABLED_Speed) {+ ASSERT_TRUE(src.Init());
+ src.SetPadding(10);
+
+ SetCols(src.TopLeftPixel(), rows, cols, src.stride());
+
+ runNTimes(100000);
+ printMedian("16x16");+}
+
class VpxMbPostProcDownTest
: public AbstractBench,
public ::testing::TestWithParam<VpxMbPostProcDownFunc> {@@ -491,6 +500,10 @@
#endif // HAVE_MSA
#if HAVE_VSX
+
+INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcAcrossIpTest,
+ ::testing::Values(vpx_mbpost_proc_across_ip_vsx));
+
INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcDownTest,
::testing::Values(vpx_mbpost_proc_down_vsx));
#endif // HAVE_VSX
--- a/vpx_dsp/ppc/deblock_vsx.c
+++ b/vpx_dsp/ppc/deblock_vsx.c
@@ -19,10 +19,171 @@
0x0C, 0x0E, 0x18, 0x19, 0x1A, 0x1B,
0x1C, 0x1D, 0x1E, 0x1F };
-static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,- 0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
- 0x0C, 0x0D, 0x1C, 0x1D };
+// C: s[c + 7]
+static INLINE int16x8_t next7l_s16(uint8x16_t c) {+ static const uint8x16_t next7_perm = {+ 0x07, 0x10, 0x08, 0x11, 0x09, 0x12, 0x0A, 0x13,
+ 0x0B, 0x14, 0x0C, 0x15, 0x0D, 0x16, 0x0E, 0x17,
+ };
+ return (int16x8_t)vec_perm(c, vec_zeros_u8, next7_perm);
+}
+// Slide across window and add.
+static INLINE int16x8_t slide_sum_s16(int16x8_t x) {+ // x = A B C D E F G H
+ //
+ // 0 A B C D E F G
+ const int16x8_t sum1 = vec_add(x, vec_slo(x, vec_splats((int8_t)(2 << 3))));
+ // 0 0 A B C D E F
+ const int16x8_t sum2 = vec_add(vec_slo(x, vec_splats((int8_t)(4 << 3))),
+ // 0 0 0 A B C D E
+ vec_slo(x, vec_splats((int8_t)(6 << 3))));
+ // 0 0 0 0 A B C D
+ const int16x8_t sum3 = vec_add(vec_slo(x, vec_splats((int8_t)(8 << 3))),
+ // 0 0 0 0 0 A B C
+ vec_slo(x, vec_splats((int8_t)(10 << 3))));
+ // 0 0 0 0 0 0 A B
+ const int16x8_t sum4 = vec_add(vec_slo(x, vec_splats((int8_t)(12 << 3))),
+ // 0 0 0 0 0 0 0 A
+ vec_slo(x, vec_splats((int8_t)(14 << 3))));
+ return vec_add(vec_add(sum1, sum2), vec_add(sum3, sum4));
+}
+
+// Slide across window and add.
+static INLINE int32x4_t slide_sumsq_s32(int32x4_t xsq_even, int32x4_t xsq_odd) {+ // 0 A C E
+ // + 0 B D F
+ int32x4_t sumsq_1 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(4 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(4 << 3))));
+ // 0 0 A C
+ // + 0 0 B D
+ int32x4_t sumsq_2 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(8 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(8 << 3))));
+ // 0 0 0 A
+ // + 0 0 0 B
+ int32x4_t sumsq_3 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(12 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(12 << 3))));
+ sumsq_1 = vec_add(sumsq_1, xsq_even);
+ sumsq_2 = vec_add(sumsq_2, sumsq_3);
+ return vec_add(sumsq_1, sumsq_2);
+}
+
+// C: (b + sum + val) >> 4
+static INLINE int16x8_t filter_s16(int16x8_t b, int16x8_t sum, int16x8_t val) {+ return vec_sra(vec_add(vec_add(b, sum), val), vec_splats((uint16_t)4));
+}
+
+// C: sumsq * 15 - sum * sum
+static INLINE bool16x8_t mask_s16(int32x4_t sumsq_even, int32x4_t sumsq_odd,
+ int16x8_t sum, int32x4_t lim) {+ static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,+ 0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
+ 0x0C, 0x0D, 0x1C, 0x1D };
+ const int32x4_t sumsq_odd_scaled =
+ vec_mul(sumsq_odd, vec_splats((int32_t)15));
+ const int32x4_t sumsq_even_scaled =
+ vec_mul(sumsq_even, vec_splats((int32_t)15));
+ const int32x4_t thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
+ const int32x4_t thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
+
+ const bool32x4_t mask_odd = vec_cmplt(thres_odd, lim);
+ const bool32x4_t mask_even = vec_cmplt(thres_even, lim);
+ return vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
+}
+
+void vpx_mbpost_proc_across_ip_vsx(unsigned char *src, int pitch, int rows,
+ int cols, int flimit) {+ int row, col;
+ const int32x4_t lim = vec_splats(flimit);
+
+ // 8 columns are processed at a time.
+ assert(cols % 8 == 0);
+
+ for (row = 0; row < rows; row++) {+ // The sum is signed and requires at most 13 bits.
+ // (8 bits + sign) * 15 (4 bits)
+ int16x8_t sum;
+ // The sum of squares requires at most 20 bits.
+ // (16 bits + sign) * 15 (4 bits)
+ int32x4_t sumsq_even, sumsq_odd;
+
+ // Fill left context with first col.
+ int16x8_t left_ctx = vec_splats((int16_t)src[0]);
+ int16_t s = src[0] * 9;
+ int32_t ssq = src[0] * src[0] * 9 + 16;
+
+ // Fill the next 6 columns of the sliding window with cols 2 to 7.
+ for (col = 1; col <= 6; ++col) {+ s += src[col];
+ ssq += src[col] * src[col];
+ }
+ // Set this sum to every element in the window.
+ sum = vec_splats(s);
+ sumsq_even = vec_splats(ssq);
+ sumsq_odd = vec_splats(ssq);
+
+ for (col = 0; col < cols; col += 8) {+ bool16x8_t mask;
+ int16x8_t filtered, masked;
+ uint8x16_t out;
+
+ const uint8x16_t val = vec_vsx_ld(0, src + col);
+ const int16x8_t val_high = unpack_to_s16_h(val);
+
+ // C: s[c + 7]
+ const int16x8_t right_ctx = (col + 8 == cols)
+ ? vec_splats((int16_t)src[col + 7])
+ : next7l_s16(val);
+
+ // C: x = s[c + 7] - s[c - 8];
+ const int16x8_t x = vec_sub(right_ctx, left_ctx);
+ const int32x4_t xsq_even =
+ vec_sub(vec_mule(right_ctx, right_ctx), vec_mule(left_ctx, left_ctx));
+ const int32x4_t xsq_odd =
+ vec_sub(vec_mulo(right_ctx, right_ctx), vec_mulo(left_ctx, left_ctx));
+
+ const int32x4_t sumsq_tmp = slide_sumsq_s32(xsq_even, xsq_odd);
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_even = vec_add(sumsq_even, sumsq_tmp);
+ // B D F G
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_odd = vec_add(sumsq_odd, vec_add(sumsq_tmp, xsq_odd));
+
+ sum = vec_add(sum, slide_sum_s16(x));
+
+ // C: (8 + sum + s[c]) >> 4
+ filtered = filter_s16(vec_splats((int16_t)8), sum, val_high);
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
+ masked = vec_sel(val_high, filtered, mask);
+
+ out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, src + col), load_merge);
+ vec_vsx_st(out, 0, src + col);
+
+ // Update window sum and square sum
+ sum = vec_splat(sum, 7);
+ sumsq_even = vec_splat(sumsq_odd, 3);
+ sumsq_odd = vec_splat(sumsq_odd, 3);
+
+ // C: s[c - 8] (for next iteration)
+ left_ctx = val_high;
+ }
+ src += pitch;
+ }
+}
+
void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols,
int flimit) {int col, row, i;
@@ -66,9 +227,6 @@
for (row = 0; row < rows; row++) {int32x4_t d15_even, d15_odd, d0_even, d0_odd;
- int32x4_t sumsq_odd_scaled, sumsq_even_scaled;
- int32x4_t thres_odd, thres_even;
- bool32x4_t mask_odd, mask_even;
bool16x8_t mask;
int16x8_t filtered, masked;
uint8x16_t out;
@@ -96,19 +254,11 @@
sumsq_odd = vec_add(sumsq_odd, vec_sub(d15_odd, d0_odd));
sumsq_even = vec_add(sumsq_even, vec_sub(d15_even, d0_even));
- // C: sumsq * 15 - sum * sum
- sumsq_odd_scaled = vec_mul(sumsq_odd, vec_splats((int32_t)15));
- sumsq_even_scaled = vec_mul(sumsq_even, vec_splats((int32_t)15));
- thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
- thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
-
// C: (vpx_rv[(r & 127) + (c & 7)] + sum + s[0]) >> 4
- filtered = vec_add(vec_add(rv, sum), window[8]);
- filtered = vec_sra(filtered, vec_splats((uint16_t)4));
+ filtered = filter_s16(rv, sum, window[8]);
- mask_odd = vec_cmplt(thres_odd, lim);
- mask_even = vec_cmplt(thres_even, lim);
- mask = vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
masked = vec_sel(window[8], filtered, mask);
// TODO(ltrudeau) If cols % 16 == 0, we could just process 16 per
--- a/vpx_dsp/ppc/types_vsx.h
+++ b/vpx_dsp/ppc/types_vsx.h
@@ -67,10 +67,13 @@
#endif
#endif
+static const uint8x16_t vec_zeros_u8 = { 0, 0, 0, 0, 0, 0, 0, 0,+ 0, 0, 0, 0, 0, 0, 0, 0 };
static const int16x8_t vec_zeros_s16 = { 0, 0, 0, 0, 0, 0, 0, 0 }; static const int16x8_t vec_ones_s16 = { 1, 1, 1, 1, 1, 1, 1, 1 }; static const uint16x8_t vec_ones_u16 = { 1, 1, 1, 1, 1, 1, 1, 1 }; static const uint32x4_t vec_ones_u32 = { 1, 1, 1, 1 };+static const int32x4_t vec_zeros_s32 = { 0, 0, 0, 0 }; static const uint16x8_t vec_shift_sign_s16 = { 15, 15, 15, 15, 15, 15, 15, 15 }; static const uint32x4_t vec_shift_sign_s32 = { 31, 31, 31, 31 }; static const uint8x16_t vec_perm64 = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,--- a/vpx_dsp/vpx_dsp_rtcd_defs.pl
+++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -1601,7 +1601,7 @@
specialize qw/vpx_mbpost_proc_down sse2 neon msa vsx/;
add_proto qw/void vpx_mbpost_proc_across_ip/, "unsigned char *dst, int pitch, int rows, int cols,int flimit";
- specialize qw/vpx_mbpost_proc_across_ip sse2 neon msa/;
+ specialize qw/vpx_mbpost_proc_across_ip sse2 neon msa vsx/;
add_proto qw/void vpx_post_proc_down_and_across_mb_row/, "unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size";
specialize qw/vpx_post_proc_down_and_across_mb_row sse2 neon msa/;