ref: 5232e35bc2609c2792114cd6fd9fd37852f4df19
parent: 7f4acf8700d5b61f544e8e6fa91f58688947eacc
author: Linfeng Zhang <linfengz@google.com>
date: Fri Jul 21 10:56:42 EDT 2017
Update vpx_idct16x16_10_add_sse2() Change-Id: Ia0e20f5fa47382af5785221eebb05212b40bd35c
--- a/vpx_dsp/x86/inv_txfm_sse2.c
+++ b/vpx_dsp/x86/inv_txfm_sse2.c
@@ -164,6 +164,19 @@
write_buffer_8x8(in, dest, stride);
}
+void vpx_idct8x8_12_add_sse2(const tran_low_t *input, uint8_t *dest,
+ int stride) {+ __m128i io[8];
+
+ io[0] = load_input_data4(input + 0 * 8);
+ io[1] = load_input_data4(input + 1 * 8);
+ io[2] = load_input_data4(input + 2 * 8);
+ io[3] = load_input_data4(input + 3 * 8);
+
+ idct8x8_12_add_kernel_sse2(io);
+ write_buffer_8x8(io, dest, stride);
+}
+
static INLINE void recon_and_store_8_dual(uint8_t *const dest,
const __m128i in_x,
const int stride) {@@ -409,72 +422,6 @@
in[7] = _mm_sub_epi16(k__const_0, s1);
}
-void vpx_idct8x8_12_add_sse2(const tran_low_t *input, uint8_t *dest,
- int stride) {- __m128i io[8];
-
- io[0] = load_input_data4(input + 0 * 8);
- io[1] = load_input_data4(input + 1 * 8);
- io[2] = load_input_data4(input + 2 * 8);
- io[3] = load_input_data4(input + 3 * 8);
-
- idct8x8_12_add_kernel_sse2(io);
- write_buffer_8x8(io, dest, stride);
-}
-
-#define IDCT16_10 \
- /* Stage2 */ \
- multiplication_and_add(&in[1], &zero, &zero, &in[3], &stg2_0, &stg2_1, \
- &stg2_6, &stg2_7, &stp1_8_0, &stp1_15, &stp1_11, \
- &stp1_12_0); \
- \
- /* Stage3 */ \
- multiplication_and_add_2(&in[2], &zero, &stg3_0, &stg3_1, &stp2_4, &stp2_7); \
- \
- stp1_9 = stp1_8_0; \
- stp1_10 = stp1_11; \
- stp1_13 = stp1_12_0; \
- stp1_14 = stp1_15; \
- \
- /* Stage4 */ \
- multiplication_and_add_2(&in[0], &zero, &stg4_0, &stg4_1, &stp1_0, &stp1_1); \
- stp2_5 = stp2_4; \
- stp2_6 = stp2_7; \
- \
- multiplication_and_add(&stp1_9, &stp1_14, &stp1_10, &stp1_13, &stg4_4, \
- &stg4_5, &stg4_6, &stg4_7, &stp2_9, &stp2_14, \
- &stp2_10, &stp2_13); \
- \
- /* Stage5 */ \
- stp1_2 = stp1_1; \
- stp1_3 = stp1_0; \
- multiplication_and_add_2(&stp2_6, &stp2_5, &stg4_1, &stg4_0, &stp1_5, \
- &stp1_6); \
- \
- stp1_8 = _mm_add_epi16(stp1_8_0, stp1_11); \
- stp1_9 = _mm_add_epi16(stp2_9, stp2_10); \
- stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); \
- stp1_11 = _mm_sub_epi16(stp1_8_0, stp1_11); \
- \
- stp1_12 = _mm_sub_epi16(stp1_15, stp1_12_0); \
- stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); \
- stp1_14 = _mm_add_epi16(stp2_14, stp2_13); \
- stp1_15 = _mm_add_epi16(stp1_15, stp1_12_0); \
- \
- /* Stage6 */ \
- stp2_0 = _mm_add_epi16(stp1_0, stp2_7); \
- stp2_1 = _mm_add_epi16(stp1_1, stp1_6); \
- stp2_2 = _mm_add_epi16(stp1_2, stp1_5); \
- stp2_3 = _mm_add_epi16(stp1_3, stp2_4); \
- stp2_4 = _mm_sub_epi16(stp1_3, stp2_4); \
- stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); \
- stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); \
- stp2_7 = _mm_sub_epi16(stp1_0, stp2_7); \
- \
- multiplication_and_add(&stp1_10, &stp1_13, &stp1_11, &stp1_12, &stg6_0, \
- &stg4_0, &stg6_0, &stg4_0, &stp2_10, &stp2_13, \
- &stp2_11, &stp2_12);
-
static INLINE void idct16_8col(__m128i *const io /*io[16]*/) {const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
@@ -682,6 +629,225 @@
}
}
+static INLINE void idct16x16_10_pass1(const __m128i *const input /*input[4]*/,
+ __m128i *const output /*output[16]*/) {+ const __m128i zero = _mm_setzero_si128();
+ const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
+ const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+ __m128i step1[16], step2[16];
+
+ transpose_16bit_4x4(input, output);
+
+ // stage 2
+ {+ const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
+ const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
+ const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
+ const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
+ const __m128i lo_1_15 = _mm_unpackhi_epi16(output[0], zero);
+ const __m128i lo_13_3 = _mm_unpackhi_epi16(zero, output[1]);
+ step2[8] = idct_calc_wraplow_sse2(k__cospi_p30_m02, k__cospi_p02_p30,
+ lo_1_15); // step2 8&15
+ step2[11] = idct_calc_wraplow_sse2(k__cospi_p06_m26, k__cospi_p26_p06,
+ lo_13_3); // step2 11&12
+ }
+
+ // stage 3
+ {+ const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
+ const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
+ const __m128i lo_2_14 = _mm_unpacklo_epi16(output[1], zero);
+ step1[4] = idct_calc_wraplow_sse2(k__cospi_p28_m04, k__cospi_p04_p28,
+ lo_2_14); // step1 4&7
+ step1[13] = _mm_unpackhi_epi64(step2[11], zero);
+ step1[14] = _mm_unpackhi_epi64(step2[8], zero);
+ }
+
+ // stage 4
+ {+ const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+ const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
+ const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+ const __m128i lo_0_8 = _mm_unpacklo_epi16(output[0], zero);
+ const __m128i lo_9_14 = _mm_unpacklo_epi16(step2[8], step1[14]);
+ const __m128i lo_10_13 = _mm_unpacklo_epi16(step2[11], step1[13]);
+ const __m128i t = idct_madd_round_shift_sse2(lo_0_8, k__cospi_p16_p16);
+ step1[0] = _mm_packs_epi32(t, t); // step2 0&1
+ step2[9] = idct_calc_wraplow_sse2(k__cospi_m08_p24, k__cospi_p24_p08,
+ lo_9_14); // step2 9&14
+ step2[10] = idct_calc_wraplow_sse2(k__cospi_m24_m08, k__cospi_m08_p24,
+ lo_10_13); // step2 10&13
+ step2[6] = _mm_unpackhi_epi64(step1[4], zero);
+ }
+
+ // stage 5
+ {+ const __m128i lo_5_6 = _mm_unpacklo_epi16(step1[4], step2[6]);
+ step1[6] = idct_calc_wraplow_sse2(k__cospi_p16_p16, k__cospi_m16_p16,
+ lo_5_6); // step1 6&5
+ step1[8] = _mm_add_epi16(step2[8], step2[11]);
+ step1[9] = _mm_add_epi16(step2[9], step2[10]);
+ step1[10] = _mm_sub_epi16(step2[9], step2[10]);
+ step1[11] = _mm_sub_epi16(step2[8], step2[11]);
+ step1[12] = _mm_unpackhi_epi64(step1[11], zero);
+ step1[13] = _mm_unpackhi_epi64(step1[10], zero);
+ step1[14] = _mm_unpackhi_epi64(step1[9], zero);
+ step1[15] = _mm_unpackhi_epi64(step1[8], zero);
+ }
+
+ // stage 6
+ {+ const __m128i lo_10_13 = _mm_unpacklo_epi16(step1[10], step1[13]);
+ const __m128i lo_11_12 = _mm_unpacklo_epi16(step1[11], step1[12]);
+ step2[10] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
+ lo_10_13); // step2 10&13
+ step2[11] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
+ lo_11_12); // step2 11&12
+ step2[13] = _mm_unpackhi_epi64(step2[10], zero);
+ step2[12] = _mm_unpackhi_epi64(step2[11], zero);
+ step2[3] = _mm_add_epi16(step1[0], step1[4]);
+ step2[1] = _mm_add_epi16(step1[0], step1[6]);
+ step2[6] = _mm_sub_epi16(step1[0], step1[6]);
+ step2[4] = _mm_sub_epi16(step1[0], step1[4]);
+ step2[0] = _mm_unpackhi_epi64(step2[3], zero);
+ step2[2] = _mm_unpackhi_epi64(step2[1], zero);
+ step2[5] = _mm_unpackhi_epi64(step2[6], zero);
+ step2[7] = _mm_unpackhi_epi64(step2[4], zero);
+ }
+
+ // stage 7. Left 8x16 only.
+ output[0] = _mm_add_epi16(step2[0], step1[15]);
+ output[1] = _mm_add_epi16(step2[1], step1[14]);
+ output[2] = _mm_add_epi16(step2[2], step2[13]);
+ output[3] = _mm_add_epi16(step2[3], step2[12]);
+ output[4] = _mm_add_epi16(step2[4], step2[11]);
+ output[5] = _mm_add_epi16(step2[5], step2[10]);
+ output[6] = _mm_add_epi16(step2[6], step1[9]);
+ output[7] = _mm_add_epi16(step2[7], step1[8]);
+ output[8] = _mm_sub_epi16(step2[7], step1[8]);
+ output[9] = _mm_sub_epi16(step2[6], step1[9]);
+ output[10] = _mm_sub_epi16(step2[5], step2[10]);
+ output[11] = _mm_sub_epi16(step2[4], step2[11]);
+ output[12] = _mm_sub_epi16(step2[3], step2[12]);
+ output[13] = _mm_sub_epi16(step2[2], step2[13]);
+ output[14] = _mm_sub_epi16(step2[1], step1[14]);
+ output[15] = _mm_sub_epi16(step2[0], step1[15]);
+}
+
+static INLINE void idct16x16_10_pass2(__m128i *const l /*l[8]*/,
+ __m128i *const io /*io[16]*/) {+ const __m128i zero = _mm_setzero_si128();
+ const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
+ const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+ __m128i step1[16], step2[16];
+
+ transpose_16bit_4x8(l, io);
+
+ // stage 2
+ {+ const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
+ const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
+ const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
+ const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
+ multiplication_and_add(&io[1], &zero, &zero, &io[3], &k__cospi_p30_m02,
+ &k__cospi_p02_p30, &k__cospi_p06_m26,
+ &k__cospi_p26_p06, &step2[8], &step2[15], &step2[11],
+ &step2[12]);
+ }
+
+ // stage 3
+ {+ const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
+ const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
+ multiplication_and_add_2(&io[2], &zero, &k__cospi_p28_m04,
+ &k__cospi_p04_p28, &step1[4], &step1[7]);
+ }
+
+ // stage 4
+ {+ const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+ const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
+ const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+ multiplication_and_add_2(&zero, &io[0], &k__cospi_p16_p16,
+ &k__cospi_m16_p16, &step1[0], &step1[1]);
+ multiplication_and_add(&step2[8], &step2[15], &step2[11], &step2[12],
+ &k__cospi_m08_p24, &k__cospi_p24_p08,
+ &k__cospi_m24_m08, &k__cospi_m08_p24, &step2[9],
+ &step2[14], &step2[10], &step2[13]);
+ }
+
+ // stage 5
+ multiplication_and_add_2(&step1[4], &step1[7], &k__cospi_m16_p16,
+ &k__cospi_p16_p16, &step1[5], &step1[6]);
+ step1[8] = _mm_add_epi16(step2[8], step2[11]);
+ step1[9] = _mm_add_epi16(step2[9], step2[10]);
+ step1[10] = _mm_sub_epi16(step2[9], step2[10]);
+ step1[11] = _mm_sub_epi16(step2[8], step2[11]);
+ step1[12] = _mm_sub_epi16(step2[15], step2[12]);
+ step1[13] = _mm_sub_epi16(step2[14], step2[13]);
+ step1[14] = _mm_add_epi16(step2[14], step2[13]);
+ step1[15] = _mm_add_epi16(step2[15], step2[12]);
+
+ // stage 6
+ step2[0] = _mm_add_epi16(step1[0], step1[7]);
+ step2[1] = _mm_add_epi16(step1[1], step1[6]);
+ step2[2] = _mm_add_epi16(step1[1], step1[5]);
+ step2[3] = _mm_add_epi16(step1[0], step1[4]);
+ step2[4] = _mm_sub_epi16(step1[0], step1[4]);
+ step2[5] = _mm_sub_epi16(step1[1], step1[5]);
+ step2[6] = _mm_sub_epi16(step1[1], step1[6]);
+ step2[7] = _mm_sub_epi16(step1[0], step1[7]);
+ multiplication_and_add(&step1[10], &step1[13], &step1[11], &step1[12],
+ &k__cospi_m16_p16, &k__cospi_p16_p16,
+ &k__cospi_m16_p16, &k__cospi_p16_p16, &step2[10],
+ &step2[13], &step2[11], &step2[12]);
+
+ // stage 7
+ io[0] = _mm_add_epi16(step2[0], step1[15]);
+ io[1] = _mm_add_epi16(step2[1], step1[14]);
+ io[2] = _mm_add_epi16(step2[2], step2[13]);
+ io[3] = _mm_add_epi16(step2[3], step2[12]);
+ io[4] = _mm_add_epi16(step2[4], step2[11]);
+ io[5] = _mm_add_epi16(step2[5], step2[10]);
+ io[6] = _mm_add_epi16(step2[6], step1[9]);
+ io[7] = _mm_add_epi16(step2[7], step1[8]);
+ io[8] = _mm_sub_epi16(step2[7], step1[8]);
+ io[9] = _mm_sub_epi16(step2[6], step1[9]);
+ io[10] = _mm_sub_epi16(step2[5], step2[10]);
+ io[11] = _mm_sub_epi16(step2[4], step2[11]);
+ io[12] = _mm_sub_epi16(step2[3], step2[12]);
+ io[13] = _mm_sub_epi16(step2[2], step2[13]);
+ io[14] = _mm_sub_epi16(step2[1], step1[14]);
+ io[15] = _mm_sub_epi16(step2[0], step1[15]);
+}
+
+void vpx_idct16x16_10_add_sse2(const tran_low_t *input, uint8_t *dest,
+ int stride) {+ __m128i in[16], l[16];
+ int i;
+
+ // First 1-D inverse DCT
+ // Load input data.
+ in[0] = load_input_data4(input + 0 * 16);
+ in[1] = load_input_data4(input + 1 * 16);
+ in[2] = load_input_data4(input + 2 * 16);
+ in[3] = load_input_data4(input + 3 * 16);
+
+ idct16x16_10_pass1(in, l);
+
+ // Second 1-D inverse transform, performed per 8x16 block
+ for (i = 0; i < 2; i++) {+ int j;
+ idct16x16_10_pass2(l + 8 * i, in);
+
+ for (j = 0; j < 16; ++j) {+ write_buffer_8x1(dest + j * stride, in[j]);
+ }
+
+ dest += 8;
+ }
+}
+
static INLINE void recon_and_store_16(uint8_t *const dest, const __m128i in_x) {const __m128i zero = _mm_setzero_si128();
__m128i d0, d1;
@@ -1142,178 +1308,6 @@
transpose_16bit_16x16(in0, in1);
iadst16_8col(in0);
iadst16_8col(in1);
-}
-
-void vpx_idct16x16_10_add_sse2(const tran_low_t *input, uint8_t *dest,
- int stride) {- const __m128i zero = _mm_setzero_si128();
-
- const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64);
- const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64);
- const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64);
- const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64);
-
- const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64);
- const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64);
-
- const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
- const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
- const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
- const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
- const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
- const __m128i stg4_7 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-
- const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
- __m128i in[16], l[16];
- __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_8,
- stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, stp1_8_0,
- stp1_12_0;
- __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
- stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14;
- __m128i tmp0, tmp1, tmp2, tmp3;
- int i;
- // First 1-D inverse DCT
- // Load input data.
- in[0] = load_input_data4(input + 0 * 16);
- in[1] = load_input_data4(input + 1 * 16);
- in[2] = load_input_data4(input + 2 * 16);
- in[3] = load_input_data4(input + 3 * 16);
-
- transpose_16bit_4x4(in, in);
-
- // Stage2
- {- const __m128i lo_1_15 = _mm_unpackhi_epi16(in[0], zero);
- const __m128i lo_13_3 = _mm_unpackhi_epi16(zero, in[1]);
-
- stp2_8 = idct_calc_wraplow_sse2(stg2_0, stg2_1, lo_1_15);
- stp2_11 = idct_calc_wraplow_sse2(stg2_6, stg2_7, lo_13_3);
- }
-
- // Stage3
- {- const __m128i lo_2_14 = _mm_unpacklo_epi16(in[1], zero);
-
- stp1_4 = idct_calc_wraplow_sse2(stg3_0, stg3_1, lo_2_14);
- stp1_13 = _mm_unpackhi_epi64(stp2_11, zero);
- stp1_14 = _mm_unpackhi_epi64(stp2_8, zero);
- }
-
- // Stage4
- {- const __m128i lo_0_8 = _mm_unpacklo_epi16(in[0], zero);
- const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp1_14);
- const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp1_13);
-
- tmp0 = idct_madd_round_shift_sse2(lo_0_8, stg4_0);
- tmp1 = idct_madd_round_shift_sse2(lo_0_8, stg4_1);
- stp1_0 = _mm_packs_epi32(tmp0, tmp0);
- stp1_1 = _mm_packs_epi32(tmp1, tmp1);
- stp2_9 = idct_calc_wraplow_sse2(stg4_4, stg4_5, lo_9_14);
- stp2_10 = idct_calc_wraplow_sse2(stg4_6, stg4_7, lo_10_13);
-
- stp2_6 = _mm_unpackhi_epi64(stp1_4, zero);
- }
-
- // Stage5 and Stage6
- {- tmp0 = _mm_add_epi16(stp2_8, stp2_11);
- tmp1 = _mm_sub_epi16(stp2_8, stp2_11);
- tmp2 = _mm_add_epi16(stp2_9, stp2_10);
- tmp3 = _mm_sub_epi16(stp2_9, stp2_10);
-
- stp1_9 = _mm_unpacklo_epi64(tmp2, zero);
- stp1_10 = _mm_unpacklo_epi64(tmp3, zero);
- stp1_8 = _mm_unpacklo_epi64(tmp0, zero);
- stp1_11 = _mm_unpacklo_epi64(tmp1, zero);
-
- stp1_13 = _mm_unpackhi_epi64(tmp3, zero);
- stp1_14 = _mm_unpackhi_epi64(tmp2, zero);
- stp1_12 = _mm_unpackhi_epi64(tmp1, zero);
- stp1_15 = _mm_unpackhi_epi64(tmp0, zero);
- }
-
- // Stage6
- {- const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp1_4);
- const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13);
- const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12);
-
- stp1_6 = idct_calc_wraplow_sse2(stg4_0, stg4_1, lo_6_5);
- tmp0 = idct_madd_round_shift_sse2(lo_10_13, stg6_0);
- tmp1 = idct_madd_round_shift_sse2(lo_10_13, stg4_0);
- tmp2 = idct_madd_round_shift_sse2(lo_11_12, stg6_0);
- tmp3 = idct_madd_round_shift_sse2(lo_11_12, stg4_0);
-
- stp2_10 = _mm_packs_epi32(tmp0, zero);
- stp2_13 = _mm_packs_epi32(tmp1, zero);
- stp2_11 = _mm_packs_epi32(tmp2, zero);
- stp2_12 = _mm_packs_epi32(tmp3, zero);
-
- tmp0 = _mm_add_epi16(stp1_0, stp1_4);
- tmp1 = _mm_sub_epi16(stp1_0, stp1_4);
- tmp2 = _mm_add_epi16(stp1_1, stp1_6);
- tmp3 = _mm_sub_epi16(stp1_1, stp1_6);
-
- stp2_0 = _mm_unpackhi_epi64(tmp0, zero);
- stp2_1 = _mm_unpacklo_epi64(tmp2, zero);
- stp2_2 = _mm_unpackhi_epi64(tmp2, zero);
- stp2_3 = _mm_unpacklo_epi64(tmp0, zero);
- stp2_4 = _mm_unpacklo_epi64(tmp1, zero);
- stp2_5 = _mm_unpackhi_epi64(tmp3, zero);
- stp2_6 = _mm_unpacklo_epi64(tmp3, zero);
- stp2_7 = _mm_unpackhi_epi64(tmp1, zero);
- }
-
- // Stage7. Left 8x16 only.
- l[0] = _mm_add_epi16(stp2_0, stp1_15);
- l[1] = _mm_add_epi16(stp2_1, stp1_14);
- l[2] = _mm_add_epi16(stp2_2, stp2_13);
- l[3] = _mm_add_epi16(stp2_3, stp2_12);
- l[4] = _mm_add_epi16(stp2_4, stp2_11);
- l[5] = _mm_add_epi16(stp2_5, stp2_10);
- l[6] = _mm_add_epi16(stp2_6, stp1_9);
- l[7] = _mm_add_epi16(stp2_7, stp1_8);
- l[8] = _mm_sub_epi16(stp2_7, stp1_8);
- l[9] = _mm_sub_epi16(stp2_6, stp1_9);
- l[10] = _mm_sub_epi16(stp2_5, stp2_10);
- l[11] = _mm_sub_epi16(stp2_4, stp2_11);
- l[12] = _mm_sub_epi16(stp2_3, stp2_12);
- l[13] = _mm_sub_epi16(stp2_2, stp2_13);
- l[14] = _mm_sub_epi16(stp2_1, stp1_14);
- l[15] = _mm_sub_epi16(stp2_0, stp1_15);
-
- // Second 1-D inverse transform, performed per 8x16 block
- for (i = 0; i < 2; i++) {- int j;
- transpose_16bit_4x8(l + 8 * i, in);
-
- IDCT16_10
-
- // Stage7
- in[0] = _mm_add_epi16(stp2_0, stp1_15);
- in[1] = _mm_add_epi16(stp2_1, stp1_14);
- in[2] = _mm_add_epi16(stp2_2, stp2_13);
- in[3] = _mm_add_epi16(stp2_3, stp2_12);
- in[4] = _mm_add_epi16(stp2_4, stp2_11);
- in[5] = _mm_add_epi16(stp2_5, stp2_10);
- in[6] = _mm_add_epi16(stp2_6, stp1_9);
- in[7] = _mm_add_epi16(stp2_7, stp1_8);
- in[8] = _mm_sub_epi16(stp2_7, stp1_8);
- in[9] = _mm_sub_epi16(stp2_6, stp1_9);
- in[10] = _mm_sub_epi16(stp2_5, stp2_10);
- in[11] = _mm_sub_epi16(stp2_4, stp2_11);
- in[12] = _mm_sub_epi16(stp2_3, stp2_12);
- in[13] = _mm_sub_epi16(stp2_2, stp2_13);
- in[14] = _mm_sub_epi16(stp2_1, stp1_14);
- in[15] = _mm_sub_epi16(stp2_0, stp1_15);
-
- for (j = 0; j < 16; ++j) {- write_buffer_8x1(dest + j * stride, in[j]);
- }
-
- dest += 8;
- }
}
#define IDCT32_34 \