shithub: libvpx

Download patch

ref: 6624f20785bb782ad02edc27a60403946c11ff29
parent: 6843e7c7f35ca2c63fe5a9a4f4663c463fd480a7
parent: 15a47db73081109aa4167b339fff1f40bcf01e67
author: Linfeng Zhang <linfengz@google.com>
date: Thu Aug 3 16:51:02 EDT 2017

Merge changes I76727df0,I66297d78,I1d000c6b

* changes:
  Extract inlined 16x16 idct sse2 code into header file
  Add transpose_32bit_8x4() sse2 optimization
  Update x86 idct optimization

--- a/vpx_dsp/x86/inv_txfm_sse2.c
+++ b/vpx_dsp/x86/inv_txfm_sse2.c
@@ -422,125 +422,6 @@
   in[7] = _mm_sub_epi16(k__const_0, s1);
 }
 
-static INLINE void idct16_8col(__m128i *const io /*io[16]*/) {
-  const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-  __m128i step1[16], step2[16];
-
-  // stage 2
-  {
-    const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
-    const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
-    const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64);
-    const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64);
-    multiplication_and_add(&io[1], &io[15], &io[9], &io[7], &k__cospi_p30_m02,
-                           &k__cospi_p02_p30, &k__cospi_p14_m18,
-                           &k__cospi_p18_p14, &step2[8], &step2[15], &step2[9],
-                           &step2[14]);
-  }
-  {
-    const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64);
-    const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64);
-    const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
-    const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
-    multiplication_and_add(&io[5], &io[11], &io[13], &io[3], &k__cospi_p22_m10,
-                           &k__cospi_p10_p22, &k__cospi_p06_m26,
-                           &k__cospi_p26_p06, &step2[10], &step2[13],
-                           &step2[11], &step2[12]);
-  }
-
-  // stage 3
-  {
-    const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-    const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
-    const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64);
-    const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64);
-    multiplication_and_add(&io[2], &io[14], &io[10], &io[6], &k__cospi_p28_m04,
-                           &k__cospi_p04_p28, &k__cospi_p12_m20,
-                           &k__cospi_p20_p12, &step1[4], &step1[7], &step1[5],
-                           &step1[6]);
-  }
-  step1[8] = _mm_add_epi16(step2[8], step2[9]);
-  step1[9] = _mm_sub_epi16(step2[8], step2[9]);
-  step1[10] = _mm_sub_epi16(step2[11], step2[10]);
-  step1[11] = _mm_add_epi16(step2[10], step2[11]);
-  step1[12] = _mm_add_epi16(step2[12], step2[13]);
-  step1[13] = _mm_sub_epi16(step2[12], step2[13]);
-  step1[14] = _mm_sub_epi16(step2[15], step2[14]);
-  step1[15] = _mm_add_epi16(step2[14], step2[15]);
-
-  // stage 4
-  {
-    const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    multiplication_and_add(&io[8], &io[0], &io[12], &io[4], &k__cospi_p16_p16,
-                           &k__cospi_m16_p16, &k__cospi_m08_p24,
-                           &k__cospi_p24_p08, &step2[0], &step2[1], &step2[2],
-                           &step2[3]);
-    step2[5] = _mm_sub_epi16(step1[4], step1[5]);
-    step1[4] = _mm_add_epi16(step1[4], step1[5]);
-    step2[6] = _mm_sub_epi16(step1[7], step1[6]);
-    step1[7] = _mm_add_epi16(step1[6], step1[7]);
-    step2[8] = step1[8];
-    multiplication_and_add(&step1[9], &step1[14], &step1[10], &step1[13],
-                           &k__cospi_m08_p24, &k__cospi_p24_p08,
-                           &k__cospi_m24_m08, &k__cospi_m08_p24, &step2[9],
-                           &step2[14], &step2[10], &step2[13]);
-  }
-  step2[11] = step1[11];
-  step2[12] = step1[12];
-  step2[15] = step1[15];
-
-  // stage 5
-  step1[0] = _mm_add_epi16(step2[0], step2[3]);
-  step1[1] = _mm_add_epi16(step2[1], step2[2]);
-  step1[2] = _mm_sub_epi16(step2[1], step2[2]);
-  step1[3] = _mm_sub_epi16(step2[0], step2[3]);
-  multiplication_and_add_2(&step2[5], &step2[6], &k__cospi_m16_p16,
-                           &k__cospi_p16_p16, &step1[5], &step1[6]);
-  step1[8] = _mm_add_epi16(step2[8], step2[11]);
-  step1[9] = _mm_add_epi16(step2[9], step2[10]);
-  step1[10] = _mm_sub_epi16(step2[9], step2[10]);
-  step1[11] = _mm_sub_epi16(step2[8], step2[11]);
-  step1[12] = _mm_sub_epi16(step2[15], step2[12]);
-  step1[13] = _mm_sub_epi16(step2[14], step2[13]);
-  step1[14] = _mm_add_epi16(step2[14], step2[13]);
-  step1[15] = _mm_add_epi16(step2[15], step2[12]);
-
-  // stage 6
-  step2[0] = _mm_add_epi16(step1[0], step1[7]);
-  step2[1] = _mm_add_epi16(step1[1], step1[6]);
-  step2[2] = _mm_add_epi16(step1[2], step1[5]);
-  step2[3] = _mm_add_epi16(step1[3], step1[4]);
-  step2[4] = _mm_sub_epi16(step1[3], step1[4]);
-  step2[5] = _mm_sub_epi16(step1[2], step1[5]);
-  step2[6] = _mm_sub_epi16(step1[1], step1[6]);
-  step2[7] = _mm_sub_epi16(step1[0], step1[7]);
-  multiplication_and_add(&step1[10], &step1[13], &step1[11], &step1[12],
-                         &k__cospi_m16_p16, &k__cospi_p16_p16,
-                         &k__cospi_m16_p16, &k__cospi_p16_p16, &step2[10],
-                         &step2[13], &step2[11], &step2[12]);
-
-  // stage 7
-  io[0] = _mm_add_epi16(step2[0], step1[15]);
-  io[1] = _mm_add_epi16(step2[1], step1[14]);
-  io[2] = _mm_add_epi16(step2[2], step2[13]);
-  io[3] = _mm_add_epi16(step2[3], step2[12]);
-  io[4] = _mm_add_epi16(step2[4], step2[11]);
-  io[5] = _mm_add_epi16(step2[5], step2[10]);
-  io[6] = _mm_add_epi16(step2[6], step1[9]);
-  io[7] = _mm_add_epi16(step2[7], step1[8]);
-  io[8] = _mm_sub_epi16(step2[7], step1[8]);
-  io[9] = _mm_sub_epi16(step2[6], step1[9]);
-  io[10] = _mm_sub_epi16(step2[5], step2[10]);
-  io[11] = _mm_sub_epi16(step2[4], step2[11]);
-  io[12] = _mm_sub_epi16(step2[3], step2[12]);
-  io[13] = _mm_sub_epi16(step2[2], step2[13]);
-  io[14] = _mm_sub_epi16(step2[1], step1[14]);
-  io[15] = _mm_sub_epi16(step2[0], step1[15]);
-}
-
 static INLINE void idct16_load8x8(const tran_low_t *const input,
                                   __m128i *const in) {
   in[0] = load_input_data8(input + 0 * 16);
@@ -629,198 +510,6 @@
   }
 }
 
-static INLINE void idct16x16_10_pass1(const __m128i *const input /*input[4]*/,
-                                      __m128i *const output /*output[16]*/) {
-  const __m128i zero = _mm_setzero_si128();
-  const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-  __m128i step1[16], step2[16];
-
-  transpose_16bit_4x4(input, output);
-
-  // stage 2
-  {
-    const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
-    const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
-    const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
-    const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
-    const __m128i lo_1_15 = _mm_unpackhi_epi16(output[0], zero);
-    const __m128i lo_13_3 = _mm_unpackhi_epi16(zero, output[1]);
-    step2[8] = idct_calc_wraplow_sse2(k__cospi_p30_m02, k__cospi_p02_p30,
-                                      lo_1_15);  // step2 8&15
-    step2[11] = idct_calc_wraplow_sse2(k__cospi_p06_m26, k__cospi_p26_p06,
-                                       lo_13_3);  // step2 11&12
-  }
-
-  // stage 3
-  {
-    const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-    const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
-    const __m128i lo_2_14 = _mm_unpacklo_epi16(output[1], zero);
-    step1[4] = idct_calc_wraplow_sse2(k__cospi_p28_m04, k__cospi_p04_p28,
-                                      lo_2_14);  // step1 4&7
-    step1[13] = _mm_unpackhi_epi64(step2[11], zero);
-    step1[14] = _mm_unpackhi_epi64(step2[8], zero);
-  }
-
-  // stage 4
-  {
-    const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    const __m128i lo_0_8 = _mm_unpacklo_epi16(output[0], zero);
-    const __m128i lo_9_14 = _mm_unpacklo_epi16(step2[8], step1[14]);
-    const __m128i lo_10_13 = _mm_unpacklo_epi16(step2[11], step1[13]);
-    const __m128i t = idct_madd_round_shift_sse2(lo_0_8, k__cospi_p16_p16);
-    step1[0] = _mm_packs_epi32(t, t);  // step2 0&1
-    step2[9] = idct_calc_wraplow_sse2(k__cospi_m08_p24, k__cospi_p24_p08,
-                                      lo_9_14);  // step2 9&14
-    step2[10] = idct_calc_wraplow_sse2(k__cospi_m24_m08, k__cospi_m08_p24,
-                                       lo_10_13);  // step2 10&13
-    step2[6] = _mm_unpackhi_epi64(step1[4], zero);
-  }
-
-  // stage 5
-  {
-    const __m128i lo_5_6 = _mm_unpacklo_epi16(step1[4], step2[6]);
-    step1[6] = idct_calc_wraplow_sse2(k__cospi_p16_p16, k__cospi_m16_p16,
-                                      lo_5_6);  // step1 6&5
-    step1[8] = _mm_add_epi16(step2[8], step2[11]);
-    step1[9] = _mm_add_epi16(step2[9], step2[10]);
-    step1[10] = _mm_sub_epi16(step2[9], step2[10]);
-    step1[11] = _mm_sub_epi16(step2[8], step2[11]);
-    step1[12] = _mm_unpackhi_epi64(step1[11], zero);
-    step1[13] = _mm_unpackhi_epi64(step1[10], zero);
-    step1[14] = _mm_unpackhi_epi64(step1[9], zero);
-    step1[15] = _mm_unpackhi_epi64(step1[8], zero);
-  }
-
-  // stage 6
-  {
-    const __m128i lo_10_13 = _mm_unpacklo_epi16(step1[10], step1[13]);
-    const __m128i lo_11_12 = _mm_unpacklo_epi16(step1[11], step1[12]);
-    step2[10] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
-                                       lo_10_13);  // step2 10&13
-    step2[11] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
-                                       lo_11_12);  // step2 11&12
-    step2[13] = _mm_unpackhi_epi64(step2[10], zero);
-    step2[12] = _mm_unpackhi_epi64(step2[11], zero);
-    step2[3] = _mm_add_epi16(step1[0], step1[4]);
-    step2[1] = _mm_add_epi16(step1[0], step1[6]);
-    step2[6] = _mm_sub_epi16(step1[0], step1[6]);
-    step2[4] = _mm_sub_epi16(step1[0], step1[4]);
-    step2[0] = _mm_unpackhi_epi64(step2[3], zero);
-    step2[2] = _mm_unpackhi_epi64(step2[1], zero);
-    step2[5] = _mm_unpackhi_epi64(step2[6], zero);
-    step2[7] = _mm_unpackhi_epi64(step2[4], zero);
-  }
-
-  // stage 7. Left 8x16 only.
-  output[0] = _mm_add_epi16(step2[0], step1[15]);
-  output[1] = _mm_add_epi16(step2[1], step1[14]);
-  output[2] = _mm_add_epi16(step2[2], step2[13]);
-  output[3] = _mm_add_epi16(step2[3], step2[12]);
-  output[4] = _mm_add_epi16(step2[4], step2[11]);
-  output[5] = _mm_add_epi16(step2[5], step2[10]);
-  output[6] = _mm_add_epi16(step2[6], step1[9]);
-  output[7] = _mm_add_epi16(step2[7], step1[8]);
-  output[8] = _mm_sub_epi16(step2[7], step1[8]);
-  output[9] = _mm_sub_epi16(step2[6], step1[9]);
-  output[10] = _mm_sub_epi16(step2[5], step2[10]);
-  output[11] = _mm_sub_epi16(step2[4], step2[11]);
-  output[12] = _mm_sub_epi16(step2[3], step2[12]);
-  output[13] = _mm_sub_epi16(step2[2], step2[13]);
-  output[14] = _mm_sub_epi16(step2[1], step1[14]);
-  output[15] = _mm_sub_epi16(step2[0], step1[15]);
-}
-
-static INLINE void idct16x16_10_pass2(__m128i *const l /*l[8]*/,
-                                      __m128i *const io /*io[16]*/) {
-  const __m128i zero = _mm_setzero_si128();
-  const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-  __m128i step1[16], step2[16];
-
-  transpose_16bit_4x8(l, io);
-
-  // stage 2
-  {
-    const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
-    const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
-    const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
-    const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
-    multiplication_and_add(&io[1], &zero, &zero, &io[3], &k__cospi_p30_m02,
-                           &k__cospi_p02_p30, &k__cospi_p06_m26,
-                           &k__cospi_p26_p06, &step2[8], &step2[15], &step2[11],
-                           &step2[12]);
-  }
-
-  // stage 3
-  {
-    const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-    const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
-    multiplication_and_add_2(&io[2], &zero, &k__cospi_p28_m04,
-                             &k__cospi_p04_p28, &step1[4], &step1[7]);
-  }
-
-  // stage 4
-  {
-    const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-    const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-    const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-    multiplication_and_add_2(&zero, &io[0], &k__cospi_p16_p16,
-                             &k__cospi_m16_p16, &step1[0], &step1[1]);
-    multiplication_and_add(&step2[8], &step2[15], &step2[11], &step2[12],
-                           &k__cospi_m08_p24, &k__cospi_p24_p08,
-                           &k__cospi_m24_m08, &k__cospi_m08_p24, &step2[9],
-                           &step2[14], &step2[10], &step2[13]);
-  }
-
-  // stage 5
-  multiplication_and_add_2(&step1[4], &step1[7], &k__cospi_m16_p16,
-                           &k__cospi_p16_p16, &step1[5], &step1[6]);
-  step1[8] = _mm_add_epi16(step2[8], step2[11]);
-  step1[9] = _mm_add_epi16(step2[9], step2[10]);
-  step1[10] = _mm_sub_epi16(step2[9], step2[10]);
-  step1[11] = _mm_sub_epi16(step2[8], step2[11]);
-  step1[12] = _mm_sub_epi16(step2[15], step2[12]);
-  step1[13] = _mm_sub_epi16(step2[14], step2[13]);
-  step1[14] = _mm_add_epi16(step2[14], step2[13]);
-  step1[15] = _mm_add_epi16(step2[15], step2[12]);
-
-  // stage 6
-  step2[0] = _mm_add_epi16(step1[0], step1[7]);
-  step2[1] = _mm_add_epi16(step1[1], step1[6]);
-  step2[2] = _mm_add_epi16(step1[1], step1[5]);
-  step2[3] = _mm_add_epi16(step1[0], step1[4]);
-  step2[4] = _mm_sub_epi16(step1[0], step1[4]);
-  step2[5] = _mm_sub_epi16(step1[1], step1[5]);
-  step2[6] = _mm_sub_epi16(step1[1], step1[6]);
-  step2[7] = _mm_sub_epi16(step1[0], step1[7]);
-  multiplication_and_add(&step1[10], &step1[13], &step1[11], &step1[12],
-                         &k__cospi_m16_p16, &k__cospi_p16_p16,
-                         &k__cospi_m16_p16, &k__cospi_p16_p16, &step2[10],
-                         &step2[13], &step2[11], &step2[12]);
-
-  // stage 7
-  io[0] = _mm_add_epi16(step2[0], step1[15]);
-  io[1] = _mm_add_epi16(step2[1], step1[14]);
-  io[2] = _mm_add_epi16(step2[2], step2[13]);
-  io[3] = _mm_add_epi16(step2[3], step2[12]);
-  io[4] = _mm_add_epi16(step2[4], step2[11]);
-  io[5] = _mm_add_epi16(step2[5], step2[10]);
-  io[6] = _mm_add_epi16(step2[6], step1[9]);
-  io[7] = _mm_add_epi16(step2[7], step1[8]);
-  io[8] = _mm_sub_epi16(step2[7], step1[8]);
-  io[9] = _mm_sub_epi16(step2[6], step1[9]);
-  io[10] = _mm_sub_epi16(step2[5], step2[10]);
-  io[11] = _mm_sub_epi16(step2[4], step2[11]);
-  io[12] = _mm_sub_epi16(step2[3], step2[12]);
-  io[13] = _mm_sub_epi16(step2[2], step2[13]);
-  io[14] = _mm_sub_epi16(step2[1], step1[14]);
-  io[15] = _mm_sub_epi16(step2[0], step1[15]);
-}
-
 void vpx_idct16x16_10_add_sse2(const tran_low_t *input, uint8_t *dest,
                                int stride) {
   __m128i in[16], l[16];
@@ -1312,20 +1001,20 @@
 
 #define IDCT32_34                                                              \
   /* Stage1 */                                                                 \
-  multiplication_and_add_2(&in[1], &zero, &stg1_0, &stg1_1, &stp1_16,          \
-                           &stp1_31);                                          \
-  multiplication_and_add_2(&zero, &in[7], &stg1_6, &stg1_7, &stp1_19,          \
-                           &stp1_28);                                          \
-  multiplication_and_add_2(&in[5], &zero, &stg1_8, &stg1_9, &stp1_20,          \
-                           &stp1_27);                                          \
-  multiplication_and_add_2(&zero, &in[3], &stg1_14, &stg1_15, &stp1_23,        \
-                           &stp1_24);                                          \
+  multiplication_and_add(in[1], zero, (int)cospi_31_64, (int)cospi_1_64,       \
+                         &stp1_16, &stp1_31);                                  \
+  multiplication_and_add(zero, in[7], (int)cospi_7_64, (int)cospi_25_64,       \
+                         &stp1_19, &stp1_28);                                  \
+  multiplication_and_add(in[5], zero, (int)cospi_27_64, (int)cospi_5_64,       \
+                         &stp1_20, &stp1_27);                                  \
+  multiplication_and_add(zero, in[3], (int)cospi_3_64, (int)cospi_29_64,       \
+                         &stp1_23, &stp1_24);                                  \
                                                                                \
   /* Stage2 */                                                                 \
-  multiplication_and_add_2(&in[2], &zero, &stg2_0, &stg2_1, &stp2_8,           \
-                           &stp2_15);                                          \
-  multiplication_and_add_2(&zero, &in[6], &stg2_6, &stg2_7, &stp2_11,          \
-                           &stp2_12);                                          \
+  multiplication_and_add(in[2], zero, (int)cospi_30_64, (int)cospi_2_64,       \
+                         &stp2_8, &stp2_15);                                   \
+  multiplication_and_add(zero, in[6], (int)cospi_6_64, (int)cospi_26_64,       \
+                         &stp2_11, &stp2_12);                                  \
                                                                                \
   stp2_16 = stp1_16;                                                           \
   stp2_19 = stp1_19;                                                           \
@@ -1340,7 +1029,8 @@
   stp2_31 = stp1_31;                                                           \
                                                                                \
   /* Stage3 */                                                                 \
-  multiplication_and_add_2(&in[4], &zero, &stg3_0, &stg3_1, &stp1_4, &stp1_7); \
+  multiplication_and_add(in[4], zero, (int)cospi_28_64, (int)cospi_4_64,       \
+                         &stp1_4, &stp1_7);                                    \
                                                                                \
   stp1_8 = stp2_8;                                                             \
   stp1_11 = stp2_11;                                                           \
@@ -1347,13 +1037,15 @@
   stp1_12 = stp2_12;                                                           \
   stp1_15 = stp2_15;                                                           \
                                                                                \
-  multiplication_and_add(&stp1_16, &stp1_31, &stp1_19, &stp1_28, &stg3_4,      \
-                         &stg3_5, &stg3_6, &stg3_4, &stp1_17, &stp1_30,        \
-                         &stp1_18, &stp1_29);                                  \
+  multiplication_and_add(stp1_31, stp1_16, (int)cospi_28_64, (int)cospi_4_64,  \
+                         &stp1_17, &stp1_30);                                  \
+  multiplication_and_add(stp1_19, stp1_28, -(int)cospi_4_64,                   \
+                         -(int)cospi_28_64, &stp1_29, &stp1_18);               \
                                                                                \
-  multiplication_and_add(&stp1_20, &stp1_27, &stp1_23, &stp1_24, &stg3_8,      \
-                         &stg3_9, &stg3_10, &stg3_8, &stp1_21, &stp1_26,       \
-                         &stp1_22, &stp1_25);                                  \
+  multiplication_and_add(stp1_27, stp1_20, (int)cospi_12_64, (int)cospi_20_64, \
+                         &stp1_21, &stp1_26);                                  \
+  multiplication_and_add(stp1_23, stp1_24, -(int)cospi_20_64,                  \
+                         -(int)cospi_12_64, &stp1_25, &stp1_22);               \
                                                                                \
   stp1_16 = stp2_16;                                                           \
   stp1_31 = stp2_31;                                                           \
@@ -1365,7 +1057,8 @@
   stp1_28 = stp2_28;                                                           \
                                                                                \
   /* Stage4 */                                                                 \
-  multiplication_and_add_2(&in[0], &zero, &stg4_0, &stg4_1, &stp2_0, &stp2_1); \
+  multiplication_and_add(in[0], zero, (int)cospi_16_64, (int)cospi_16_64,      \
+                         &stp2_1, &stp2_0);                                    \
                                                                                \
   stp2_4 = stp1_4;                                                             \
   stp2_5 = stp1_4;                                                             \
@@ -1372,9 +1065,10 @@
   stp2_6 = stp1_7;                                                             \
   stp2_7 = stp1_7;                                                             \
                                                                                \
-  multiplication_and_add(&stp2_8, &stp2_15, &stp2_11, &stp2_12, &stg4_4,       \
-                         &stg4_5, &stg4_6, &stg4_4, &stp2_9, &stp2_14,         \
-                         &stp2_10, &stp2_13);                                  \
+  multiplication_and_add(stp2_15, stp2_8, (int)cospi_24_64, (int)cospi_8_64,   \
+                         &stp2_9, &stp2_14);                                   \
+  multiplication_and_add(stp2_11, stp2_12, -(int)cospi_8_64,                   \
+                         -(int)cospi_24_64, &stp2_13, &stp2_10);               \
                                                                                \
   stp2_8 = stp1_8;                                                             \
   stp2_15 = stp1_15;                                                           \
@@ -1404,8 +1098,8 @@
   stp1_1 = stp2_1;                                                             \
   stp1_2 = stp2_1;                                                             \
   stp1_3 = stp2_0;                                                             \
-  multiplication_and_add_2(&stp2_6, &stp2_5, &stg4_1, &stg4_0, &stp1_5,        \
-                           &stp1_6);                                           \
+  multiplication_and_add(stp2_6, stp2_5, (int)cospi_16_64, (int)cospi_16_64,   \
+                         &stp1_5, &stp1_6);                                    \
                                                                                \
   stp1_4 = stp2_4;                                                             \
   stp1_7 = stp2_7;                                                             \
@@ -1422,12 +1116,14 @@
   stp1_16 = stp2_16;                                                           \
   stp1_17 = stp2_17;                                                           \
                                                                                \
-  multiplication_and_add(&stp2_18, &stp2_29, &stp2_19, &stp2_28, &stg4_4,      \
-                         &stg4_5, &stg4_4, &stg4_5, &stp1_18, &stp1_29,        \
+  multiplication_and_add(stp2_29, stp2_18, (int)cospi_24_64, (int)cospi_8_64,  \
+                         &stp1_18, &stp1_29);                                  \
+  multiplication_and_add(stp2_28, stp2_19, (int)cospi_24_64, (int)cospi_8_64,  \
                          &stp1_19, &stp1_28);                                  \
-  multiplication_and_add(&stp2_20, &stp2_27, &stp2_21, &stp2_26, &stg4_6,      \
-                         &stg4_4, &stg4_6, &stg4_4, &stp1_20, &stp1_27,        \
-                         &stp1_21, &stp1_26);                                  \
+  multiplication_and_add(stp2_20, stp2_27, -(int)cospi_8_64,                   \
+                         -(int)cospi_24_64, &stp1_27, &stp1_20);               \
+  multiplication_and_add(stp2_21, stp2_26, -(int)cospi_8_64,                   \
+                         -(int)cospi_24_64, &stp1_26, &stp1_21);               \
                                                                                \
   stp1_22 = stp2_22;                                                           \
   stp1_23 = stp2_23;                                                           \
@@ -1451,8 +1147,9 @@
   stp2_14 = stp1_14;                                                           \
   stp2_15 = stp1_15;                                                           \
                                                                                \
-  multiplication_and_add(&stp1_10, &stp1_13, &stp1_11, &stp1_12, &stg6_0,      \
-                         &stg4_0, &stg6_0, &stg4_0, &stp2_10, &stp2_13,        \
+  multiplication_and_add(stp1_13, stp1_10, (int)cospi_16_64, (int)cospi_16_64, \
+                         &stp2_10, &stp2_13);                                  \
+  multiplication_and_add(stp1_12, stp1_11, (int)cospi_16_64, (int)cospi_16_64, \
                          &stp2_11, &stp2_12);                                  \
                                                                                \
   stp2_16 = _mm_add_epi16(stp1_16, stp1_23);                                   \
@@ -1496,11 +1193,13 @@
   stp1_18 = stp2_18;                                                           \
   stp1_19 = stp2_19;                                                           \
                                                                                \
-  multiplication_and_add(&stp2_20, &stp2_27, &stp2_21, &stp2_26, &stg6_0,      \
-                         &stg4_0, &stg6_0, &stg4_0, &stp1_20, &stp1_27,        \
+  multiplication_and_add(stp2_27, stp2_20, (int)cospi_16_64, (int)cospi_16_64, \
+                         &stp1_20, &stp1_27);                                  \
+  multiplication_and_add(stp2_26, stp2_21, (int)cospi_16_64, (int)cospi_16_64, \
                          &stp1_21, &stp1_26);                                  \
-  multiplication_and_add(&stp2_22, &stp2_25, &stp2_23, &stp2_24, &stg6_0,      \
-                         &stg4_0, &stg6_0, &stg4_0, &stp1_22, &stp1_25,        \
+  multiplication_and_add(stp2_25, stp2_22, (int)cospi_16_64, (int)cospi_16_64, \
+                         &stp1_22, &stp1_25);                                  \
+  multiplication_and_add(stp2_24, stp2_23, (int)cospi_16_64, (int)cospi_16_64, \
                          &stp1_23, &stp1_24);                                  \
                                                                                \
   stp1_28 = stp2_28;                                                           \
@@ -1512,39 +1211,6 @@
 void vpx_idct32x32_34_add_sse2(const tran_low_t *input, uint8_t *dest,
                                int stride) {
   const __m128i zero = _mm_setzero_si128();
-
-  // idct constants for each stage
-  const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64);
-  const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64);
-  const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64);
-  const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64);
-  const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64);
-  const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64);
-  const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64);
-  const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64);
-
-  const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64);
-  const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64);
-  const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64);
-  const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64);
-
-  const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-  const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64);
-  const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
-  const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
-  const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
-  const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-  const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
-  const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
-
-  const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-  const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-  const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
-
-  const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-
   __m128i in[32], col[32];
   __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7,
       stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15,
--- a/vpx_dsp/x86/inv_txfm_sse2.h
+++ b/vpx_dsp/x86/inv_txfm_sse2.h
@@ -77,27 +77,17 @@
   return _mm_packs_epi32(t0, t1);
 }
 
-static INLINE void multiplication_and_add_2(const __m128i *const in0,
-                                            const __m128i *const in1,
-                                            const __m128i *const cst0,
-                                            const __m128i *const cst1,
-                                            __m128i *const res0,
-                                            __m128i *const res1) {
-  const __m128i lo = _mm_unpacklo_epi16(*in0, *in1);
-  const __m128i hi = _mm_unpackhi_epi16(*in0, *in1);
-  *res0 = idct_calc_wraplow_sse2(lo, hi, *cst0);
-  *res1 = idct_calc_wraplow_sse2(lo, hi, *cst1);
-}
-
 // Multiply elements by constants and add them together.
-static INLINE void multiplication_and_add(
-    const __m128i *const in0, const __m128i *const in1,
-    const __m128i *const in2, const __m128i *const in3,
-    const __m128i *const cst0, const __m128i *const cst1,
-    const __m128i *const cst2, const __m128i *const cst3, __m128i *const res0,
-    __m128i *const res1, __m128i *const res2, __m128i *const res3) {
-  multiplication_and_add_2(in0, in1, cst0, cst1, res0, res1);
-  multiplication_and_add_2(in2, in3, cst2, cst3, res2, res3);
+static INLINE void multiplication_and_add(const __m128i in0, const __m128i in1,
+                                          const int c0, const int c1,
+                                          __m128i *const res0,
+                                          __m128i *const res1) {
+  const __m128i cst0 = pair_set_epi16(c0, -c1);
+  const __m128i cst1 = pair_set_epi16(c1, c0);
+  const __m128i lo = _mm_unpacklo_epi16(in0, in1);
+  const __m128i hi = _mm_unpackhi_epi16(in0, in1);
+  *res0 = idct_calc_wraplow_sse2(lo, hi, cst0);
+  *res1 = idct_calc_wraplow_sse2(lo, hi, cst1);
 }
 
 // Functions to allow 8 bit optimisations to be used when profile 0 is used with
@@ -361,29 +351,19 @@
 
 static INLINE void idct8(const __m128i *const in /*in[8]*/,
                          __m128i *const out /*out[8]*/) {
-  const __m128i cp_16_16 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i cp_16_n16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
   __m128i step1[8], step2[8];
 
   // stage 1
-  {
-    const __m128i cp_28_n4 = pair_set_epi16(cospi_28_64, -cospi_4_64);
-    const __m128i cp_4_28 = pair_set_epi16(cospi_4_64, cospi_28_64);
-    const __m128i cp_n20_12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-    const __m128i cp_12_20 = pair_set_epi16(cospi_12_64, cospi_20_64);
-    multiplication_and_add(&in[1], &in[7], &in[3], &in[5], &cp_28_n4, &cp_4_28,
-                           &cp_n20_12, &cp_12_20, &step1[4], &step1[7],
-                           &step1[5], &step1[6]);
-  }
+  multiplication_and_add(in[1], in[7], (int)cospi_28_64, (int)cospi_4_64,
+                         &step1[4], &step1[7]);
+  multiplication_and_add(in[5], in[3], (int)cospi_12_64, (int)cospi_20_64,
+                         &step1[5], &step1[6]);
 
   // stage 2
-  {
-    const __m128i cp_24_n8 = pair_set_epi16(cospi_24_64, -cospi_8_64);
-    const __m128i cp_8_24 = pair_set_epi16(cospi_8_64, cospi_24_64);
-    multiplication_and_add(&in[0], &in[4], &in[2], &in[6], &cp_16_16,
-                           &cp_16_n16, &cp_24_n8, &cp_8_24, &step2[0],
-                           &step2[1], &step2[2], &step2[3]);
-  }
+  multiplication_and_add(in[0], in[4], (int)cospi_16_64, (int)cospi_16_64,
+                         &step2[1], &step2[0]);
+  multiplication_and_add(in[2], in[6], (int)cospi_24_64, (int)cospi_8_64,
+                         &step2[2], &step2[3]);
 
   step2[4] = _mm_add_epi16(step1[4], step1[5]);
   step2[5] = _mm_sub_epi16(step1[4], step1[5]);
@@ -395,8 +375,8 @@
   step1[1] = _mm_add_epi16(step2[1], step2[2]);
   step1[2] = _mm_sub_epi16(step2[1], step2[2]);
   step1[3] = _mm_sub_epi16(step2[0], step2[3]);
-  multiplication_and_add_2(&step2[6], &step2[5], &cp_16_n16, &cp_16_16,
-                           &step1[5], &step1[6]);
+  multiplication_and_add(step2[6], step2[5], (int)cospi_16_64, (int)cospi_16_64,
+                         &step1[5], &step1[6]);
 
   // stage 4
   out[0] = _mm_add_epi16(step1[0], step2[7]);
@@ -465,6 +445,275 @@
   io[4] = io[5] = io[6] = io[7] = zero;
 
   idct8(io, io);
+}
+
+static INLINE void idct16_8col(__m128i *const io /*io[16]*/) {
+  __m128i step1[16], step2[16];
+
+  // stage 2
+  multiplication_and_add(io[1], io[15], (int)cospi_30_64, (int)cospi_2_64,
+                         &step2[8], &step2[15]);
+  multiplication_and_add(io[9], io[7], (int)cospi_14_64, (int)cospi_18_64,
+                         &step2[9], &step2[14]);
+  multiplication_and_add(io[5], io[11], (int)cospi_22_64, (int)cospi_10_64,
+                         &step2[10], &step2[13]);
+  multiplication_and_add(io[13], io[3], (int)cospi_6_64, (int)cospi_26_64,
+                         &step2[11], &step2[12]);
+
+  // stage 3
+  multiplication_and_add(io[2], io[14], (int)cospi_28_64, (int)cospi_4_64,
+                         &step1[4], &step1[7]);
+  multiplication_and_add(io[10], io[6], (int)cospi_12_64, (int)cospi_20_64,
+                         &step1[5], &step1[6]);
+  step1[8] = _mm_add_epi16(step2[8], step2[9]);
+  step1[9] = _mm_sub_epi16(step2[8], step2[9]);
+  step1[10] = _mm_sub_epi16(step2[11], step2[10]);
+  step1[11] = _mm_add_epi16(step2[10], step2[11]);
+  step1[12] = _mm_add_epi16(step2[12], step2[13]);
+  step1[13] = _mm_sub_epi16(step2[12], step2[13]);
+  step1[14] = _mm_sub_epi16(step2[15], step2[14]);
+  step1[15] = _mm_add_epi16(step2[14], step2[15]);
+
+  // stage 4
+  multiplication_and_add(io[0], io[8], (int)cospi_16_64, (int)cospi_16_64,
+                         &step2[1], &step2[0]);
+  multiplication_and_add(io[4], io[12], (int)cospi_24_64, (int)cospi_8_64,
+                         &step2[2], &step2[3]);
+  multiplication_and_add(step1[14], step1[9], (int)cospi_24_64, (int)cospi_8_64,
+                         &step2[9], &step2[14]);
+  multiplication_and_add(step1[10], step1[13], -(int)cospi_8_64,
+                         -(int)cospi_24_64, &step2[13], &step2[10]);
+  step2[5] = _mm_sub_epi16(step1[4], step1[5]);
+  step1[4] = _mm_add_epi16(step1[4], step1[5]);
+  step2[6] = _mm_sub_epi16(step1[7], step1[6]);
+  step1[7] = _mm_add_epi16(step1[6], step1[7]);
+  step2[8] = step1[8];
+  step2[11] = step1[11];
+  step2[12] = step1[12];
+  step2[15] = step1[15];
+
+  // stage 5
+  step1[0] = _mm_add_epi16(step2[0], step2[3]);
+  step1[1] = _mm_add_epi16(step2[1], step2[2]);
+  step1[2] = _mm_sub_epi16(step2[1], step2[2]);
+  step1[3] = _mm_sub_epi16(step2[0], step2[3]);
+  multiplication_and_add(step2[6], step2[5], (int)cospi_16_64, (int)cospi_16_64,
+                         &step1[5], &step1[6]);
+  step1[8] = _mm_add_epi16(step2[8], step2[11]);
+  step1[9] = _mm_add_epi16(step2[9], step2[10]);
+  step1[10] = _mm_sub_epi16(step2[9], step2[10]);
+  step1[11] = _mm_sub_epi16(step2[8], step2[11]);
+  step1[12] = _mm_sub_epi16(step2[15], step2[12]);
+  step1[13] = _mm_sub_epi16(step2[14], step2[13]);
+  step1[14] = _mm_add_epi16(step2[14], step2[13]);
+  step1[15] = _mm_add_epi16(step2[15], step2[12]);
+
+  // stage 6
+  step2[0] = _mm_add_epi16(step1[0], step1[7]);
+  step2[1] = _mm_add_epi16(step1[1], step1[6]);
+  step2[2] = _mm_add_epi16(step1[2], step1[5]);
+  step2[3] = _mm_add_epi16(step1[3], step1[4]);
+  step2[4] = _mm_sub_epi16(step1[3], step1[4]);
+  step2[5] = _mm_sub_epi16(step1[2], step1[5]);
+  step2[6] = _mm_sub_epi16(step1[1], step1[6]);
+  step2[7] = _mm_sub_epi16(step1[0], step1[7]);
+  multiplication_and_add(step1[13], step1[10], (int)cospi_16_64,
+                         (int)cospi_16_64, &step2[10], &step2[13]);
+  multiplication_and_add(step1[12], step1[11], (int)cospi_16_64,
+                         (int)cospi_16_64, &step2[11], &step2[12]);
+
+  // stage 7
+  io[0] = _mm_add_epi16(step2[0], step1[15]);
+  io[1] = _mm_add_epi16(step2[1], step1[14]);
+  io[2] = _mm_add_epi16(step2[2], step2[13]);
+  io[3] = _mm_add_epi16(step2[3], step2[12]);
+  io[4] = _mm_add_epi16(step2[4], step2[11]);
+  io[5] = _mm_add_epi16(step2[5], step2[10]);
+  io[6] = _mm_add_epi16(step2[6], step1[9]);
+  io[7] = _mm_add_epi16(step2[7], step1[8]);
+  io[8] = _mm_sub_epi16(step2[7], step1[8]);
+  io[9] = _mm_sub_epi16(step2[6], step1[9]);
+  io[10] = _mm_sub_epi16(step2[5], step2[10]);
+  io[11] = _mm_sub_epi16(step2[4], step2[11]);
+  io[12] = _mm_sub_epi16(step2[3], step2[12]);
+  io[13] = _mm_sub_epi16(step2[2], step2[13]);
+  io[14] = _mm_sub_epi16(step2[1], step1[14]);
+  io[15] = _mm_sub_epi16(step2[0], step1[15]);
+}
+
+static INLINE void idct16x16_10_pass1(const __m128i *const input /*input[4]*/,
+                                      __m128i *const output /*output[16]*/) {
+  const __m128i zero = _mm_setzero_si128();
+  const __m128i k__cospi_p16_p16 = pair_set_epi16(cospi_16_64, cospi_16_64);
+  const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+  __m128i step1[16], step2[16];
+
+  transpose_16bit_4x4(input, output);
+
+  // stage 2
+  {
+    const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
+    const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
+    const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
+    const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
+    const __m128i lo_1_15 = _mm_unpackhi_epi16(output[0], zero);
+    const __m128i lo_13_3 = _mm_unpackhi_epi16(zero, output[1]);
+    step2[8] = idct_calc_wraplow_sse2(k__cospi_p30_m02, k__cospi_p02_p30,
+                                      lo_1_15);  // step2 8&15
+    step2[11] = idct_calc_wraplow_sse2(k__cospi_p06_m26, k__cospi_p26_p06,
+                                       lo_13_3);  // step2 11&12
+  }
+
+  // stage 3
+  {
+    const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
+    const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
+    const __m128i lo_2_14 = _mm_unpacklo_epi16(output[1], zero);
+    step1[4] = idct_calc_wraplow_sse2(k__cospi_p28_m04, k__cospi_p04_p28,
+                                      lo_2_14);  // step1 4&7
+    step1[13] = _mm_unpackhi_epi64(step2[11], zero);
+    step1[14] = _mm_unpackhi_epi64(step2[8], zero);
+  }
+
+  // stage 4
+  {
+    const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+    const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
+    const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+    const __m128i lo_0_8 = _mm_unpacklo_epi16(output[0], zero);
+    const __m128i lo_9_14 = _mm_unpacklo_epi16(step2[8], step1[14]);
+    const __m128i lo_10_13 = _mm_unpacklo_epi16(step2[11], step1[13]);
+    const __m128i t = idct_madd_round_shift_sse2(lo_0_8, k__cospi_p16_p16);
+    step1[0] = _mm_packs_epi32(t, t);  // step2 0&1
+    step2[9] = idct_calc_wraplow_sse2(k__cospi_m08_p24, k__cospi_p24_p08,
+                                      lo_9_14);  // step2 9&14
+    step2[10] = idct_calc_wraplow_sse2(k__cospi_m24_m08, k__cospi_m08_p24,
+                                       lo_10_13);  // step2 10&13
+    step2[6] = _mm_unpackhi_epi64(step1[4], zero);
+  }
+
+  // stage 5
+  {
+    const __m128i lo_5_6 = _mm_unpacklo_epi16(step1[4], step2[6]);
+    step1[6] = idct_calc_wraplow_sse2(k__cospi_p16_p16, k__cospi_m16_p16,
+                                      lo_5_6);  // step1 6&5
+    step1[8] = _mm_add_epi16(step2[8], step2[11]);
+    step1[9] = _mm_add_epi16(step2[9], step2[10]);
+    step1[10] = _mm_sub_epi16(step2[9], step2[10]);
+    step1[11] = _mm_sub_epi16(step2[8], step2[11]);
+    step1[12] = _mm_unpackhi_epi64(step1[11], zero);
+    step1[13] = _mm_unpackhi_epi64(step1[10], zero);
+    step1[14] = _mm_unpackhi_epi64(step1[9], zero);
+    step1[15] = _mm_unpackhi_epi64(step1[8], zero);
+  }
+
+  // stage 6
+  {
+    const __m128i lo_10_13 = _mm_unpacklo_epi16(step1[10], step1[13]);
+    const __m128i lo_11_12 = _mm_unpacklo_epi16(step1[11], step1[12]);
+    step2[10] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
+                                       lo_10_13);  // step2 10&13
+    step2[11] = idct_calc_wraplow_sse2(k__cospi_m16_p16, k__cospi_p16_p16,
+                                       lo_11_12);  // step2 11&12
+    step2[13] = _mm_unpackhi_epi64(step2[10], zero);
+    step2[12] = _mm_unpackhi_epi64(step2[11], zero);
+    step2[3] = _mm_add_epi16(step1[0], step1[4]);
+    step2[1] = _mm_add_epi16(step1[0], step1[6]);
+    step2[6] = _mm_sub_epi16(step1[0], step1[6]);
+    step2[4] = _mm_sub_epi16(step1[0], step1[4]);
+    step2[0] = _mm_unpackhi_epi64(step2[3], zero);
+    step2[2] = _mm_unpackhi_epi64(step2[1], zero);
+    step2[5] = _mm_unpackhi_epi64(step2[6], zero);
+    step2[7] = _mm_unpackhi_epi64(step2[4], zero);
+  }
+
+  // stage 7. Left 8x16 only.
+  output[0] = _mm_add_epi16(step2[0], step1[15]);
+  output[1] = _mm_add_epi16(step2[1], step1[14]);
+  output[2] = _mm_add_epi16(step2[2], step2[13]);
+  output[3] = _mm_add_epi16(step2[3], step2[12]);
+  output[4] = _mm_add_epi16(step2[4], step2[11]);
+  output[5] = _mm_add_epi16(step2[5], step2[10]);
+  output[6] = _mm_add_epi16(step2[6], step1[9]);
+  output[7] = _mm_add_epi16(step2[7], step1[8]);
+  output[8] = _mm_sub_epi16(step2[7], step1[8]);
+  output[9] = _mm_sub_epi16(step2[6], step1[9]);
+  output[10] = _mm_sub_epi16(step2[5], step2[10]);
+  output[11] = _mm_sub_epi16(step2[4], step2[11]);
+  output[12] = _mm_sub_epi16(step2[3], step2[12]);
+  output[13] = _mm_sub_epi16(step2[2], step2[13]);
+  output[14] = _mm_sub_epi16(step2[1], step1[14]);
+  output[15] = _mm_sub_epi16(step2[0], step1[15]);
+}
+
+static INLINE void idct16x16_10_pass2(__m128i *const l /*l[8]*/,
+                                      __m128i *const io /*io[16]*/) {
+  const __m128i zero = _mm_setzero_si128();
+  __m128i step1[16], step2[16];
+
+  transpose_16bit_4x8(l, io);
+
+  // stage 2
+  multiplication_and_add(io[1], zero, (int)cospi_30_64, (int)cospi_2_64,
+                         &step2[8], &step2[15]);
+  multiplication_and_add(zero, io[3], (int)cospi_6_64, (int)cospi_26_64,
+                         &step2[11], &step2[12]);
+
+  // stage 3
+  multiplication_and_add(io[2], zero, (int)cospi_28_64, (int)cospi_4_64,
+                         &step1[4], &step1[7]);
+
+  // stage 4
+  multiplication_and_add(io[0], zero, (int)cospi_16_64, (int)cospi_16_64,
+                         &step1[1], &step1[0]);
+  multiplication_and_add(step2[15], step2[8], (int)cospi_24_64, (int)cospi_8_64,
+                         &step2[9], &step2[14]);
+  multiplication_and_add(step2[11], step2[12], -(int)cospi_8_64,
+                         -(int)cospi_24_64, &step2[13], &step2[10]);
+
+  // stage 5
+  multiplication_and_add(step1[7], step1[4], (int)cospi_16_64, (int)cospi_16_64,
+                         &step1[5], &step1[6]);
+  step1[8] = _mm_add_epi16(step2[8], step2[11]);
+  step1[9] = _mm_add_epi16(step2[9], step2[10]);
+  step1[10] = _mm_sub_epi16(step2[9], step2[10]);
+  step1[11] = _mm_sub_epi16(step2[8], step2[11]);
+  step1[12] = _mm_sub_epi16(step2[15], step2[12]);
+  step1[13] = _mm_sub_epi16(step2[14], step2[13]);
+  step1[14] = _mm_add_epi16(step2[14], step2[13]);
+  step1[15] = _mm_add_epi16(step2[15], step2[12]);
+
+  // stage 6
+  step2[0] = _mm_add_epi16(step1[0], step1[7]);
+  step2[1] = _mm_add_epi16(step1[1], step1[6]);
+  step2[2] = _mm_add_epi16(step1[1], step1[5]);
+  step2[3] = _mm_add_epi16(step1[0], step1[4]);
+  step2[4] = _mm_sub_epi16(step1[0], step1[4]);
+  step2[5] = _mm_sub_epi16(step1[1], step1[5]);
+  step2[6] = _mm_sub_epi16(step1[1], step1[6]);
+  step2[7] = _mm_sub_epi16(step1[0], step1[7]);
+  multiplication_and_add(step1[13], step1[10], (int)cospi_16_64,
+                         (int)cospi_16_64, &step2[10], &step2[13]);
+  multiplication_and_add(step1[12], step1[11], (int)cospi_16_64,
+                         (int)cospi_16_64, &step2[11], &step2[12]);
+
+  // stage 7
+  io[0] = _mm_add_epi16(step2[0], step1[15]);
+  io[1] = _mm_add_epi16(step2[1], step1[14]);
+  io[2] = _mm_add_epi16(step2[2], step2[13]);
+  io[3] = _mm_add_epi16(step2[3], step2[12]);
+  io[4] = _mm_add_epi16(step2[4], step2[11]);
+  io[5] = _mm_add_epi16(step2[5], step2[10]);
+  io[6] = _mm_add_epi16(step2[6], step1[9]);
+  io[7] = _mm_add_epi16(step2[7], step1[8]);
+  io[8] = _mm_sub_epi16(step2[7], step1[8]);
+  io[9] = _mm_sub_epi16(step2[6], step1[9]);
+  io[10] = _mm_sub_epi16(step2[5], step2[10]);
+  io[11] = _mm_sub_epi16(step2[4], step2[11]);
+  io[12] = _mm_sub_epi16(step2[3], step2[12]);
+  io[13] = _mm_sub_epi16(step2[2], step2[13]);
+  io[14] = _mm_sub_epi16(step2[1], step1[14]);
+  io[15] = _mm_sub_epi16(step2[0], step1[15]);
 }
 
 void idct4_sse2(__m128i *in);
--- a/vpx_dsp/x86/inv_txfm_ssse3.h
+++ b/vpx_dsp/x86/inv_txfm_ssse3.h
@@ -92,8 +92,8 @@
   step1[1] = _mm_add_epi16(step2[0], step2[2]);
   step1[2] = _mm_sub_epi16(step2[0], step2[2]);
   step1[3] = _mm_sub_epi16(step2[0], step2[3]);
-  multiplication_and_add_2(&step2[6], &step2[5], &cp_16_n16, &cp_16_16,
-                           &step1[5], &step1[6]);
+  multiplication_and_add(step2[6], step2[5], (int)cospi_16_64, (int)cospi_16_64,
+                         &step1[5], &step1[6]);
 
   // stage 4
   io[0] = _mm_add_epi16(step1[0], step2[7]);
--- a/vpx_dsp/x86/transpose_sse2.h
+++ b/vpx_dsp/x86/transpose_sse2.h
@@ -237,4 +237,52 @@
   out[7] = _mm_unpackhi_epi64(a6, a7);
 }
 
+static INLINE void transpose_32bit_8x4(const __m128i *const in,
+                                       __m128i *const out) {
+  // Unpack 32 bit elements. Goes from:
+  // in[0]: 00 01 02 03
+  // in[1]: 04 05 06 07
+  // in[2]: 10 11 12 13
+  // in[3]: 14 15 16 17
+  // in[4]: 20 21 22 23
+  // in[5]: 24 25 26 27
+  // in[6]: 30 31 32 33
+  // in[7]: 34 35 36 37
+  // to:
+  // a0: 00 10 01 11
+  // a1: 20 30 21 31
+  // a2: 02 12 03 13
+  // a3: 22 32 23 33
+  // a4: 04 14 05 15
+  // a5: 24 34 25 35
+  // a6: 06 16 07 17
+  // a7: 26 36 27 37
+  const __m128i a0 = _mm_unpacklo_epi32(in[0], in[2]);
+  const __m128i a1 = _mm_unpacklo_epi32(in[4], in[6]);
+  const __m128i a2 = _mm_unpackhi_epi32(in[0], in[2]);
+  const __m128i a3 = _mm_unpackhi_epi32(in[4], in[6]);
+  const __m128i a4 = _mm_unpacklo_epi32(in[1], in[3]);
+  const __m128i a5 = _mm_unpacklo_epi32(in[5], in[7]);
+  const __m128i a6 = _mm_unpackhi_epi32(in[1], in[3]);
+  const __m128i a7 = _mm_unpackhi_epi32(in[5], in[7]);
+
+  // Unpack 64 bit elements resulting in:
+  // out[0]: 00 10 20 30
+  // out[1]: 01 11 21 31
+  // out[2]: 02 12 22 32
+  // out[3]: 03 13 23 33
+  // out[4]: 04 14 24 34
+  // out[5]: 05 15 25 35
+  // out[6]: 06 16 26 36
+  // out[7]: 07 17 27 37
+  out[0] = _mm_unpacklo_epi64(a0, a1);
+  out[1] = _mm_unpackhi_epi64(a0, a1);
+  out[2] = _mm_unpacklo_epi64(a2, a3);
+  out[3] = _mm_unpackhi_epi64(a2, a3);
+  out[4] = _mm_unpacklo_epi64(a4, a5);
+  out[5] = _mm_unpackhi_epi64(a4, a5);
+  out[6] = _mm_unpacklo_epi64(a6, a7);
+  out[7] = _mm_unpackhi_epi64(a6, a7);
+}
+
 #endif  // VPX_DSP_X86_TRANSPOSE_SSE2_H_