shithub: libvpx

Download patch

ref: 018290a344c8144f012f08aaa83491b9749f7e96
parent: f951881e8cc8487b812b2cd562fb1d9d583ec30b
parent: 327add990f85b386726c7b8a8fb2c937e899fdd5
author: Yi Luo <luoyi@google.com>
date: Fri Mar 10 12:14:30 EST 2017

Merge "Improve idct32x32_135_add SSSE3 intrinsics performance"

--- a/vpx_dsp/x86/inv_txfm_ssse3.c
+++ b/vpx_dsp/x86/inv_txfm_ssse3.c
@@ -378,15 +378,35 @@
   RECON_AND_STORE(dest + 7 * stride, in7);
 }
 
-static INLINE void simple_group_butterfly(const __m128i *in, __m128i *out) {
+// Only do addition and subtraction butterfly, size = 16, 32
+static INLINE void add_sub_butterfly(const __m128i *in, __m128i *out,
+                                     int size) {
   int i = 0;
-  while (i < 16) {
-    out[i] = _mm_add_epi16(in[i], in[31 - i]);
-    out[31 - i] = _mm_sub_epi16(in[i], in[31 - i]);
+  const int num = size >> 1;
+  const int bound = size - 1;
+  while (i < num) {
+    out[i] = _mm_add_epi16(in[i], in[bound - i]);
+    out[bound - i] = _mm_sub_epi16(in[i], in[bound - i]);
     i++;
   }
 }
 
+#define BUTTERFLY_PAIR(x0, x1, co0, co1)         \
+  do {                                           \
+    tmp0 = _mm_madd_epi16(x0, co0);              \
+    tmp1 = _mm_madd_epi16(x1, co0);              \
+    tmp2 = _mm_madd_epi16(x0, co1);              \
+    tmp3 = _mm_madd_epi16(x1, co1);              \
+    tmp0 = _mm_add_epi32(tmp0, rounding);        \
+    tmp1 = _mm_add_epi32(tmp1, rounding);        \
+    tmp2 = _mm_add_epi32(tmp2, rounding);        \
+    tmp3 = _mm_add_epi32(tmp3, rounding);        \
+    tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); \
+    tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); \
+    tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); \
+    tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); \
+  } while (0)
+
 static INLINE void butterfly(const __m128i x0, const __m128i x1,
                              const __m128i c0, const __m128i c1, __m128i *y0,
                              __m128i *y1) {
@@ -395,22 +415,7 @@
 
   u0 = _mm_unpacklo_epi16(x0, x1);
   u1 = _mm_unpackhi_epi16(x0, x1);
-
-  tmp0 = _mm_madd_epi16(u0, c0);
-  tmp1 = _mm_madd_epi16(u1, c0);
-  tmp2 = _mm_madd_epi16(u0, c1);
-  tmp3 = _mm_madd_epi16(u1, c1);
-
-  tmp0 = _mm_add_epi32(tmp0, rounding);
-  tmp1 = _mm_add_epi32(tmp1, rounding);
-  tmp2 = _mm_add_epi32(tmp2, rounding);
-  tmp3 = _mm_add_epi32(tmp3, rounding);
-
-  tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
-  tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
-  tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
-  tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
-
+  BUTTERFLY_PAIR(u0, u1, c0, c1);
   *y0 = _mm_packs_epi32(tmp0, tmp1);
   *y1 = _mm_packs_epi32(tmp2, tmp3);
 }
@@ -422,22 +427,7 @@
 
   u0 = _mm_unpacklo_epi16(*x0, *x1);
   u1 = _mm_unpackhi_epi16(*x0, *x1);
-
-  tmp0 = _mm_madd_epi16(u0, c0);
-  tmp1 = _mm_madd_epi16(u1, c0);
-  tmp2 = _mm_madd_epi16(u0, c1);
-  tmp3 = _mm_madd_epi16(u1, c1);
-
-  tmp0 = _mm_add_epi32(tmp0, rounding);
-  tmp1 = _mm_add_epi32(tmp1, rounding);
-  tmp2 = _mm_add_epi32(tmp2, rounding);
-  tmp3 = _mm_add_epi32(tmp3, rounding);
-
-  tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
-  tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
-  tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
-  tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
-
+  BUTTERFLY_PAIR(u0, u1, c0, c1);
   *x0 = _mm_packs_epi32(tmp0, tmp1);
   *x1 = _mm_packs_epi32(tmp2, tmp3);
 }
@@ -649,7 +639,7 @@
   idct32_34_second_half(in, stp1);
 
   // 1_D: Store 32 intermediate results for each 8x32 block.
-  simple_group_butterfly(stp1, col);
+  add_sub_butterfly(stp1, col, 32);
   for (i = 0; i < 4; i++) {
     int j;
     // Transpose 32x8 block to 8x32 block
@@ -658,7 +648,7 @@
     idct32_34_second_half(in, stp1);
 
     // 2_D: Calculate the results and store them to destination.
-    simple_group_butterfly(stp1, in);
+    add_sub_butterfly(stp1, in, 32);
     for (j = 0; j < 32; ++j) {
       // Final rounding and shift
       in[j] = _mm_adds_epi16(in[j], final_rounding);
@@ -690,402 +680,272 @@
   array_transpose_8x8(&in1[8], &out1[8]);
 }
 
-// For each 8x16 block __m128i in[16], output __m128i col[32]
-static void idct32_8x16_135(__m128i *in) {
-  const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
-  const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
-  const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
-  const __m128i stk1_2 = pair_set_epi16(-2 * cospi_17_64, -2 * cospi_17_64);
-  const __m128i stk1_3 = pair_set_epi16(2 * cospi_15_64, 2 * cospi_15_64);
+// Group the coefficient calculation into smaller functions
+// to prevent stack spillover:
+// quarter_1: 0-7
+// quarter_2: 8-15
+// quarter_3_4: 16-23, 24-31
+static void idct32_8x32_135_quarter_1(const __m128i in[16], __m128i out[8]) {
+  __m128i u0, u1, u2, u3, u4, u5, u6, u7;
+  __m128i v0, v1, v2, v3, v4, v5, v6, v7;
 
-  const __m128i stk1_4 = pair_set_epi16(2 * cospi_23_64, 2 * cospi_23_64);
-  const __m128i stk1_5 = pair_set_epi16(2 * cospi_9_64, 2 * cospi_9_64);
-  const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
-  const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
+  {
+    const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
+    const __m128i stk4_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64);
+    const __m128i stk4_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64);
+    u0 = _mm_mulhrs_epi16(in[0], stk4_0);
+    u2 = _mm_mulhrs_epi16(in[8], stk4_2);
+    u3 = _mm_mulhrs_epi16(in[8], stk4_3);
+    u1 = u0;
+  }
 
-  const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
-  const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
-  const __m128i stk1_10 = pair_set_epi16(-2 * cospi_21_64, -2 * cospi_21_64);
-  const __m128i stk1_11 = pair_set_epi16(2 * cospi_11_64, 2 * cospi_11_64);
+  v0 = _mm_add_epi16(u0, u3);
+  v1 = _mm_add_epi16(u1, u2);
+  v2 = _mm_sub_epi16(u1, u2);
+  v3 = _mm_sub_epi16(u0, u3);
 
-  const __m128i stk1_12 = pair_set_epi16(2 * cospi_19_64, 2 * cospi_19_64);
-  const __m128i stk1_13 = pair_set_epi16(2 * cospi_13_64, 2 * cospi_13_64);
-  const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
-  const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
+  {
+    const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
+    const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
+    const __m128i stk3_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64);
+    const __m128i stk3_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64);
+    u4 = _mm_mulhrs_epi16(in[4], stk3_0);
+    u7 = _mm_mulhrs_epi16(in[4], stk3_1);
+    u5 = _mm_mulhrs_epi16(in[12], stk3_2);
+    u6 = _mm_mulhrs_epi16(in[12], stk3_3);
+  }
 
-  const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
-  const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
-  const __m128i stk2_2 = pair_set_epi16(-2 * cospi_18_64, -2 * cospi_18_64);
-  const __m128i stk2_3 = pair_set_epi16(2 * cospi_14_64, 2 * cospi_14_64);
+  v4 = _mm_add_epi16(u4, u5);
+  v5 = _mm_sub_epi16(u4, u5);
+  v6 = _mm_sub_epi16(u7, u6);
+  v7 = _mm_add_epi16(u7, u6);
 
-  const __m128i stk2_4 = pair_set_epi16(2 * cospi_22_64, 2 * cospi_22_64);
-  const __m128i stk2_5 = pair_set_epi16(2 * cospi_10_64, 2 * cospi_10_64);
-  const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
-  const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
+  {
+    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
+    const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
+    butterfly(v6, v5, stg4_1, stg4_0, &v5, &v6);
+  }
 
-  const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
-  const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
-  const __m128i stk3_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64);
-  const __m128i stk3_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64);
+  out[0] = _mm_add_epi16(v0, v7);
+  out[1] = _mm_add_epi16(v1, v6);
+  out[2] = _mm_add_epi16(v2, v5);
+  out[3] = _mm_add_epi16(v3, v4);
+  out[4] = _mm_sub_epi16(v3, v4);
+  out[5] = _mm_sub_epi16(v2, v5);
+  out[6] = _mm_sub_epi16(v1, v6);
+  out[7] = _mm_sub_epi16(v0, v7);
+}
 
-  const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
-  const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
-  const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
-  const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
-  const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
-  const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
+static void idct32_8x32_135_quarter_2(const __m128i in[16], __m128i out[8]) {
+  __m128i u8, u9, u10, u11, u12, u13, u14, u15;
+  __m128i v8, v9, v10, v11, v12, v13, v14, v15;
 
-  const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
-  const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-  const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
-  const __m128i stk4_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64);
-  const __m128i stk4_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64);
+  {
+    const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
+    const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
+    const __m128i stk2_2 = pair_set_epi16(-2 * cospi_18_64, -2 * cospi_18_64);
+    const __m128i stk2_3 = pair_set_epi16(2 * cospi_14_64, 2 * cospi_14_64);
+    const __m128i stk2_4 = pair_set_epi16(2 * cospi_22_64, 2 * cospi_22_64);
+    const __m128i stk2_5 = pair_set_epi16(2 * cospi_10_64, 2 * cospi_10_64);
+    const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
+    const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
+    u8 = _mm_mulhrs_epi16(in[2], stk2_0);
+    u15 = _mm_mulhrs_epi16(in[2], stk2_1);
+    u9 = _mm_mulhrs_epi16(in[14], stk2_2);
+    u14 = _mm_mulhrs_epi16(in[14], stk2_3);
+    u10 = _mm_mulhrs_epi16(in[10], stk2_4);
+    u13 = _mm_mulhrs_epi16(in[10], stk2_5);
+    u11 = _mm_mulhrs_epi16(in[6], stk2_6);
+    u12 = _mm_mulhrs_epi16(in[6], stk2_7);
+  }
 
-  const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
-  const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+  v8 = _mm_add_epi16(u8, u9);
+  v9 = _mm_sub_epi16(u8, u9);
+  v10 = _mm_sub_epi16(u11, u10);
+  v11 = _mm_add_epi16(u11, u10);
+  v12 = _mm_add_epi16(u12, u13);
+  v13 = _mm_sub_epi16(u12, u13);
+  v14 = _mm_sub_epi16(u15, u14);
+  v15 = _mm_add_epi16(u15, u14);
 
-  const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
-  __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7,
-      stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15,
-      stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23,
-      stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31;
-  __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
-      stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15,
-      stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23,
-      stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31;
-  __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+  {
+    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
+    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+    butterfly_self(&v9, &v14, stg4_4, stg4_5);
+    butterfly_self(&v10, &v13, stg4_6, stg4_4);
+  }
 
-  /* Stage1 */
-  stp1_16 = _mm_mulhrs_epi16(in[1], stk1_0);
-  stp1_31 = _mm_mulhrs_epi16(in[1], stk1_1);
-  stp1_17 = _mm_mulhrs_epi16(in[15], stk1_2);
-  stp1_30 = _mm_mulhrs_epi16(in[15], stk1_3);
+  out[0] = _mm_add_epi16(v8, v11);
+  out[1] = _mm_add_epi16(v9, v10);
+  out[2] = _mm_sub_epi16(v9, v10);
+  out[3] = _mm_sub_epi16(v8, v11);
+  out[4] = _mm_sub_epi16(v15, v12);
+  out[5] = _mm_sub_epi16(v14, v13);
+  out[6] = _mm_add_epi16(v14, v13);
+  out[7] = _mm_add_epi16(v15, v12);
 
-  stp1_18 = _mm_mulhrs_epi16(in[9], stk1_4);
-  stp1_29 = _mm_mulhrs_epi16(in[9], stk1_5);
-  stp1_19 = _mm_mulhrs_epi16(in[7], stk1_6);
-  stp1_28 = _mm_mulhrs_epi16(in[7], stk1_7);
+  {
+    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
+    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+    butterfly_self(&out[2], &out[5], stg6_0, stg4_0);
+    butterfly_self(&out[3], &out[4], stg6_0, stg4_0);
+  }
+}
 
-  stp1_20 = _mm_mulhrs_epi16(in[5], stk1_8);
-  stp1_27 = _mm_mulhrs_epi16(in[5], stk1_9);
-  stp1_21 = _mm_mulhrs_epi16(in[11], stk1_10);
-  stp1_26 = _mm_mulhrs_epi16(in[11], stk1_11);
+// 8x32 block even indexed 8 inputs of in[16],
+// output first half 16 to out[32]
+static void idct32_8x32_quarter_1_2(const __m128i in[16], __m128i out[32]) {
+  __m128i temp[16];
+  idct32_8x32_135_quarter_1(in, temp);
+  idct32_8x32_135_quarter_2(in, &temp[8]);
+  add_sub_butterfly(temp, out, 16);
+}
 
-  stp1_22 = _mm_mulhrs_epi16(in[13], stk1_12);
-  stp1_25 = _mm_mulhrs_epi16(in[13], stk1_13);
-  stp1_23 = _mm_mulhrs_epi16(in[3], stk1_14);
-  stp1_24 = _mm_mulhrs_epi16(in[3], stk1_15);
+// 8x32 block odd indexed 8 inputs of in[16],
+// output second half 16 to out[32]
+static void idct32_8x32_quarter_3_4(const __m128i in[16], __m128i out[32]) {
+  __m128i v16, v17, v18, v19, v20, v21, v22, v23;
+  __m128i v24, v25, v26, v27, v28, v29, v30, v31;
+  __m128i u16, u17, u18, u19, u20, u21, u22, u23;
+  __m128i u24, u25, u26, u27, u28, u29, u30, u31;
 
-  /* Stage2 */
-  stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0);
-  stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1);
-  stp2_9 = _mm_mulhrs_epi16(in[14], stk2_2);
-  stp2_14 = _mm_mulhrs_epi16(in[14], stk2_3);
-
-  stp2_10 = _mm_mulhrs_epi16(in[10], stk2_4);
-  stp2_13 = _mm_mulhrs_epi16(in[10], stk2_5);
-  stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6);
-  stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7);
-
-  stp2_16 = _mm_add_epi16(stp1_16, stp1_17);
-  stp2_17 = _mm_sub_epi16(stp1_16, stp1_17);
-  stp2_18 = _mm_sub_epi16(stp1_19, stp1_18);
-  stp2_19 = _mm_add_epi16(stp1_19, stp1_18);
-
-  stp2_20 = _mm_add_epi16(stp1_20, stp1_21);
-  stp2_21 = _mm_sub_epi16(stp1_20, stp1_21);
-  stp2_22 = _mm_sub_epi16(stp1_23, stp1_22);
-  stp2_23 = _mm_add_epi16(stp1_23, stp1_22);
-
-  stp2_24 = _mm_add_epi16(stp1_24, stp1_25);
-  stp2_25 = _mm_sub_epi16(stp1_24, stp1_25);
-  stp2_26 = _mm_sub_epi16(stp1_27, stp1_26);
-  stp2_27 = _mm_add_epi16(stp1_27, stp1_26);
-
-  stp2_28 = _mm_add_epi16(stp1_28, stp1_29);
-  stp2_29 = _mm_sub_epi16(stp1_28, stp1_29);
-  stp2_30 = _mm_sub_epi16(stp1_31, stp1_30);
-  stp2_31 = _mm_add_epi16(stp1_31, stp1_30);
-
-  /* Stage3 */
   {
-    const __m128i lo_17_30 = _mm_unpacklo_epi16(stp2_17, stp2_30);
-    const __m128i hi_17_30 = _mm_unpackhi_epi16(stp2_17, stp2_30);
-    const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
-    const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
+    const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
+    const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
+    const __m128i stk1_2 = pair_set_epi16(-2 * cospi_17_64, -2 * cospi_17_64);
+    const __m128i stk1_3 = pair_set_epi16(2 * cospi_15_64, 2 * cospi_15_64);
 
-    const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
-    const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
-    const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
-    const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
+    const __m128i stk1_4 = pair_set_epi16(2 * cospi_23_64, 2 * cospi_23_64);
+    const __m128i stk1_5 = pair_set_epi16(2 * cospi_9_64, 2 * cospi_9_64);
+    const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
+    const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
+    const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
+    const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
+    const __m128i stk1_10 = pair_set_epi16(-2 * cospi_21_64, -2 * cospi_21_64);
+    const __m128i stk1_11 = pair_set_epi16(2 * cospi_11_64, 2 * cospi_11_64);
 
-    stp1_4 = _mm_mulhrs_epi16(in[4], stk3_0);
-    stp1_7 = _mm_mulhrs_epi16(in[4], stk3_1);
-    stp1_5 = _mm_mulhrs_epi16(in[12], stk3_2);
-    stp1_6 = _mm_mulhrs_epi16(in[12], stk3_3);
+    const __m128i stk1_12 = pair_set_epi16(2 * cospi_19_64, 2 * cospi_19_64);
+    const __m128i stk1_13 = pair_set_epi16(2 * cospi_13_64, 2 * cospi_13_64);
+    const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
+    const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
+    u16 = _mm_mulhrs_epi16(in[1], stk1_0);
+    u31 = _mm_mulhrs_epi16(in[1], stk1_1);
+    u17 = _mm_mulhrs_epi16(in[15], stk1_2);
+    u30 = _mm_mulhrs_epi16(in[15], stk1_3);
 
-    stp2_0 = _mm_mulhrs_epi16(in[0], stk4_0);
-    stp2_1 = _mm_mulhrs_epi16(in[0], stk4_0);  // stk4_1 = stk4_0
-    stp2_2 = _mm_mulhrs_epi16(in[8], stk4_2);
-    stp2_3 = _mm_mulhrs_epi16(in[8], stk4_3);
+    u18 = _mm_mulhrs_epi16(in[9], stk1_4);
+    u29 = _mm_mulhrs_epi16(in[9], stk1_5);
+    u19 = _mm_mulhrs_epi16(in[7], stk1_6);
+    u28 = _mm_mulhrs_epi16(in[7], stk1_7);
 
-    stp1_8 = _mm_add_epi16(stp2_8, stp2_9);
-    stp1_9 = _mm_sub_epi16(stp2_8, stp2_9);
-    stp1_10 = _mm_sub_epi16(stp2_11, stp2_10);
-    stp1_11 = _mm_add_epi16(stp2_11, stp2_10);
-    stp1_12 = _mm_add_epi16(stp2_12, stp2_13);
-    stp1_13 = _mm_sub_epi16(stp2_12, stp2_13);
-    stp1_14 = _mm_sub_epi16(stp2_15, stp2_14);
-    stp1_15 = _mm_add_epi16(stp2_15, stp2_14);
+    u20 = _mm_mulhrs_epi16(in[5], stk1_8);
+    u27 = _mm_mulhrs_epi16(in[5], stk1_9);
+    u21 = _mm_mulhrs_epi16(in[11], stk1_10);
+    u26 = _mm_mulhrs_epi16(in[11], stk1_11);
 
-    MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4,
-                           stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18,
-                           stp1_29)
-    MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8,
-                           stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22,
-                           stp1_25)
-
-    stp1_16 = stp2_16;
-    stp1_31 = stp2_31;
-    stp1_19 = stp2_19;
-    stp1_20 = stp2_20;
-    stp1_23 = stp2_23;
-    stp1_24 = stp2_24;
-    stp1_27 = stp2_27;
-    stp1_28 = stp2_28;
+    u22 = _mm_mulhrs_epi16(in[13], stk1_12);
+    u25 = _mm_mulhrs_epi16(in[13], stk1_13);
+    u23 = _mm_mulhrs_epi16(in[3], stk1_14);
+    u24 = _mm_mulhrs_epi16(in[3], stk1_15);
   }
 
-  /* Stage4 */
-  {
-    const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14);
-    const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14);
-    const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13);
-    const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13);
+  v16 = _mm_add_epi16(u16, u17);
+  v17 = _mm_sub_epi16(u16, u17);
+  v18 = _mm_sub_epi16(u19, u18);
+  v19 = _mm_add_epi16(u19, u18);
 
-    stp2_4 = _mm_add_epi16(stp1_4, stp1_5);
-    stp2_5 = _mm_sub_epi16(stp1_4, stp1_5);
-    stp2_6 = _mm_sub_epi16(stp1_7, stp1_6);
-    stp2_7 = _mm_add_epi16(stp1_7, stp1_6);
+  v20 = _mm_add_epi16(u20, u21);
+  v21 = _mm_sub_epi16(u20, u21);
+  v22 = _mm_sub_epi16(u23, u22);
+  v23 = _mm_add_epi16(u23, u22);
 
-    MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5,
-                           stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13)
+  v24 = _mm_add_epi16(u24, u25);
+  v25 = _mm_sub_epi16(u24, u25);
+  v26 = _mm_sub_epi16(u27, u26);
+  v27 = _mm_add_epi16(u27, u26);
 
-    stp2_8 = stp1_8;
-    stp2_15 = stp1_15;
-    stp2_11 = stp1_11;
-    stp2_12 = stp1_12;
+  v28 = _mm_add_epi16(u28, u29);
+  v29 = _mm_sub_epi16(u28, u29);
+  v30 = _mm_sub_epi16(u31, u30);
+  v31 = _mm_add_epi16(u31, u30);
 
-    stp2_16 = _mm_add_epi16(stp1_16, stp1_19);
-    stp2_17 = _mm_add_epi16(stp1_17, stp1_18);
-    stp2_18 = _mm_sub_epi16(stp1_17, stp1_18);
-    stp2_19 = _mm_sub_epi16(stp1_16, stp1_19);
-    stp2_20 = _mm_sub_epi16(stp1_23, stp1_20);
-    stp2_21 = _mm_sub_epi16(stp1_22, stp1_21);
-    stp2_22 = _mm_add_epi16(stp1_22, stp1_21);
-    stp2_23 = _mm_add_epi16(stp1_23, stp1_20);
-
-    stp2_24 = _mm_add_epi16(stp1_24, stp1_27);
-    stp2_25 = _mm_add_epi16(stp1_25, stp1_26);
-    stp2_26 = _mm_sub_epi16(stp1_25, stp1_26);
-    stp2_27 = _mm_sub_epi16(stp1_24, stp1_27);
-    stp2_28 = _mm_sub_epi16(stp1_31, stp1_28);
-    stp2_29 = _mm_sub_epi16(stp1_30, stp1_29);
-    stp2_30 = _mm_add_epi16(stp1_29, stp1_30);
-    stp2_31 = _mm_add_epi16(stp1_28, stp1_31);
-  }
-
-  /* Stage5 */
   {
-    const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5);
-    const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5);
-    const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
-    const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
+    const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
+    const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
+    const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
+    const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
+    const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
+    const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
 
-    const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28);
-    const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28);
-    const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
-    const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
-
-    const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
-    const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
-
-    stp1_0 = _mm_add_epi16(stp2_0, stp2_3);
-    stp1_1 = _mm_add_epi16(stp2_1, stp2_2);
-    stp1_2 = _mm_sub_epi16(stp2_1, stp2_2);
-    stp1_3 = _mm_sub_epi16(stp2_0, stp2_3);
-
-    tmp0 = _mm_madd_epi16(lo_6_5, stg4_1);
-    tmp1 = _mm_madd_epi16(hi_6_5, stg4_1);
-    tmp2 = _mm_madd_epi16(lo_6_5, stg4_0);
-    tmp3 = _mm_madd_epi16(hi_6_5, stg4_0);
-
-    tmp0 = _mm_add_epi32(tmp0, rounding);
-    tmp1 = _mm_add_epi32(tmp1, rounding);
-    tmp2 = _mm_add_epi32(tmp2, rounding);
-    tmp3 = _mm_add_epi32(tmp3, rounding);
-
-    tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
-    tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
-    tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
-    tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
-
-    stp1_5 = _mm_packs_epi32(tmp0, tmp1);
-    stp1_6 = _mm_packs_epi32(tmp2, tmp3);
-
-    stp1_4 = stp2_4;
-    stp1_7 = stp2_7;
-
-    stp1_8 = _mm_add_epi16(stp2_8, stp2_11);
-    stp1_9 = _mm_add_epi16(stp2_9, stp2_10);
-    stp1_10 = _mm_sub_epi16(stp2_9, stp2_10);
-    stp1_11 = _mm_sub_epi16(stp2_8, stp2_11);
-    stp1_12 = _mm_sub_epi16(stp2_15, stp2_12);
-    stp1_13 = _mm_sub_epi16(stp2_14, stp2_13);
-    stp1_14 = _mm_add_epi16(stp2_14, stp2_13);
-    stp1_15 = _mm_add_epi16(stp2_15, stp2_12);
-
-    stp1_16 = stp2_16;
-    stp1_17 = stp2_17;
-
-    MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4,
-                           stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19,
-                           stp1_28)
-    MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6,
-                           stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21,
-                           stp1_26)
-
-    stp1_22 = stp2_22;
-    stp1_23 = stp2_23;
-    stp1_24 = stp2_24;
-    stp1_25 = stp2_25;
-    stp1_30 = stp2_30;
-    stp1_31 = stp2_31;
+    butterfly_self(&v17, &v30, stg3_4, stg3_5);
+    butterfly_self(&v18, &v29, stg3_6, stg3_4);
+    butterfly_self(&v21, &v26, stg3_8, stg3_9);
+    butterfly_self(&v22, &v25, stg3_10, stg3_8);
   }
 
-  /* Stage6 */
-  {
-    const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13);
-    const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13);
-    const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12);
-    const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12);
+  u16 = _mm_add_epi16(v16, v19);
+  u17 = _mm_add_epi16(v17, v18);
+  u18 = _mm_sub_epi16(v17, v18);
+  u19 = _mm_sub_epi16(v16, v19);
+  u20 = _mm_sub_epi16(v23, v20);
+  u21 = _mm_sub_epi16(v22, v21);
+  u22 = _mm_add_epi16(v22, v21);
+  u23 = _mm_add_epi16(v23, v20);
 
-    stp2_0 = _mm_add_epi16(stp1_0, stp1_7);
-    stp2_1 = _mm_add_epi16(stp1_1, stp1_6);
-    stp2_2 = _mm_add_epi16(stp1_2, stp1_5);
-    stp2_3 = _mm_add_epi16(stp1_3, stp1_4);
-    stp2_4 = _mm_sub_epi16(stp1_3, stp1_4);
-    stp2_5 = _mm_sub_epi16(stp1_2, stp1_5);
-    stp2_6 = _mm_sub_epi16(stp1_1, stp1_6);
-    stp2_7 = _mm_sub_epi16(stp1_0, stp1_7);
+  u24 = _mm_add_epi16(v24, v27);
+  u25 = _mm_add_epi16(v25, v26);
+  u26 = _mm_sub_epi16(v25, v26);
+  u27 = _mm_sub_epi16(v24, v27);
+  u28 = _mm_sub_epi16(v31, v28);
+  u29 = _mm_sub_epi16(v30, v29);
+  u30 = _mm_add_epi16(v29, v30);
+  u31 = _mm_add_epi16(v28, v31);
 
-    stp2_8 = stp1_8;
-    stp2_9 = stp1_9;
-    stp2_14 = stp1_14;
-    stp2_15 = stp1_15;
-
-    MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0,
-                           stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11,
-                           stp2_12)
-
-    stp2_16 = _mm_add_epi16(stp1_16, stp1_23);
-    stp2_17 = _mm_add_epi16(stp1_17, stp1_22);
-    stp2_18 = _mm_add_epi16(stp1_18, stp1_21);
-    stp2_19 = _mm_add_epi16(stp1_19, stp1_20);
-    stp2_20 = _mm_sub_epi16(stp1_19, stp1_20);
-    stp2_21 = _mm_sub_epi16(stp1_18, stp1_21);
-    stp2_22 = _mm_sub_epi16(stp1_17, stp1_22);
-    stp2_23 = _mm_sub_epi16(stp1_16, stp1_23);
-
-    stp2_24 = _mm_sub_epi16(stp1_31, stp1_24);
-    stp2_25 = _mm_sub_epi16(stp1_30, stp1_25);
-    stp2_26 = _mm_sub_epi16(stp1_29, stp1_26);
-    stp2_27 = _mm_sub_epi16(stp1_28, stp1_27);
-    stp2_28 = _mm_add_epi16(stp1_27, stp1_28);
-    stp2_29 = _mm_add_epi16(stp1_26, stp1_29);
-    stp2_30 = _mm_add_epi16(stp1_25, stp1_30);
-    stp2_31 = _mm_add_epi16(stp1_24, stp1_31);
+  {
+    const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+    const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
+    const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+    butterfly_self(&u18, &u29, stg4_4, stg4_5);
+    butterfly_self(&u19, &u28, stg4_4, stg4_5);
+    butterfly_self(&u20, &u27, stg4_6, stg4_4);
+    butterfly_self(&u21, &u26, stg4_6, stg4_4);
   }
 
-  /* Stage7 */
-  {
-    const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
-    const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
-    const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
-    const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
+  out[0] = _mm_add_epi16(u16, u23);
+  out[1] = _mm_add_epi16(u17, u22);
+  out[2] = _mm_add_epi16(u18, u21);
+  out[3] = _mm_add_epi16(u19, u20);
+  v20 = _mm_sub_epi16(u19, u20);
+  v21 = _mm_sub_epi16(u18, u21);
+  v22 = _mm_sub_epi16(u17, u22);
+  v23 = _mm_sub_epi16(u16, u23);
 
-    const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
-    const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
-    const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24);
-    const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24);
+  v24 = _mm_sub_epi16(u31, u24);
+  v25 = _mm_sub_epi16(u30, u25);
+  v26 = _mm_sub_epi16(u29, u26);
+  v27 = _mm_sub_epi16(u28, u27);
+  out[12] = _mm_add_epi16(u27, u28);
+  out[13] = _mm_add_epi16(u26, u29);
+  out[14] = _mm_add_epi16(u25, u30);
+  out[15] = _mm_add_epi16(u24, u31);
 
-    stp1_0 = _mm_add_epi16(stp2_0, stp2_15);
-    stp1_1 = _mm_add_epi16(stp2_1, stp2_14);
-    stp1_2 = _mm_add_epi16(stp2_2, stp2_13);
-    stp1_3 = _mm_add_epi16(stp2_3, stp2_12);
-    stp1_4 = _mm_add_epi16(stp2_4, stp2_11);
-    stp1_5 = _mm_add_epi16(stp2_5, stp2_10);
-    stp1_6 = _mm_add_epi16(stp2_6, stp2_9);
-    stp1_7 = _mm_add_epi16(stp2_7, stp2_8);
-    stp1_8 = _mm_sub_epi16(stp2_7, stp2_8);
-    stp1_9 = _mm_sub_epi16(stp2_6, stp2_9);
-    stp1_10 = _mm_sub_epi16(stp2_5, stp2_10);
-    stp1_11 = _mm_sub_epi16(stp2_4, stp2_11);
-    stp1_12 = _mm_sub_epi16(stp2_3, stp2_12);
-    stp1_13 = _mm_sub_epi16(stp2_2, stp2_13);
-    stp1_14 = _mm_sub_epi16(stp2_1, stp2_14);
-    stp1_15 = _mm_sub_epi16(stp2_0, stp2_15);
-
-    stp1_16 = stp2_16;
-    stp1_17 = stp2_17;
-    stp1_18 = stp2_18;
-    stp1_19 = stp2_19;
-
-    MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0,
-                           stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21,
-                           stp1_26)
-    MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0,
-                           stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23,
-                           stp1_24)
-
-    stp1_28 = stp2_28;
-    stp1_29 = stp2_29;
-    stp1_30 = stp2_30;
-    stp1_31 = stp2_31;
+  {
+    const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
+    const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+    butterfly(v20, v27, stg6_0, stg4_0, &out[4], &out[11]);
+    butterfly(v21, v26, stg6_0, stg4_0, &out[5], &out[10]);
+    butterfly(v22, v25, stg6_0, stg4_0, &out[6], &out[9]);
+    butterfly(v23, v24, stg6_0, stg4_0, &out[7], &out[8]);
   }
+}
 
-  in[0] = _mm_add_epi16(stp1_0, stp1_31);
-  in[1] = _mm_add_epi16(stp1_1, stp1_30);
-  in[2] = _mm_add_epi16(stp1_2, stp1_29);
-  in[3] = _mm_add_epi16(stp1_3, stp1_28);
-  in[4] = _mm_add_epi16(stp1_4, stp1_27);
-  in[5] = _mm_add_epi16(stp1_5, stp1_26);
-  in[6] = _mm_add_epi16(stp1_6, stp1_25);
-  in[7] = _mm_add_epi16(stp1_7, stp1_24);
-  in[8] = _mm_add_epi16(stp1_8, stp1_23);
-  in[9] = _mm_add_epi16(stp1_9, stp1_22);
-  in[10] = _mm_add_epi16(stp1_10, stp1_21);
-  in[11] = _mm_add_epi16(stp1_11, stp1_20);
-  in[12] = _mm_add_epi16(stp1_12, stp1_19);
-  in[13] = _mm_add_epi16(stp1_13, stp1_18);
-  in[14] = _mm_add_epi16(stp1_14, stp1_17);
-  in[15] = _mm_add_epi16(stp1_15, stp1_16);
-  in[16] = _mm_sub_epi16(stp1_15, stp1_16);
-  in[17] = _mm_sub_epi16(stp1_14, stp1_17);
-  in[18] = _mm_sub_epi16(stp1_13, stp1_18);
-  in[19] = _mm_sub_epi16(stp1_12, stp1_19);
-  in[20] = _mm_sub_epi16(stp1_11, stp1_20);
-  in[21] = _mm_sub_epi16(stp1_10, stp1_21);
-  in[22] = _mm_sub_epi16(stp1_9, stp1_22);
-  in[23] = _mm_sub_epi16(stp1_8, stp1_23);
-  in[24] = _mm_sub_epi16(stp1_7, stp1_24);
-  in[25] = _mm_sub_epi16(stp1_6, stp1_25);
-  in[26] = _mm_sub_epi16(stp1_5, stp1_26);
-  in[27] = _mm_sub_epi16(stp1_4, stp1_27);
-  in[28] = _mm_sub_epi16(stp1_3, stp1_28);
-  in[29] = _mm_sub_epi16(stp1_2, stp1_29);
-  in[30] = _mm_sub_epi16(stp1_1, stp1_30);
-  in[31] = _mm_sub_epi16(stp1_0, stp1_31);
+// 8x16 block, input __m128i in[16], output __m128i in[32]
+static void idct32_8x32_135(__m128i in[32]) {
+  __m128i out[32];
+  idct32_8x32_quarter_1_2(in, out);
+  idct32_8x32_quarter_3_4(in, &out[16]);
+  add_sub_butterfly(out, in, 32);
 }
 
 static INLINE void store_buffer_8x32(__m128i *in, uint8_t *dst, int stride) {
@@ -1114,30 +974,55 @@
 }
 
 static INLINE void idct32_135(__m128i *col0, __m128i *col1) {
-  idct32_8x16_135(col0);
-  idct32_8x16_135(col1);
+  idct32_8x32_135(col0);
+  idct32_8x32_135(col1);
 }
 
+typedef enum { left_16, right_16 } ColsIndicator;
+
+static void transpose_and_copy_16x16(__m128i *in0, __m128i *in1, __m128i *store,
+                                     ColsIndicator cols) {
+  switch (cols) {
+    case left_16: {
+      int i;
+      array_transpose_16x16(in0, in1);
+      for (i = 0; i < 16; ++i) {
+        store[i] = in0[16 + i];
+        store[16 + i] = in1[16 + i];
+      }
+      break;
+    }
+    case right_16: {
+      array_transpose_16x16_2(store, &store[16], in0, in1);
+      break;
+    }
+    default: { assert(0); }
+  }
+}
+
 // Only upper-left 16x16 has non-zero coeff
 void vpx_idct32x32_135_add_ssse3(const tran_low_t *input, uint8_t *dest,
                                  int stride) {
-  __m128i col0[32], col1[32], col2[32], col3[32];
+  // Each array represents an 8x32 block
+  __m128i col0[32], col1[32];
+  // This array represents a 16x16 block
+  __m128i temp[32];
 
   // Load input data. Only need to load the top left 16x16 block.
-  load_buffer_16x16(input, col2, col3);
+  load_buffer_16x16(input, col0, col1);
 
   // columns
-  array_transpose_16x16_2(col2, col3, col0, col1);
+  array_transpose_16x16(col0, col1);
   idct32_135(col0, col1);
 
   // rows
-  array_transpose_16x16_2(col0, col1, col2, col3);
-  idct32_135(col2, col3);
-  recon_and_store(col2, col3, dest, stride);
+  transpose_and_copy_16x16(col0, col1, temp, left_16);
+  idct32_135(col0, col1);
+  recon_and_store(col0, col1, dest, stride);
 
-  array_transpose_16x16_2(&col0[16], &col1[16], col2, col3);
-  idct32_135(col2, col3);
-  recon_and_store(col2, col3, dest + 16, stride);
+  transpose_and_copy_16x16(col0, col1, temp, right_16);
+  idct32_135(col0, col1);
+  recon_and_store(col0, col1, dest + 16, stride);
 }
 
 // For each 8x32 block __m128i in[32], output __m128i in[32]