ref: 9461ae8388df869acfeeaa4fc26b260711a9959b
dir: /codec/encoder/core/x86/matrix_transpose.asm/
;*!
;* \copy
;* Copyright (c) 2009-2013, Cisco Systems
;* All rights reserved.
;*
;* Redistribution and use in source and binary forms, with or without
;* modification, are permitted provided that the following conditions
;* are met:
;*
;* ?Redistributions of source code must retain the above copyright
;* notice, this list of conditions and the following disclaimer.
;*
;* ?Redistributions in binary form must reproduce the above copyright
;* notice, this list of conditions and the following disclaimer in
;* the documentation and/or other materials provided with the
;* distribution.
;*
;* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
;* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
;* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
;* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
;* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
;* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
;* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
;* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
;* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
;* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
;* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
;* POSSIBILITY OF SUCH DAMAGE.
;*************************************************************************/
%include "asm_inc.asm"
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m0, m3, m5, m2, m7, m1, m6, m4
%macro TRANSPOSE_8x8B_MMX 10
MMX_XSwap bw, %1, %2, %8
MMX_XSwap bw, %3, %4, %2
MMX_XSwap bw, %5, %6, %4
movq %6, %9
movq %10, %4
MMX_XSwap bw, %7, %6, %4
MMX_XSwap wd, %1, %3, %6
MMX_XSwap wd, %8, %2, %3
MMX_XSwap wd, %5, %7, %2
movq %7, %10
movq %10, %3
MMX_XSwap wd, %7, %4, %3
MMX_XSwap dq, %1, %5, %4
MMX_XSwap dq, %6, %2, %5
MMX_XSwap dq, %8, %7, %2
movq %7, %10
movq %10, %5
MMX_XSwap dq, %7, %3, %5
movq %3, %10
%endmacro
;in: m0, m3, m5, m2, m7, m1, m6, m4
%macro TRANSPOSE8x8_WRITE_MMX 2 ; dst, dst_stride
movq [%1], mm0 ; result of line 1, x8 bytes
movq [%1+%2], mm3 ; result of line 2
lea %1, [%1+2*%2]
movq [%1], mm5 ; result of line 3
movq [%1+%2], mm2 ; result of line 4
lea %1, [%1+2*%2]
movq [%1], mm7 ; result of line 5
movq [%1+%2], mm1 ; result of line 6
lea %1, [%1+2*%2]
movq [%1], mm6 ; result of line 7
movq [%1+%2], mm4 ; result of line 8
%endmacro
;in: m0, m3, m5, m2, m7, m1, m6, m4
%macro TRANSPOSE8x8_WRITE_ALT_MMX 3 ; dst, dst_stride, reg32
movq [%1], mm0 ; result of line 1, x8 bytes
movq [%1+%2], mm3 ; result of line 2
lea %3, [%1+2*%2]
movq [%3], mm5 ; result of line 3
movq [%3+%2], mm2 ; result of line 4
lea %3, [%3+2*%2]
movq [%3], mm7 ; result of line 5
movq [%3+%2], mm1 ; result of line 6
lea %3, [%3+2*%2]
movq [%3], mm6 ; result of line 7
movq [%3+%2], mm4 ; result of line 8
%endmacro ; end of TRANSPOSE8x8_WRITE_ALT_MMX
; for transpose 16x8
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m4, m2, m3, m7, m5, m1, m6, m0
%macro TRANSPOSE_8x16B_SSE2 10
SSE2_XSawp bw, %1, %2, %8
SSE2_XSawp bw, %3, %4, %2
SSE2_XSawp bw, %5, %6, %4
movdqa %6, %9
movdqa %10, %4
SSE2_XSawp bw, %7, %6, %4
SSE2_XSawp wd, %1, %3, %6
SSE2_XSawp wd, %8, %2, %3
SSE2_XSawp wd, %5, %7, %2
movdqa %7, %10
movdqa %10, %3
SSE2_XSawp wd, %7, %4, %3
SSE2_XSawp dq, %1, %5, %4
SSE2_XSawp dq, %6, %2, %5
SSE2_XSawp dq, %8, %7, %2
movdqa %7, %10
movdqa %10, %5
SSE2_XSawp dq, %7, %3, %5
SSE2_XSawp qdq, %1, %8, %3
SSE2_XSawp qdq, %4, %2, %8
SSE2_XSawp qdq, %6, %7, %2
movdqa %7, %10
movdqa %10, %1
SSE2_XSawp qdq, %7, %5, %1
movdqa %5, %10
%endmacro ; end of TRANSPOSE_8x16B_SSE2
%macro TRANSPOSE8x16_WRITE_SSE2 2 ; dst, dst_stride
movq [%1], xmm4 ; result of line 1, x8 bytes
movq [%1+%2], xmm2 ; result of line 2
lea %1, [%1+2*%2]
movq [%1], xmm3 ; result of line 3
movq [%1+%2], xmm7 ; result of line 4
lea %1, [%1+2*%2]
movq [%1], xmm5 ; result of line 5
movq [%1+%2], xmm1 ; result of line 6
lea %1, [%1+2*%2]
movq [%1], xmm6 ; result of line 7
movq [%1+%2], xmm0 ; result of line 8
lea %1, [%1+2*%2]
movhpd [%1], xmm4 ; result of line 9
movhpd [%1+%2], xmm2 ; result of line 10
lea %1, [%1+2*%2]
movhpd [%1], xmm3 ; result of line 11
movhpd [%1+%2], xmm7 ; result of line 12
lea %1, [%1+2*%2]
movhpd [%1], xmm5 ; result of line 13
movhpd [%1+%2], xmm1 ; result of line 14
lea %1, [%1+2*%2]
movhpd [%1], xmm6 ; result of line 15
movhpd [%1+%2], xmm0 ; result of line 16
%endmacro ; end of TRANSPOSE_WRITE_RESULT_SSE2
%macro TRANSPOSE8x16_WRITE_ALT_SSE2 3 ; dst, dst_stride, reg32
movq [%1], xmm4 ; result of line 1, x8 bytes
movq [%1+%2], xmm2 ; result of line 2
lea %3, [%1+2*%2]
movq [%3], xmm3 ; result of line 3
movq [%3+%2], xmm7 ; result of line 4
lea %3, [%3+2*%2]
movq [%3], xmm5 ; result of line 5
movq [%3+%2], xmm1 ; result of line 6
lea %3, [%3+2*%2]
movq [%3], xmm6 ; result of line 7
movq [%3+%2], xmm0 ; result of line 8
lea %3, [%3+2*%2]
movhpd [%3], xmm4 ; result of line 9
movhpd [%3+%2], xmm2 ; result of line 10
lea %3, [%3+2*%2]
movhpd [%3], xmm3 ; result of line 11
movhpd [%3+%2], xmm7 ; result of line 12
lea %3, [%3+2*%2]
movhpd [%3], xmm5 ; result of line 13
movhpd [%3+%2], xmm1 ; result of line 14
lea %3, [%3+2*%2]
movhpd [%3], xmm6 ; result of line 15
movhpd [%3+%2], xmm0 ; result of line 16
%endmacro ; end of TRANSPOSE8x16_WRITE_ALT_SSE2
SECTION .text
WELS_EXTERN TransposeMatrixBlock16x16_sse2
; void TransposeMatrixBlock16x16_sse2( void *dst/*16x16*/, const int32_t dst_stride, void *src/*16x16*/, const int32_t src_stride );
push r4
push r5
%assign push_num 2
LOAD_4_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
mov r4, r7
and r4, 0Fh
sub r7, 10h
sub r7, r4
lea r5, [r3+r3*2]
; top 8x16 block
movdqa xmm0, [r2]
movdqa xmm1, [r2+r3]
movdqa xmm2, [r2+r3*2]
movdqa xmm3, [r2+r5]
lea r2, [r2+r3*4]
movdqa xmm4, [r2]
movdqa xmm5, [r2+r3]
movdqa xmm6, [r2+r3*2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m4, m2, m3, m7, m5, m1, m6, m0
TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r5], [r7]
TRANSPOSE8x16_WRITE_SSE2 r0, r1
; bottom 8x16 block
lea r2, [r2+r3*4]
movdqa xmm0, [r2]
movdqa xmm1, [r2+r3]
movdqa xmm2, [r2+r3*2]
movdqa xmm3, [r2+r5]
lea r2, [r2+r3*4]
movdqa xmm4, [r2]
movdqa xmm5, [r2+r3]
movdqa xmm6, [r2+r3*2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m4, m2, m3, m7, m5, m1, m6, m0
TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r5], [r7]
mov r5, r1
sal r5, 4
sub r0, r5
lea r0, [r0+r1*2+8]
TRANSPOSE8x16_WRITE_SSE2 r0, r1
add r7, r4
add r7, 10h
POP_XMM
LOAD_4_PARA_POP
pop r5
pop r4
ret
WELS_EXTERN TransposeMatrixBlocksx16_sse2
; void TransposeMatrixBlocksx16_sse2( void *dst/*W16x16*/, const int32_t dst_stride, void *src/*16xW16*/, const int32_t src_stride, const int32_t num_blocks );
push r5
push r6
%assign push_num 2
LOAD_5_PARA
PUSH_XMM 8
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
SIGN_EXTENSION r4, r4d
mov r5, r7
and r5, 0Fh
sub r7, 10h
sub r7, r5
TRANSPOSE_LOOP_SSE2:
; explictly loading next loop data
lea r6, [r2+r3*8]
push r4
%rep 8
mov r4, [r6]
mov r4, [r6+r3]
lea r6, [r6+r3*2]
%endrep
pop r4
; top 8x16 block
movdqa xmm0, [r2]
movdqa xmm1, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm2, [r2]
movdqa xmm3, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm4, [r2]
movdqa xmm5, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm6, [r2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m4, m2, m3, m7, m5, m1, m6, m0
TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r3], [r7]
TRANSPOSE8x16_WRITE_ALT_SSE2 r0, r1, r6
lea r2, [r2+r3*2]
; bottom 8x16 block
movdqa xmm0, [r2]
movdqa xmm1, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm2, [r2]
movdqa xmm3, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm4, [r2]
movdqa xmm5, [r2+r3]
lea r2, [r2+r3*2]
movdqa xmm6, [r2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m4, m2, m3, m7, m5, m1, m6, m0
TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r3], [r7]
TRANSPOSE8x16_WRITE_ALT_SSE2 r0+8, r1, r6
lea r2, [r2+r3*2]
lea r0, [r0+16]
dec r4
jg near TRANSPOSE_LOOP_SSE2
add r7, r5
add r7, 10h
POP_XMM
LOAD_5_PARA_POP
pop r6
pop r5
ret
WELS_EXTERN TransposeMatrixBlock8x8_mmx
; void TransposeMatrixBlock8x8_mmx( void *dst/*8x8*/, const int32_t dst_stride, void *src/*8x8*/, const int32_t src_stride );
%assign push_num 0
LOAD_4_PARA
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
sub r7, 8
movq mm0, [r2]
movq mm1, [r2+r3]
lea r2, [r2+2*r3]
movq mm2, [r2]
movq mm3, [r2+r3]
lea r2, [r2+2*r3]
movq mm4, [r2]
movq mm5, [r2+r3]
lea r2, [r2+2*r3]
movq mm6, [r2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m0, m3, m5, m2, m7, m1, m6, m4
TRANSPOSE_8x8B_MMX mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, [r2+r3], [r7]
TRANSPOSE8x8_WRITE_MMX r0, r1
emms
add r7, 8
LOAD_4_PARA_POP
ret
WELS_EXTERN TransposeMatrixBlocksx8_mmx
; void TransposeMatrixBlocksx8_mmx( void *dst/*8xW8*/, const int32_t dst_stride, void *src/*W8x8*/, const int32_t src_stride, const int32_t num_blocks );
push r5
push r6
%assign push_num 2
LOAD_5_PARA
SIGN_EXTENSION r1, r1d
SIGN_EXTENSION r3, r3d
SIGN_EXTENSION r4, r4d
sub r7, 8
lea r5, [r2+r3*8]
TRANSPOSE_BLOCKS_X8_LOOP_MMX:
; explictly loading next loop data
%rep 4
mov r6, [r5]
mov r6, [r5+r3]
lea r5, [r5+r3*2]
%endrep
movq mm0, [r2]
movq mm1, [r2+r3]
lea r2, [r2+2*r3]
movq mm2, [r2]
movq mm3, [r2+r3]
lea r2, [r2+2*r3]
movq mm4, [r2]
movq mm5, [r2+r3]
lea r2, [r2+2*r3]
movq mm6, [r2]
;in: m0, m1, m2, m3, m4, m5, m6, m7
;out: m0, m3, m5, m2, m7, m1, m6, m4
TRANSPOSE_8x8B_MMX mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, [r2+r3], [r7]
TRANSPOSE8x8_WRITE_ALT_MMX r0, r1, r6
lea r0, [r0+8]
lea r2, [r2+2*r3]
dec r4
jg near TRANSPOSE_BLOCKS_X8_LOOP_MMX
emms
add r7, 8
LOAD_5_PARA_POP
pop r6
pop r5
ret