diff --git a/src/math/big/arith_s390x.s b/src/math/big/arith_s390x.s new file mode 100644 index 0000000000..a691970810 --- /dev/null +++ b/src/math/big/arith_s390x.s @@ -0,0 +1,565 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !math_big_pure_go,s390x + +#include "textflag.h" + +// This file provides fast assembly versions for the elementary +// arithmetic operations on vectors implemented in arith.go. + +TEXT ·mulWW(SB),NOSPLIT,$0 + MOVD x+0(FP), R3 + MOVD y+8(FP), R4 + MULHDU R3, R4 + MOVD R10, z1+16(FP) + MOVD R11, z0+24(FP) + RET + +// func divWW(x1, x0, y Word) (q, r Word) +TEXT ·divWW(SB),NOSPLIT,$0 + MOVD x1+0(FP), R10 + MOVD x0+8(FP), R11 + MOVD y+16(FP), R5 + WORD $0xb98700a5 // dlgr r10,r5 + MOVD R11, q+24(FP) + MOVD R10, r+32(FP) + RET + +// DI = R3, CX = R4, SI = r10, r8 = r8, r9=r9, r10 = r2 , r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + use R11 +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB),NOSPLIT,$0 + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD z+0(FP), R2 + + MOVD $0, R4 // c = 0 + MOVD $0, R0 // make sure it's zero + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 // n -= 4 + BLT v1 // if n < 0 goto v1 + +U1: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + ADDC R4, R4 // restore CF + MOVD 0(R9)(R10*1), R11 + ADDE R11, R5 + MOVD 8(R9)(R10*1), R11 + ADDE R11, R6 + MOVD 16(R9)(R10*1), R11 + ADDE R11, R7 + MOVD 24(R9)(R10*1), R11 + ADDE R11, R1 + MOVD R0, R4 + ADDE R4, R4 // save CF + NEG R4, R4 + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + + ADD $32, R10 // i += 4 + SUB $4, R3 // n -= 4 + BGE U1 // if n >= 0 goto U1 + +v1: ADD $4, R3 // n += 4 + BLE E1 // if n <= 0 goto E1 + +L1: // n > 0 + ADDC R4, R4 // restore CF + MOVD 0(R8)(R10*1), R5 + MOVD 0(R9)(R10*1), R11 + ADDE R11, R5 + MOVD R5, 0(R2)(R10*1) + MOVD R0, R4 + ADDE R4, R4 // save CF + NEG R4, R4 + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L1 // if n > 0 goto L1 + +E1: NEG R4, R4 + MOVD R4, c+72(FP) // return c + RET + +// DI = R3, CX = R4, SI = r10, r8 = r8, r9=r9, r10 = r2 , r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + use R11 +// func subVV(z, x, y []Word) (c Word) +// (same as addVV except for SUBC/SUBE instead of ADDC/ADDE and label names) +TEXT ·subVV(SB),NOSPLIT,$0 + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD z+0(FP), R2 + + MOVD $0, R4 // c = 0 + MOVD $0, R0 // make sure it's zero + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 // n -= 4 + BLT v1 // if n < 0 goto v1 + +U1: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + MOVD R0, R11 + SUBC R4, R11 // restore CF + MOVD 0(R9)(R10*1), R11 + SUBE R11, R5 + MOVD 8(R9)(R10*1), R11 + SUBE R11, R6 + MOVD 16(R9)(R10*1), R11 + SUBE R11, R7 + MOVD 24(R9)(R10*1), R11 + SUBE R11, R1 + MOVD R0, R4 + SUBE R4, R4 // save CF + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + + ADD $32, R10 // i += 4 + SUB $4, R3 // n -= 4 + BGE U1 // if n >= 0 goto U1 + +v1: ADD $4, R3 // n += 4 + BLE E1 // if n <= 0 goto E1 + +L1: // n > 0 + MOVD R0, R11 + SUBC R4, R11 // restore CF + MOVD 0(R8)(R10*1), R5 + MOVD 0(R9)(R10*1), R11 + SUBE R11, R5 + MOVD R5, 0(R2)(R10*1) + MOVD R0, R4 + SUBE R4, R4 // save CF + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L1 // if n > 0 goto L1 + +E1: NEG R4, R4 + MOVD R4, c+72(FP) // return c + RET + + +// func addVW(z, x []Word, y Word) (c Word) +TEXT ·addVW(SB),NOSPLIT,$0 +//DI = R3, CX = R4, SI = r10, r8 = r8, r10 = r2 , r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R4 // c = y + MOVD z+0(FP), R2 + MOVD $0, R0 // make sure it's 0 + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 // n -= 4 + BLT v4 // if n < 4 goto v4 + +U4: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + ADDC R4, R5 + ADDE R0, R6 + ADDE R0, R7 + ADDE R0, R1 + ADDE R0, R0 + MOVD R0, R4 // save CF + SUB R0, R0 + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + ADD $32, R10 // i += 4 -> i +=32 + SUB $4, R3 // n -= 4 + BGE U4 // if n >= 0 goto U4 + +v4: ADD $4, R3 // n += 4 + BLE E4 // if n <= 0 goto E4 + +L4: // n > 0 + MOVD 0(R8)(R10*1), R5 + ADDC R4, R5 + ADDE R0, R0 + MOVD R0, R4 // save CF + SUB R0, R0 + MOVD R5, 0(R2)(R10*1) + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L4 // if n > 0 goto L4 + +E4: MOVD R4, c+56(FP) // return c + + RET + +//DI = R3, CX = R4, SI = r10, r8 = r8, r10 = r2 , r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) +// func subVW(z, x []Word, y Word) (c Word) +// (same as addVW except for SUBC/SUBE instead of ADDC/ADDE and label names) +TEXT ·subVW(SB),NOSPLIT,$0 + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R4 // c = y + MOVD z+0(FP), R2 + MOVD $0, R0 // make sure it's 0 + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 // n -= 4 + BLT v4 // if n < 4 goto v4 + +U4: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + SUBC R4, R5 //SLGR -> SUBC + SUBE R0, R6 //SLBGR -> SUBE + SUBE R0, R7 + SUBE R0, R1 + SUBE R4, R4 // save CF + NEG R4, R4 + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + ADD $32, R10 // i += 4 -> i +=32 + SUB $4, R3 // n -= 4 + BGE U4 // if n >= 0 goto U4 + +v4: ADD $4, R3 // n += 4 + BLE E4 // if n <= 0 goto E4 + +L4: // n > 0 + MOVD 0(R8)(R10*1), R5 + SUBC R4, R5 + SUBE R4, R4 // save CF + NEG R4, R4 + MOVD R5, 0(R2)(R10*1) + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L4 // if n > 0 goto L4 + +E4: MOVD R4, c+56(FP) // return c + + RET + +// func shlVU(z, x []Word, s uint) (c Word) +TEXT ·shlVU(SB),NOSPLIT,$0 + MOVD z_len+8(FP), R5 + SUB $1, R5 // n-- + BLT X8b // n < 0 (n <= 0) + + // n > 0 + MOVD s+48(FP), R4 + CMPBEQ R0, R4, Z80 //handle 0 case beq + MOVD $64, R6 + CMPBEQ R6, R4, Z864 //handle 64 case beq + MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + SLD $3, R5 // n = n*8 + SUB R4, R6, R7 + MOVD (R8)(R5*1), R10 // w1 = x[i-1] + SRD R7, R10, R3 + MOVD R3, c+56(FP) + + MOVD $0, R1 // i = 0 + BR E8 + + // i < n-1 +L8: MOVD R10, R3 // w = w1 + MOVD -8(R8)(R5*1), R10 // w1 = x[i+1] + + SLD R4, R3 // w<>ŝ + SRD R7, R10, R6 + OR R6, R3 + MOVD R3, (R2)(R5*1) // z[i] = w<>ŝ + SUB $8, R5 // i-- + +E8: CMPBGT R5, R0, L8 // i < n-1 + + // i >= n-1 +X8a: SLD R4, R10 // w1<= n-1 + MOVD R10, (R2)(R5*1) + RET + +Z864: MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + SLD $3, R5 // n = n*8 + MOVD (R8)(R5*1), R3 // w1 = x[n-1] + MOVD R3, c+56(FP) // z[i] = x[n-1] + + BR E864 + + // i < n-1 +L864: MOVD -8(R8)(R5*1), R3 + + MOVD R3, (R2)(R5*1) // z[i] = x[n-1] + SUB $8, R5 // i-- + +E864: CMPBGT R5, R0, L864 // i < n-1 + + MOVD R0, (R2) // z[n-1] = 0 + RET + + +// CX = R4, r8 = r8, r10 = r2 , r11 = r5, DX = r3, AX = r10 , BX = R1 , 64-count = r7 (R0 set to 0) temp = R6 +// func shrVU(z, x []Word, s uint) (c Word) +TEXT ·shrVU(SB),NOSPLIT,$0 + MOVD z_len+8(FP), R5 + SUB $1, R5 // n-- + BLT X9b // n < 0 (n <= 0) + + // n > 0 + MOVD s+48(FP), R4 + CMPBEQ R0, R4, ZB0 //handle 0 case beq + MOVD $64, R6 + CMPBEQ R6, R4, ZB64 //handle 64 case beq + MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + SLD $3, R5 // n = n*8 + SUB R4, R6, R7 + MOVD (R8), R10 // w1 = x[0] + SLD R7, R10, R3 + MOVD R3, c+56(FP) + + MOVD $0, R1 // i = 0 + BR E9 + + // i < n-1 +L9: MOVD R10, R3 // w = w1 + MOVD 8(R8)(R1*1), R10 // w1 = x[i+1] + + SRD R4, R3 // w>>s | w1<>s | w1<= n-1 +X9a: SRD R4, R10 // w1>>s + MOVD R10, (R2)(R5*1) // z[n-1] = w1>>s + RET + +X9b: MOVD R0, c+56(FP) + RET + +ZB0: MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + SLD $3, R5 // n = n*8 + + MOVD (R8), R10 // w1 = x[0] + MOVD $0, R3 // R10 << 64 + MOVD R3, c+56(FP) + + MOVD $0, R1 // i = 0 + BR E9Z + + // i < n-1 +L9Z: MOVD R10, R3 // w = w1 + MOVD 8(R8)(R1*1), R10 // w1 = x[i+1] + + MOVD R3, (R2)(R1*1) // z[i] = w>>s | w1<= n-1 + MOVD R10, (R2)(R5*1) // z[n-1] = w1>>s + RET + +ZB64: MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + SLD $3, R5 // n = n*8 + MOVD (R8), R3 // w1 = x[0] + MOVD R3, c+56(FP) + + MOVD $0, R1 // i = 0 + BR E964 + + // i < n-1 +L964: MOVD 8(R8)(R1*1), R3 // w1 = x[i+1] + + MOVD R3, (R2)(R1*1) // z[i] = w>>s | w1<= n-1 + MOVD $0, R10 // w1>>s + MOVD R10, (R2)(R5*1) // z[n-1] = w1>>s + RET + +// CX = R4, r8 = r8, r9=r9, r10 = r2 , r11 = r5, DX = r3, AX = r6 , BX = R1 , (R0 set to 0) + use R11 + use R7 for i +// func mulAddVWW(z, x []Word, y, r Word) (c Word) +TEXT ·mulAddVWW(SB),NOSPLIT,$0 + MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD r+56(FP), R4 // c = r + MOVD z_len+8(FP), R5 + MOVD $0, R1 // i = 0 + MOVD $0, R7 // i*8 = 0 + MOVD $0, R0 // make sure it's zero + BR E5 + +L5: MOVD (R8)(R1*1), R6 + MULHDU R9, R6 + ADDC R4, R11 //add to low order bits + ADDE R0, R6 + MOVD R11, (R2)(R1*1) + MOVD R6, R4 + ADD $8, R1 // i*8 + 8 + ADD $1, R7 // i++ + +E5: CMPBLT R7, R5, L5 // i < n + + MOVD R4, c+64(FP) + RET + +// func addMulVVW(z, x []Word, y Word) (c Word) +// CX = R4, r8 = r8, r9=r9, r10 = r2 , r11 = r5, AX = r11, DX = R6, r12=r12, BX = R1 , (R0 set to 0) + use R11 + use R7 for i +TEXT ·addMulVVW(SB),NOSPLIT,$0 + MOVD z+0(FP), R2 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD z_len+8(FP), R5 + + MOVD $0, R1 // i*8 = 0 + MOVD $0, R7 // i = 0 + MOVD $0, R0 // make sure it's zero + MOVD $0, R4 // c = 0 + + MOVD R5, R12 + AND $-2, R12 + CMPBGE R5, $2, A6 + BR E6 + +A6: MOVD (R8)(R1*1), R6 + MULHDU R9, R6 + MOVD (R2)(R1*1), R10 + ADDC R10, R11 //add to low order bits + ADDE R0, R6 + ADDC R4, R11 + ADDE R0, R6 + MOVD R6, R4 + MOVD R11, (R2)(R1*1) + + MOVD (8)(R8)(R1*1), R6 + MULHDU R9, R6 + MOVD (8)(R2)(R1*1), R10 + ADDC R10, R11 //add to low order bits + ADDE R0, R6 + ADDC R4, R11 + ADDE R0, R6 + MOVD R6, R4 + MOVD R11, (8)(R2)(R1*1) + + ADD $16, R1 // i*8 + 8 + ADD $2, R7 // i++ + + CMPBLT R7, R12, A6 + BR E6 + +L6: MOVD (R8)(R1*1), R6 + MULHDU R9, R6 + MOVD (R2)(R1*1), R10 + ADDC R10, R11 //add to low order bits + ADDE R0, R6 + ADDC R4, R11 + ADDE R0, R6 + MOVD R6, R4 + MOVD R11, (R2)(R1*1) + + ADD $8, R1 // i*8 + 8 + ADD $1, R7 // i++ + +E6: CMPBLT R7, R5, L6 // i < n + + MOVD R4, c+56(FP) + RET + +// func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) +// CX = R4, r8 = r8, r9=r9, r10 = r2 , r11 = r5, AX = r11, DX = R6, r12=r12, BX = R1(*8) , (R0 set to 0) + use R11 + use R7 for i +TEXT ·divWVW(SB),NOSPLIT,$0 + MOVD z+0(FP), R2 + MOVD xn+24(FP), R10 // r = xn + MOVD x+32(FP), R8 + MOVD y+56(FP), R9 + MOVD z_len+8(FP), R7 // i = z + SLD $3, R7, R1 // i*8 + MOVD $0, R0 // make sure it's zero + BR E7 + +L7: MOVD (R8)(R1*1), R11 + WORD $0xB98700A9 //DLGR R10,R9 + MOVD R11, (R2)(R1*1) + +E7: SUB $1, R7 // i-- + SUB $8, R1 + BGE L7 // i >= 0 + + MOVD R10, r+64(FP) + RET + +// func bitLen(x Word) (n int) +TEXT ·bitLen(SB),NOSPLIT,$0 + MOVD x+0(FP), R2 + WORD $0xb9830022 // FLOGR R2,R2 + MOVD $64, R3 + SUB R2, R3 + MOVD R3, n+8(FP) + RET