go/src/runtime/memclr_arm64.s

// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "textflag.h"

// void runtime·memclrNoHeapPointers(void*, uintptr)
TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16
	MOVD	ptr+0(FP), R0
	MOVD	n+8(FP), R1

	CMP	$16, R1
	// If n is equal to 16 bytes, use zero_exact_16 to zero
	BEQ	zero_exact_16

	// If n is greater than 16 bytes, use zero_by_16 to zero
	BHI	zero_by_16

	// n is less than 16 bytes
	ADD	R1, R0, R7
	TBZ	$3, R1, less_than_8
	MOVD	ZR, (R0)
	MOVD	ZR, -8(R7)
	RET

less_than_8:
	TBZ	$2, R1, less_than_4
	MOVW	ZR, (R0)
	MOVW	ZR, -4(R7)
	RET

less_than_4:
	CBZ	R1, ending
	MOVB	ZR, (R0)
	TBZ	$1, R1, ending
	MOVH	ZR, -2(R7)

ending:
	RET

zero_exact_16:
	// n is exactly 16 bytes
	STP	(ZR, ZR), (R0)
	RET

zero_by_16:
	// n greater than 16 bytes, check if the start address is aligned
	NEG	R0, R4
	ANDS	$15, R4, R4
	// Try zeroing using zva if the start address is aligned with 16
	BEQ	try_zva

	// Non-aligned store
	STP	(ZR, ZR), (R0)
	// Make the destination aligned
	SUB	R4, R1, R1
	ADD	R4, R0, R0
	B	try_zva

tail_maybe_long:
	CMP	$64, R1
	BHS	no_zva

tail63:
	ANDS	$48, R1, R3
	BEQ	last16
	CMPW	$32, R3
	BEQ	last48
	BLT	last32
	STP.P	(ZR, ZR), 16(R0)
last48:
	STP.P	(ZR, ZR), 16(R0)
last32:
	STP.P	(ZR, ZR), 16(R0)
	// The last store length is at most 16, so it is safe to use
	// stp to write last 16 bytes
last16:
	ANDS	$15, R1, R1
	CBZ	R1, last_end
	ADD	R1, R0, R0
	STP	(ZR, ZR), -16(R0)
last_end:
	RET

no_zva:
	SUB	$16, R0, R0
	SUB	$64, R1, R1

loop_64:
	STP	(ZR, ZR), 16(R0)
	STP	(ZR, ZR), 32(R0)
	STP	(ZR, ZR), 48(R0)
	STP.W	(ZR, ZR), 64(R0)
	SUBS	$64, R1, R1
	BGE	loop_64
	ANDS	$63, R1, ZR
	ADD	$16, R0, R0
	BNE	tail63
	RET

try_zva:
	// Try using the ZVA feature to zero entire cache lines
	// It is not meaningful to use ZVA if the block size is less than 64,
	// so make sure that n is greater than or equal to 64
	CMP	$63, R1
	BLE	tail63

	CMP	$128, R1
	// Ensure n is at least 128 bytes, so that there is enough to copy after
	// alignment.
	BLT	no_zva
	// Check if ZVA is allowed from user code, and if so get the block size
	MOVW	block_size<>(SB), R5
	TBNZ	$31, R5, no_zva
	CBNZ	R5, zero_by_line
        // DCZID_EL0 bit assignments
        // [63:5] Reserved
        // [4]    DZP, if bit set DC ZVA instruction is prohibited, else permitted
        // [3:0]  log2 of the block size in words, eg. if it returns 0x4 then block size is 16 words
	MRS	DCZID_EL0, R3
	TBZ	$4, R3, init
	// ZVA not available
	MOVW	$~0, R5
	MOVW	R5, block_size<>(SB)
	B	no_zva

init:
	MOVW	$4, R9
	ANDW	$15, R3, R5
	LSLW	R5, R9, R5
	MOVW	R5, block_size<>(SB)

	ANDS	$63, R5, R9
	// Block size is less than 64.
	BNE	no_zva

zero_by_line:
	CMP	R5, R1
	// Not enough memory to reach alignment
	BLO	no_zva
	SUB	$1, R5, R6
	NEG	R0, R4
	ANDS	R6, R4, R4
	// Already aligned
	BEQ	aligned

	// check there is enough to copy after alignment
	SUB	R4, R1, R3

	// Check that the remaining length to ZVA after alignment
	// is greater than 64.
	CMP	$64, R3
	CCMP	GE, R3, R5, $10  // condition code GE, NZCV=0b1010
	BLT	no_zva

	// We now have at least 64 bytes to zero, update n
	MOVD	R3, R1

loop_zva_prolog:
	STP	(ZR, ZR), (R0)
	STP	(ZR, ZR), 16(R0)
	STP	(ZR, ZR), 32(R0)
	SUBS	$64, R4, R4
	STP	(ZR, ZR), 48(R0)
	ADD	$64, R0, R0
	BGE	loop_zva_prolog

	ADD	R4, R0, R0

aligned:
	SUB	R5, R1, R1

loop_zva:
	WORD	$0xd50b7420 // DC ZVA, R0
	ADD	R5, R0, R0
	SUBS	R5, R1, R1
	BHS	loop_zva
	ANDS	R6, R1, R1
	BNE	tail_maybe_long
	RET

GLOBL block_size<>(SB), NOPTR, $8