cache.s   [plain text]

/*
 * Copyright (c) 2011-2017 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <mach/arm/syscall_sw.h>

#define MMU_I_CLINE	6		// cache line size as 1<<MMU_I_CLINE (64)

/* void sys_icache_invalidate(void *start, size_t length) */
.globl	_sys_icache_invalidate
.p2align	2
_sys_icache_invalidate:
	// see InvalidatePoU_IcacheRegion() in xnu/osfmk/arm64/caches_asm.s
	cbz		x1, 2f							// length > 0 ?
	and		x8, x0, #~((1<<MMU_I_CLINE)-1)	// cacheline align address
	and		x9, x0, #((1<<MMU_I_CLINE)-1)	// extend length by alignment
	add		x9, x1, x9
	sub		x9, x9, #1
	mov		x10, #-1
	eor		x9, x10, x9, lsr #MMU_I_CLINE	// compute cacheline counter
1:
	ic		ivau, x8						// invalidate icache line
	add		x8, x8, #1<<MMU_I_CLINE			// next cacheline address
	add		x9, x9, #1						// decrement cacheline counter
	cbnz	x9, 1b
	dsb		ish
	isb
2:
	ret

/* void sys_dcache_flush(void *start, size_t length) */
.globl	_sys_dcache_flush
.p2align	2
_sys_dcache_flush:
	// see FlushPoC_DcacheRegion() in xnu/osfmk/arm64/caches_asm.s
	dsb		ish								// noop, we are fully coherent
	ret

#if 0
// Above generated by clang from:
static void __attribute((used))
sys_icache_invalidate(uintptr_t start, size_t length)
{
	if (!length) return;
	uintptr_t addr = start & ~((1 << MMU_I_CLINE) - 1);
	length += start & ((1 << MMU_I_CLINE) - 1);
	size_t count = ((length - 1) >> MMU_I_CLINE) + 1;
	while (count--) {
		asm("ic ivau, %[addr]" :: [addr] "r" (addr) : "memory");
		addr += (1 << MMU_I_CLINE);
	}
	asm volatile("dsb ish\n\tisb" ::: "memory");
}
#endif