/* * Copyright (c) 2003 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this * file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_LICENSE_HEADER_END@ */ /* ==================================== * Very Long Operand BCOPY for Mac OS X * ==================================== * * Version of 6/11/2003, tuned for the IBM 970. This is for operands at * least several pages long. It is called from bcopy()/memcpy()/memmove(). * * We use the following additional strategies not used by the shorter * operand paths. Mostly, we try to optimize for memory bandwidth: * 1. Use DCBZ128 to avoid reading destination lines. Because this code * resides on the commmpage, it can use a private interface with the * kernel to minimize alignment exceptions if the destination is * uncached. The kernel will clear cr7 whenever it emulates a DCBZ or * DCBZ128 on the commpage. Thus we take at most one exception per call, * which is amortized across the very long operand. * 2. Copy larger chunks per iteration to minimize R/W bus turnaround * and maximize DRAM page locality (opening a new page is expensive.) * 3. Touch in one source chunk ahead with DCBT. This is probably the * least important change, and probably only helps restart the * hardware stream at the start of each source page. * * Register usage. Note the rather delicate way we assign multiple uses * to the same register. Beware. * r0 = temp (NB: cannot use r0 for any constant such as "c16") * r3 = not used, as memcpy and memmove return 1st parameter as a value * r4 = source ptr ("rs") * r5 = count of bytes to move ("rc") * r6 = constant 16 ("c16") * r7 = constant 32 (""c32") * r8 = constant 48 (""c48") * r9 = constant 128 (""c128") * r10 = vrsave ("rv") * r11 = constant 256 (""c256") * r12 = destination ptr ("rd") * r13 = constant 384 (""c384") * r14 = temp ("rx") * r15 = temp ("rt") */ #define rs r4 #define rd r12 #define rc r5 #define rv r10 #define rx r14 #define rt r15 #define c16 r6 #define c32 r7 #define c48 r8 #define c128 r9 #define c256 r11 #define c384 r13 // Offsets within the "red zone" (which is 224 bytes long): #define rzR13 -8 #define rzR14 -12 #define rzR15 -16 #define rzV20 -32 #define rzV21 -48 #define rzV22 -64 #define rzV23 -80 #define rzV24 -96 #define rzV25 -112 #define rzV26 -128 #define rzV27 -144 #define rzV28 -160 #define rzV29 -176 #define rzV30 -192 #define rzV31 -208 #include <sys/appleapiopts.h> #include <ppc/asm.h> #include <machine/cpu_capabilities.h> #include <machine/commpage.h> .text .globl EXT(bigcopy_970) // Entry point. This is a subroutine of bcopy(). When called: // r4 = source ptr (aka "rs") // r12 = dest ptr (aka "rd") // r5 = length (>= 16K bytes) (aka "rc") // // We only do "forward" moves, ie non-overlapping or toward 0. // // We return with non-volatiles and r3 preserved. .align 5 bigcopy_970: stw r13,rzR13(r1) // spill non-volatile regs we use to redzone stw r14,rzR14(r1) stw r15,rzR15(r1) li r0,rzV20 neg rt,rd // start to cache-line-align destination stvx v20,r1,r0 // we use all 32 VRs li r0,rzV21 stvx v21,r1,r0 li r0,rzV22 stvx v22,r1,r0 li r0,rzV23 stvx v23,r1,r0 li r0,rzV24 andi. rt,rt,127 // get #bytes to 128-byte align stvx v24,r1,r0 li r0,rzV25 stvx v25,r1,r0 li r0,rzV26 sub rc,rc,rt // adjust length by #bytes to align destination stvx v26,r1,r0 li r0,rzV27 stvx v27,r1,r0 li r0,rzV28 mtctr rt // #bytes to align destination stvx v28,r1,r0 li r0,rzV29 stvx v29,r1,r0 li r0,rzV30 stvx v30,r1,r0 li r0,rzV31 stvx v31,r1,r0 beq 2f // dest already 128-byte aligned b 1f // Cache-line-align destination. .align 5 1: lbz r0,0(rs) addi rs,rs,1 stb r0,0(rd) addi rd,rd,1 bdnz 1b // Is source 16-byte aligned? Load constant offsets. 2: andi. r0,rs,15 // check source alignment mfspr rv,vrsave // save caller's bitmask li r0,-1 // we use all 32 VRs li c16,16 // load the constant offsets for x-form ops li c32,32 li c48,48 li c128,128 li c256,256 li c384,384 mtspr vrsave,r0 // NB: the kernel clears cr7 if it emulates a dcbz128 on the commpage, // and we dcbz only if cr7 beq is set. We check to be sure the dcbz's // won't zero source bytes before we load them, since we zero before // loading as this is faster than zeroing after loading and before storing. cmpw cr7,r0,r0 // initialize cr7 beq to use dcbz128 sub rt,rs,rd // get (rs-rd) cmplwi cr1,rt,512 // are we moving down less than 512 bytes? // Start fetching in source cache lines. dcbt c128,rs // first line already touched in dcbt c256,rs dcbt c384,rs bge++ cr1,3f // skip if not moving down less than 512 bytes cmpw cr7,c16,c32 // cannot dcbz since it would zero source bytes 3: beq LalignedLoop // handle aligned sources lvsl v0,0,rs // get permute vector for left shift lvxl v1,0,rs // prime the loop b LunalignedLoop // enter unaligned loop // Main loop for unaligned operands. We loop over 384-byte chunks (3 cache lines) // since we need a few VRs for permuted destination QWs and the permute vector. .align 5 LunalignedLoop: subi rc,rc,384 // decrement byte count addi rx,rs,384 // get address of next chunk lvxl v2,c16,rs lvxl v3,c32,rs bne-- cr7,1f // skip dcbz's if cr7 beq has been turned off by kernel dcbz128 0,rd // (also skip if moving down less than 512 bytes) bne-- cr7,1f // catch it first time through dcbz128 c128,rd dcbz128 c256,rd 1: addi rt,rs,64 dcbt 0,rx // touch in next chunk dcbt c128,rx dcbt c256,rx lvxl v4,c48,rs addi rs,rs,128 lvxl v5,0,rt cmplwi rc,384 // another chunk to go? lvxl v6,c16,rt lvxl v7,c32,rt lvxl v8,c48,rt addi rt,rs,64 vperm v25,v1,v2,v0 lvxl v9,0,rs lvxl v10,c16,rs vperm v26,v2,v3,v0 lvxl v11,c32,rs lvxl v12,c48,rs vperm v27,v3,v4,v0 addi rs,rs,128 lvxl v13,0,rt lvxl v14,c16,rt vperm v28,v4,v5,v0 lvxl v15,c32,rt lvxl v16,c48,rt vperm v29,v5,v6,v0 addi rt,rs,64 lvxl v17,0,rs lvxl v18,c16,rs vperm v30,v6,v7,v0 lvxl v19,c32,rs lvxl v20,c48,rs vperm v31,v7,v8,v0 addi rs,rs,128 lvxl v21,0,rt lvxl v22,c16,rt vperm v2,v8,v9,v0 lvxl v23,c32,rt lvxl v24,c48,rt vperm v3,v9,v10,v0 lvx v1,0,rs // get 1st qw of next chunk vperm v4,v10,v11,v0 addi rt,rd,64 stvxl v25,0,rd stvxl v26,c16,rd vperm v5,v11,v12,v0 stvxl v27,c32,rd stvxl v28,c48,rd vperm v6,v12,v13,v0 addi rd,rd,128 stvxl v29,0,rt stvxl v30,c16,rt vperm v7,v13,v14,v0 stvxl v31,c32,rt stvxl v2,c48,rt vperm v8,v14,v15,v0 addi rt,rd,64 stvxl v3,0,rd stvxl v4,c16,rd vperm v9,v15,v16,v0 stvxl v5,c32,rd stvxl v6,c48,rd vperm v10,v16,v17,v0 addi rd,rd,128 stvxl v7,0,rt vperm v11,v17,v18,v0 stvxl v8,c16,rt stvxl v9,c32,rt vperm v12,v18,v19,v0 stvxl v10,c48,rt addi rt,rd,64 vperm v13,v19,v20,v0 stvxl v11,0,rd stvxl v12,c16,rd vperm v14,v20,v21,v0 stvxl v13,c32,rd vperm v15,v21,v22,v0 stvxl v14,c48,rd vperm v16,v22,v23,v0 addi rd,rd,128 stvxl v15,0,rt vperm v17,v23,v24,v0 stvxl v16,c16,rt vperm v18,v24,v1,v0 stvxl v17,c32,rt stvxl v18,c48,rt bge++ LunalignedLoop // loop if another 384 bytes to go // End of unaligned main loop. Handle up to 384 leftover bytes. srwi. r0,rc,5 // get count of 32-byte chunks remaining beq Ldone // none rlwinm rc,rc,0,0x1F // mask count down to 0..31 leftover bytes mtctr r0 1: // loop over 32-byte chunks lvx v2,c16,rs lvx v3,c32,rs addi rs,rs,32 vperm v8,v1,v2,v0 vperm v9,v2,v3,v0 vor v1,v3,v3 // v1 <- v3 stvx v8,0,rd stvx v9,c16,rd addi rd,rd,32 bdnz 1b b Ldone // Aligned loop. Destination is 128-byte aligned, and source is 16-byte // aligned. Loop over 512-byte chunks (4 cache lines.) .align 5 LalignedLoop: subi rc,rc,512 // decrement count addi rx,rs,512 // address of next chunk lvxl v1,0,rs lvxl v2,c16,rs bne-- cr7,1f // skip dcbz's if cr7 beq has been turned off by kernel dcbz128 0,rd // (also skip if moving down less than 512 bytes) bne-- cr7,1f // catch it first time through dcbz128 c128,rd dcbz128 c256,rd dcbz128 c384,rd 1: addi rt,rs,64 dcbt 0,rx // touch in next chunk dcbt c128,rx dcbt c256,rx dcbt c384,rx lvxl v3,c32,rs lvxl v4,c48,rs addi rs,rs,128 lvxl v5,0,rt cmplwi rc,512 // another chunk to go? lvxl v6,c16,rt lvxl v7,c32,rt lvxl v8,c48,rt addi rt,rs,64 lvxl v9,0,rs lvxl v10,c16,rs lvxl v11,c32,rs lvxl v12,c48,rs addi rs,rs,128 lvxl v13,0,rt lvxl v14,c16,rt lvxl v15,c32,rt lvxl v16,c48,rt addi rt,rs,64 lvxl v17,0,rs lvxl v18,c16,rs lvxl v19,c32,rs lvxl v20,c48,rs addi rs,rs,128 lvxl v21,0,rt lvxl v22,c16,rt lvxl v23,c32,rt lvxl v24,c48,rt addi rt,rs,64 lvxl v25,0,rs lvxl v26,c16,rs lvxl v27,c32,rs lvxl v28,c48,rs addi rs,rs,128 lvxl v29,0,rt lvxl v30,c16,rt lvxl v31,c32,rt lvxl v0,c48,rt addi rt,rd,64 stvxl v1,0,rd stvxl v2,c16,rd stvxl v3,c32,rd stvxl v4,c48,rd addi rd,rd,128 stvxl v5,0,rt stvxl v6,c16,rt stvxl v7,c32,rt stvxl v8,c48,rt addi rt,rd,64 stvxl v9,0,rd stvxl v10,c16,rd stvxl v11,c32,rd stvxl v12,c48,rd addi rd,rd,128 stvxl v13,0,rt stvxl v14,c16,rt stvxl v15,c32,rt stvxl v16,c48,rt addi rt,rd,64 stvxl v17,0,rd stvxl v18,c16,rd stvxl v19,c32,rd stvxl v20,c48,rd addi rd,rd,128 stvxl v21,0,rt stvxl v22,c16,rt stvxl v23,c32,rt stvxl v24,c48,rt addi rt,rd,64 stvxl v25,0,rd stvxl v26,c16,rd stvxl v27,c32,rd stvxl v28,c48,rd addi rd,rd,128 stvxl v29,0,rt stvxl v30,c16,rt stvxl v31,c32,rt stvxl v0,c48,rt bge++ LalignedLoop // loop if another 512 bytes to go // End of aligned main loop. Handle up to 511 leftover bytes. srwi. r0,rc,5 // get count of 32-byte chunks remaining beq Ldone // none rlwinm rc,rc,0,0x1F // mask count down to 0..31 leftover bytes mtctr r0 1: // loop over 32-byte chunks lvx v1,0,rs lvx v2,c16,rs addi rs,rs,32 stvx v1,0,rd stvx v2,c16,rd addi rd,rd,32 bdnz 1b // Done, except for 0..31 leftovers at end. Restore non-volatiles. // rs = source ptr // rd = dest ptr // rc = count (0..31) // rv = caller's vrsave Ldone: cmpwi rc,0 // any leftover bytes? lwz r13,rzR13(r1) // restore non-volatiles from redzone lwz r14,rzR14(r1) lwz r15,rzR15(r1) li r0,rzV20 lvx v20,r1,r0 li r0,rzV21 lvx v21,r1,r0 li r0,rzV22 lvx v22,r1,r0 li r0,rzV23 lvx v23,r1,r0 li r0,rzV24 lvx v24,r1,r0 li r0,rzV25 lvx v25,r1,r0 li r0,rzV26 lvx v26,r1,r0 li r0,rzV27 lvx v27,r1,r0 li r0,rzV28 lvx v28,r1,r0 li r0,rzV29 lvx v29,r1,r0 li r0,rzV30 lvx v30,r1,r0 li r0,rzV31 lvx v31,r1,r0 mtspr vrsave,rv // restore caller's bitmask beqlr // done if no leftover bytes // Handle 1..31 leftover bytes at end. mtctr rc // set up loop count b 1f .align 5 1: lbz r0,0(rs) addi rs,rs,1 stb r0,0(rd) addi rd,rd,1 bdnz 1b blr COMMPAGE_DESCRIPTOR(bigcopy_970,_COMM_PAGE_BIGCOPY,0,0,0) // load on all machines for now