/*
* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* The contents of this file constitute Original Code as defined in and
* are subject to the Apple Public Source License Version 1.1 (the
* "License"). You may not use this file except in compliance with the
* License. Please obtain a copy of the License at
* http://www.apple.com/publicsource and read it before using this file.
*
* This Original Code and all software distributed under the License are
* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
* License for the specific language governing rights and limitations
* under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
#include <debug.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <mach/ppc/vm_param.h>
#include <assym.s>
#include <sys/errno.h>
/*
* void pmap_zero_page(vm_offset_t pa)
*
* zero a page of physical memory.
*/
#if DEBUG
/* C debug stub in pmap.c calls this */
ENTRY(pmap_zero_page_assembler, TAG_NO_FRAME_USED)
#else
ENTRY(pmap_zero_page, TAG_NO_FRAME_USED)
#endif /* DEBUG */
mfmsr r6 /* Get the MSR */
rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1 /* Turn off DR */
rlwinm r7,r7,0,MSR_EE_BIT+1,MSR_EE_BIT-1 mtmsr r7 /* Set MSR to DR off */
isync /* Ensure data translations are off */
.L_phys_zero_loop:
subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
dcbz r4, r3 /* Clear the whole thing to 0s */
subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
dcbz r5, r3 /* Clear the next to zeros */
bgt+ .L_phys_zero_loop /* Keep going until we do the page... */
sync /* Make sure they're all done */
li r4,PPC_PGBYTES-CACHE_LINE_SIZE /* Point to the end of the page */
.L_inst_inval_loop:
subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
icbi r4, r3 /* Clear the whole thing to 0s */
subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
icbi r5, r3 /* Clear the next to zeros */
bgt+ .L_inst_inval_loop /* Keep going until we do the page... */
sync /* Make sure they're all done */
mtmsr r6 /* Restore original translations */
isync /* Ensure data translations are on */
blr
/* void
* phys_copy(src, dst, bytecount)
* vm_offset_t src * int bytecount
*
* This routine will copy bytecount bytes from physical address src to physical
* address dst.
*/
ENTRY(phys_copy, TAG_NO_FRAME_USED)
/* Switch off data translations */
mfmsr r6
rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
rlwinm r7, r7, 0, MSR_EE_BIT+1, MSR_EE_BIT-1
mtmsr r7
isync /* Ensure data translations are off */
subi r3, r3, 4
subi r4, r4, 4
cmpwi r5, 3
ble- .L_phys_copy_bytes
.L_phys_copy_loop:
lwz r0, 4(r3)
addi r3, r3, 4
subi r5, r5, 4
stw r0, 4(r4)
addi r4, r4, 4
cmpwi r5, 3
bgt+ .L_phys_copy_loop
/* If no leftover bytes, we're done now */
cmpwi r5, 0
beq+ .L_phys_copy_done
.L_phys_copy_bytes:
addi r3, r3, 3
addi r4, r4, 3
.L_phys_copy_byte_loop:
lbz r0, 1(r3)
addi r3, r3, 1
subi r5, r5, 1
stb r0, 1(r4)
addi r4, r4, 1
cmpwi r5, 0
bne+ .L_phys_copy_byte_loop
.L_phys_copy_done:
mtmsr r6 /* Restore original translations */
isync /* Ensure data translations are off */
blr
/* void
* pmap_copy_page(src, dst)
* vm_offset_t src *
* This routine will copy the physical page src to physical page dst
*
* This routine assumes that the src and dst are page aligned and that the
* destination is cached.
*
* We also must assume that noone will be executing within the destination
* page. We also assume that this will be used for paging
*
*/
#if DEBUG
/* if debug, we have a little piece of C around this
* in pmap.c that gives some trace ability
*/
ENTRY(pmap_copy_page_assembler, TAG_NO_FRAME_USED)
#else
ENTRY(pmap_copy_page, TAG_NO_FRAME_USED)
#endif /* DEBUG */
#if 0
mfpvr r9 cmplwi r9,PROCESSOR_VERSION_Max #endif
mfmsr r9 rlwinm r9,r9,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 mtmsr r7
stfd f0,FM_SIZE+0(r1) stfd f1,FM_SIZE+8(r1) stfd f2,FM_SIZE+16(r1) stfd f3,FM_SIZE+24(r1) mtmsr r7
dcbt br0, r3 /* Start in first input line */
li r5, CACHE_LINE_SIZE /* Get the line size */
.L_pmap_copy_page_loop:
dcbz 0, r4 /* Allocate a line for the output */
lfd f0, 0(r3) /* Get first 8 */
lfd f1, 8(r3) /* Get second 8 */
lfd f2, 16(r3) /* Get third 8 */
stfd f0, 0(r4) /* Put first 8 */
dcbt r5, r3 /* Start next line coming in */
lfd f3, 24(r3) /* Get fourth 8 */
stfd f1, 8(r4) /* Put second 8 */
addi r3,r3,CACHE_LINE_SIZE /* Point to the next line in */
stfd f2, 16(r4) /* Put third 8 */
cmplw cr0,r3,r6 /* See if we're finished yet */
stfd f3, 24(r4) /* Put fourth 8 */
dcbst br0,r4 /* Force it out */
addi r4,r4,CACHE_LINE_SIZE /* Point to the next line out */
blt+ .L_pmap_copy_page_loop /* Copy the whole page */
sync /* Make sure they're all done */
li r4,PPC_PGBYTES-CACHE_LINE_SIZE /* Point to the end of the page */
invalinst:
subic. r5,r4,CACHE_LINE_SIZE /* Point to the next one */
icbi r4, r8 /* Trash the i-cache */
subi r4,r5,CACHE_LINE_SIZE /* Point to the next one */
icbi r5, r8 /* Trash the i-cache */
bgt+ invalinst /* Keep going until we do the page... */
rlwimi r7,r9,0,MSR_DR_BIT,MSR_DR_BIT
mtmsr r7
lfd f0,FM_SIZE+0(r1) lfd f2,FM_SIZE+16(r1)
lwz r1,0(r1) mtmsr r9 blr
#if 0
wegotaltivec:
mfmsr r9 rlwinm r7,r9,0,MSR_EE_BIT+1,MSR_EE_BIT-1 rlwinm r7,r7,0,MSR_DR_BIT+1,MSR_DR_BIT-1 mtsprg 256,r8 isync
addi r11,r3,4096
avmovepg: lvxl v0,br0,r3 lvxl v1,r10,r3 addi r3,r3,32 stvxl v1,r10,r4 dcbst br0,r4 blt+ avmovepg li r8,0 mtsprg 256,r8 isync
blr
#endif
/*
* int
* copyin(src, dst, count)
* vm_offset_t src * int count */
ENTRY2(copyin, copyinmsg, TAG_NO_FRAME_USED)
/* Preamble allowing us to call a sub-function */
mflr r0
stw r0,FM_LR_SAVE(r1)
stwu r1,-(FM_SIZE+16)(r1)
cmpli cr0,r5,0
ble- cr0,.L_copyinout_trivial
/* we know we have a valid copyin to do now */
/* Set up thread_recover in case we hit an illegal address */
mfsprg r8,1 /* Get the current act */
lwz r10,ACT_THREAD(r8)
lis r11,hi16(.L_copyinout_error)
lwz r8,ACT_VMMAP(r8)
ori r11,r11,lo16(.L_copyinout_error)
add r9,r3,r5 /* Get the end of the source */
lwz r8,VMMAP_PMAP(r8) subi r9,r9,1 /* Make sure we don't go too far */
add r8,r8,r12 xor r9,r9,r3 /* Smoosh 'em together */
lwz r8,PMAP_SEGS(r8) mtsr SR_COPYIN,r8 #if 0
lis r0,HIGH_ADDR(EXT(dbgRegsCall)) /* (TEST/DEBUG) */
ori r0,r0,LOW_ADDR(EXT(dbgRegsCall)) /* (TEST/DEBUG) */
sc /* (TEST/DEBUG) */
#endif
/* For optimization, we check if the copyin lies on a segment
* boundary. If it doesn't, we can use a simple copy. If it
* does, we split it into two separate copies in some C code.
*/
bne- .L_call_copyin_multiple /* Nope, we went past the segment boundary... */
rlwinm r3,r3,0,4,31
oris r3,r3,(SR_COPYIN_NUM << (28-16)) /* Set the copyin segment as the source */
bl EXT(bcopy)
/* Now that copyin is done, we don't need a recovery point */
addi r1,r1,FM_SIZE+16
mfsprg r6,1 /* Get the current act */
lwz r10,ACT_THREAD(r6)
li r3,0
lwz r0,FM_LR_SAVE(r1)
stw r3,THREAD_RECOVER(r10) /* Clear recovery */
mtlr r0
blr
/* we get here via the exception handler if an illegal
* user memory reference was made.
*/
.L_copyinout_error:
/* Now that copyin is done, we don't need a recovery point */
mfsprg r6,1 /* Get the current act */
addi r1,r1,FM_SIZE+16
lwz r10,ACT_THREAD(r6)
li r4,0
lwz r0,FM_LR_SAVE(r1)
stw r4,THREAD_RECOVER(r10) /* Clear recovery */
mtlr r0
li r3,EFAULT
.L_copyinout_trivial:
/* The copyin/out was for either 0 bytes or a negative
* number of bytes, return an appropriate value (0 == SUCCESS).
* cr0 still contains result of comparison of len with 0.
*/
li r3, 0
beq+ cr0, .L_copyinout_negative
li r3, 1
.L_copyinout_negative:
/* unwind the stack */
addi r1, r1, FM_SIZE+16
lwz r0, FM_LR_SAVE(r1)
mtlr r0
blr
.L_call_copyin_multiple:
/* unwind the stack */
addi r1, r1, FM_SIZE+16
lwz r0, FM_LR_SAVE(r1)
mtlr r0
b EXT(copyin_multiple) /* not a call - a jump! */
/*
* int
* copyout(src, dst, count)
* vm_offset_t src * int count */
ENTRY2(copyout, copyoutmsg, TAG_NO_FRAME_USED)
/* Preamble allowing us to call a sub-function */
mflr r0
stw r0,FM_LR_SAVE(r1)
stwu r1,-(FM_SIZE+16)(r1)
#if 0
stw r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
stw r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
stw r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
mr r6,r0 /* (TEST/DEBUG) */
bl EXT(tracecopyout) /* (TEST/DEBUG) */
lwz r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
lwz r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
lwz r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
#endif
cmpli cr0,r5,0
ble- cr0,.L_copyinout_trivial
/* we know we have a valid copyout to do now */
/* Set up thread_recover in case we hit an illegal address */
mfsprg r8,1 /* Get the current act */
lwz r10,ACT_THREAD(r8)
lis r11,HIGH_ADDR(.L_copyinout_error)
lwz r8,ACT_VMMAP(r8)
rlwinm r12,r4,6,26,29 add r9,r4,r5 /* Get the end of the destination */
lwz r8,VMMAP_PMAP(r8)
subi r9,r9,1 /* Make sure we don't go too far */
add r8,r8,r12 xor r9,r9,r4 /* Smoosh 'em together */
lwz r8,PMAP_SEGS(r8) mtsr SR_COPYIN,r8
isync
/* For optimisation, we check if the copyout lies on a segment
* boundary. If it doesn't, we can use a simple copy. If it
* does, we split it into two separate copies in some C code.
*/
bne- .L_call_copyout_multiple /* Nope, we went past the segment boundary... */
rlwinm r4,r4,0,4,31
oris r4,r4,(SR_COPYIN_NUM << (28-16)) /* Set the copyin segment as the source */
bl EXT(bcopy)
/* Now that copyout is done, we don't need a recovery point */
mfsprg r6,1 /* Get the current act */
addi r1,r1,FM_SIZE+16
lwz r10,ACT_THREAD(r6)
li r3,0
lwz r0,FM_LR_SAVE(r1)
stw r3,THREAD_RECOVER(r10) /* Clear recovery */
mtlr r0
blr
.L_call_copyout_multiple:
/* unwind the stack */
addi r1, r1, FM_SIZE+16
lwz r0, FM_LR_SAVE(r1)
mtlr r0
b EXT(copyout_multiple) /* not a call - a jump! */
/*
* boolean_t
* copyinstr(src, dst, count, maxcount)
* vm_offset_t src * vm_size_t maxcount *
* Set *count to the number of bytes copied
*
* If dst == NULL, don't copy, just count bytes.
* Only currently called from klcopyinstr.
*/
ENTRY(copyinstr, TAG_NO_FRAME_USED)
/* Preamble allowing us to call a sub-function */
mflr r0
stw r0,FM_LR_SAVE(r1)
stwu r1,-(FM_SIZE+16)(r1)
#if 0
stw r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
stw r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
stw r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
stw r6,FM_SIZE+12(r1) /* (TEST/DEBUG) */
mr r7,r0 /* (TEST/DEBUG) */
bl EXT(tracecopystr) /* (TEST/DEBUG) */
lwz r3,FM_SIZE+0(r1) /* (TEST/DEBUG) */
lwz r4,FM_SIZE+4(r1) /* (TEST/DEBUG) */
lwz r5,FM_SIZE+8(r1) /* (TEST/DEBUG) */
stw r6,FM_SIZE+12(r1) /* (TEST/DEBUG) */
#endif
cmpli cr0,r5,0
ble- cr0,.L_copyinout_trivial
/* we know we have a valid copyin to do now */
/* Set up thread_recover in case we hit an illegal address */
li r0,0
mfsprg r8,1 /* Get the current act */
lwz r10,ACT_THREAD(r8)
stw r0,0(r6) /* Clear result length */
lis r11,HIGH_ADDR(.L_copyinout_error)
lwz r8,ACT_VMMAP(r8) lwz r8,VMMAP_PMAP(r8)
ori r11,r11,LOW_ADDR(.L_copyinout_error)
add r8,r8,r12 rlwinm r3,r3,0,4,31
lwz r7,PMAP_SEGS(r8)
/* Copy byte by byte for now - TODO NMGS speed this up with
* some clever (but fairly standard) logic for word copies.
* We don't use a copyinstr_multiple since copyinstr is called
* with INT_MAX in the linux server. Eugh.
*/
li r9,0 /* Clear byte counter */
/* If the destination is NULL, don't do writes,
* just count bytes. We set CR7 outside the loop to save time
*/
cmpwi cr7,r4,0 /* Is the destination null? */
nxtseg: mtsr SR_COPYIN,r7 /* Set the source SR */
isync
.L_copyinstr_loop:
lbz r0,0(r3) /* Get the source */
addic. r5,r5,-1 /* Have we gone far enough? */
addi r3,r3,1 /* Bump source pointer */
cmpwi cr1,r0,0 /* Did we hit a null? */
beq cr7,.L_copyinstr_no_store /* If we are just counting, skip the store... */
stb r0,0(r4) /* Move to sink */
addi r4,r4,1 /* Advance sink pointer */
.L_copyinstr_no_store:
addi r9,r9,1 /* Count the character */
beq- cr1,.L_copyinstr_done /* We're done if we did a null... */
beq- cr0,L_copyinstr_toobig /* Also if we maxed the count... */
/* Check to see if the copyin pointer has moved out of the
* copyin segment, if it has we must remap.
*/
rlwinm. r0,r3,0,4,31 /* Did we wrap around to 0? */
bne+ cr0,.L_copyinstr_loop /* Nope... */
lwz r7,PMAP_SEGS+4(r8) oris r3,r0,(SR_COPYIN_NUM << (28-16)) /* Reset the segment number */
b nxtseg /* Keep going... */
L_copyinstr_toobig:
li r3,ENAMETOOLONG
b L_copyinstr_return
.L_copyinstr_done:
li r3,0 /* Normal return */
L_copyinstr_return:
li r4,0 /* to clear thread_recover */
stw r9,0(r6) /* Set how many bytes we did */
stw r4,THREAD_RECOVER(r10) /* Clear recovery exit */
addi r1, r1, FM_SIZE+16
lwz r0, FM_LR_SAVE(r1)
mtlr r0
blr