#include "IOCopyMapper.h"
#include <sys/sysctl.h>
#if 0
#define DEBG(fmt, args...) { kprintf(fmt, ## args); }
#else
#define DEBG(fmt, args...) {}
#endif
extern "C" {
extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
extern void ml_get_bouncepool_info(
vm_offset_t *phys_addr,
vm_size_t *size);
extern unsigned int vm_lopage_max_count;
extern unsigned int vm_himemory_mode;
}
#define super IOMapper
OSDefineMetaClassAndStructors(IOCopyMapper, IOMapper);
typedef struct FreeDARTEntry {
#if __BIG_ENDIAN__
unsigned int
fValid : 1,
fInUse : 1, : 5, fSize : 5,
: 2,
fNext :18;
#elif __LITTLE_ENDIAN__
unsigned int
fNext :18, : 2,
fSize : 5,
: 5, fInUse : 1, fValid : 1;
#endif
#if __BIG_ENDIAN__
unsigned int
:14,
fPrev :18;
#elif __LITTLE_ENDIAN__
unsigned int
fPrev :18, :14;
#endif
} FreeDARTEntry;
typedef struct ActiveDARTEntry {
#if __BIG_ENDIAN__
unsigned int
fValid : 1, fPPNum :31; #define ACTIVEDARTENTRY(page) { true, page }
#elif __LITTLE_ENDIAN__
unsigned int
fPPNum :31, fValid : 1; #define ACTIVEDARTENTRY(page) { page, true }
#endif
};
#define kActivePerFree (sizeof(freeDART[0]) / sizeof(ActiveDARTEntry))
static SYSCTL_UINT(_kern, OID_AUTO, copyregionmax,
CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
NULL, 0, "");
static SYSCTL_UINT(_kern, OID_AUTO, lowpagemax,
CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
&vm_lopage_max_count, 0, "");
static SYSCTL_UINT(_kern, OID_AUTO, himemorymode,
CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
&vm_himemory_mode, 0, "");
bool IOCopyMapper::initHardware(IOService * provider)
{
UInt32 dartSizePages = 0;
vm_offset_t phys_addr;
vm_size_t size;
ml_get_bouncepool_info(&phys_addr, &size);
if (!size)
return (false);
fBufferPage = atop_32(phys_addr);
dartSizePages = (atop_32(size) + kTransPerPage - 1) / kTransPerPage;
fTableLock = IOLockAlloc();
if (!fTableLock)
return false;
if (!allocTable(dartSizePages * kMapperPage))
return false;
UInt32 canMapPages = dartSizePages * kTransPerPage;
fMapperRegionSize = canMapPages;
for (fNumZones = 0; canMapPages; fNumZones++)
canMapPages >>= 1;
fNumZones -= 3;
invalidateDART(0, fMapperRegionSize);
breakUp(0, fNumZones, 0);
((FreeDARTEntry *) fTable)->fInUse = true;
fMapperRegionUsed = kMinZoneSize;
fMapperRegionMaxUsed = fMapperRegionUsed;
sysctl__kern_copyregionmax.oid_arg1 = &fMapperRegionMaxUsed;
sysctl_register_oid(&sysctl__kern_copyregionmax);
sysctl_register_oid(&sysctl__kern_lowpagemax);
sysctl_register_oid(&sysctl__kern_himemorymode);
fDummyPage = IOMallocAligned(0x1000, 0x1000);
fDummyPageNumber =
pmap_find_phys(kernel_pmap, (addr64_t) (uintptr_t) fDummyPage);
return true;
}
void IOCopyMapper::free()
{
if (fDummyPage) {
IOFreeAligned(fDummyPage, 0x1000);
fDummyPage = 0;
fDummyPageNumber = 0;
}
if (fTableLock) {
IOLockFree(fTableLock);
fTableLock = 0;
}
super::free();
}
void IOCopyMapper::breakUp(unsigned startIndex, unsigned endIndex, unsigned freeInd)
{
unsigned int zoneSize;
FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
do {
endIndex--;
zoneSize = (kMinZoneSize/2 << endIndex);
ppnum_t tail = freeInd + zoneSize;
DEBG("breakup z %d start %x tail %x\n", endIndex, freeInd, tail);
fFreeLists[endIndex] = tail;
freeDART[tail].fSize = endIndex;
freeDART[tail].fNext = freeDART[tail].fPrev = 0;
} while (endIndex != startIndex);
freeDART[freeInd].fSize = endIndex;
}
ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages)
{
unsigned int zone, zoneSize, z, cnt;
ppnum_t next, ret = 0;
FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
if (pages < kMinZoneSize)
pages = kMinZoneSize;
if (pages >= fMapperRegionSize/2)
{
panic("iovmAlloc 0x%lx", pages);
return 0;
}
for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
zoneSize <<= 1;
{
IOLockLock(fTableLock);
for (;;) {
for (z = zone; z < fNumZones; z++) {
if ( (ret = fFreeLists[z]) )
break;
}
if (ret)
break;
fFreeSleepers++;
IOLockSleep(fTableLock, fFreeLists, THREAD_UNINT);
fFreeSleepers--;
}
if (zone != z)
{
DEBG("breakup %d, %d, 0x%x\n", zone, z, ret);
breakUp(zone, z, ret);
}
freeDART[ret].fInUse = true; next = freeDART[ret].fNext;
DEBG("va: 0x%lx, %ld, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next);
fFreeLists[z] = next;
if (next)
freeDART[next].fPrev = 0;
ret *= kActivePerFree;
ActiveDARTEntry pageEntry = ACTIVEDARTENTRY(fDummyPageNumber);
for (cnt = 0; cnt < pages; cnt++) {
ActiveDARTEntry *activeDART = &fMappings[ret + cnt];
*activeDART = pageEntry;
}
fMapperRegionUsed += pages;
if (fMapperRegionUsed > fMapperRegionMaxUsed)
fMapperRegionMaxUsed = fMapperRegionUsed;
IOLockUnlock(fTableLock);
}
if (ret)
ret += fBufferPage;
return ret;
}
void IOCopyMapper::invalidateDART(ppnum_t pnum, IOItemCount size)
{
bzero((void *) &fMappings[pnum], size * sizeof(fMappings[0]));
}
void IOCopyMapper::iovmFree(ppnum_t addr, IOItemCount pages)
{
unsigned int zone, zoneSize, z;
FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
if (addr < fBufferPage)
IOPanic("addr < fBufferPage");
addr -= fBufferPage;
if (pages < kMinZoneSize)
pages = kMinZoneSize;
if (pages >= fMapperRegionSize/2)
return;
for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
zoneSize <<= 1;
IOLockLock(fTableLock);
invalidateDART(addr, pages);
addr /= kActivePerFree;
for (z = zone; z < fNumZones; z++) {
ppnum_t pair = addr ^ (kMinZoneSize/2 << z); if (freeDART[pair].fValid || freeDART[pair].fInUse || (freeDART[pair].fSize != z))
break;
ppnum_t next = freeDART[pair].fNext;
ppnum_t prev = freeDART[pair].fPrev;
if (prev)
freeDART[prev].fNext = next;
else
fFreeLists[z] = next;
if (next)
freeDART[next].fPrev = prev;
if (addr > pair)
addr = pair;
}
DEBG("vf: 0x%lx, %ld, z %d, head %lx, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr);
freeDART[addr].fSize = z;
freeDART[addr].fNext = fFreeLists[z];
if (fFreeLists[z])
freeDART[fFreeLists[z]].fPrev = addr;
freeDART[addr].fPrev = 0;
fFreeLists[z] = addr;
fMapperRegionUsed -= pages;
if (fFreeSleepers)
IOLockWakeup(fTableLock, fFreeLists, false);
IOLockUnlock(fTableLock);
}
addr64_t IOCopyMapper::mapAddr(IOPhysicalAddress addr)
{
if (addr < ptoa_32(fBufferPage))
{
return (addr64_t) addr; }
addr -= ptoa_32(fBufferPage);
if (addr >= ptoa_32(fMapperRegionSize))
{
return (addr64_t) addr; }
else
{
ActiveDARTEntry *activeDART = (ActiveDARTEntry *) fTable;
UInt offset = addr & PAGE_MASK;
ActiveDARTEntry mappedPage = activeDART[atop_32(addr)];
if (mappedPage.fValid)
{
return (ptoa_64(mappedPage.fPPNum) | offset);
}
panic("%s::mapAddr(0x%08lx) not mapped for I/O\n", getName(), addr);
return 0;
}
}
void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page)
{
addr -= fBufferPage;
addr += offset;
ActiveDARTEntry *activeDART = &fMappings[addr];
ActiveDARTEntry entry = ACTIVEDARTENTRY(page);
*activeDART = entry;
}
void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
ppnum_t *pageList, IOItemCount pageCount)
{
addr -= fBufferPage;
addr += offset;
IOItemCount i;
ActiveDARTEntry *activeDART = &fMappings[addr];
for (i = 0; i < pageCount; i++)
{
ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i]);
activeDART[i] = entry;
}
}
void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
upl_page_info_t *pageList, IOItemCount pageCount)
{
addr -= fBufferPage;
addr += offset;
IOItemCount i;
ActiveDARTEntry *activeDART = &fMappings[addr];
for (i = 0; i < pageCount; i++)
{
ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i].phys_addr);
activeDART[i] = entry;
}
}