#if ACPI_SUPPORT
#include <IOKit/IOMapper.h>
#include <IOKit/IOKitKeysPrivate.h>
#include <libkern/tree.h>
#include <libkern/OSDebug.h>
#include <i386/cpuid.h>
#include "dmar.h"
extern "C" vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
#define ENA_QI 1
#define TABLE_CB 0
#define BSIMPLE 0
#define KP 0
#define VTASRT 0
#define kLargeThresh (128)
#define kLargeThresh2 (32)
#define kVPages (1<<22)
#define kBPagesLog2 (18)
#define kBPagesSafe ((1<<kBPagesLog2)-(1<<(kBPagesLog2 - 2)))
#define kBPagesReserve ((1<<kBPagesLog2)-(1<<(kBPagesLog2 - 3)))
#define kRPages (1<<20)
#define kQIPageCount (2)
#define kTlbDrainReads (0ULL)
#define kTlbDrainWrites (0ULL)
#define VTLOG(fmt, args...) \
do { \
if ((gIOPCIFlags & kIOPCIConfiguratorVTLog) && !ml_at_interrupt_context()) \
IOLog(fmt, ## args); \
if (gIOPCIFlags & kIOPCIConfiguratorVTLog) \
kprintf(fmt, ## args); \
} while(0)
#if VTASRT
#define vtassert(ex) \
((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
#define vtd_space_nfault(x,y,z) _vtd_space_nfault(x,y,z)
#define STAT_ADD(space, name, value) do { space->stats.name += value; } while (false);
#else
#define vtassert(ex)
#define vtd_space_nfault(x,y,z)
#define STAT_ADD(space, name, value) do { space->stats.name += value; } while (false);
#endif
#if TABLE_CB
#define table_flush(addr, size, linesize) clflush((uintptr_t)(addr), (size), linesize);
#else
#define table_flush(addr, size, linesize) __mfence();
#endif
#if BSIMPLE
#define BLOCK(l) IOSimpleLockLock(l)
#define BUNLOCK(l) IOSimpleLockUnlock(l)
#else
#define BLOCK(l) IOLockLock(l)
#define BUNLOCK(l) IOLockUnlock(l)
#endif
#define arrayCount(x) (sizeof(x) / sizeof(x[0]))
enum
{
kEntryPresent = 0x00000001ULL
};
struct root_entry_t
{
uint64_t context_entry_ptr;
uint64_t resv;
};
struct context_entry_t
{
uint64_t address_space_root;
uint64_t context_entry;
};
struct qi_descriptor_t
{
uint64_t command;
uint64_t address;
};
enum
{
kFaultProcessingDisable = 0x00000002ULL,
kTranslationType0 = 0x00000000ULL,
kTranslationType1 = 0x00000004ULL,
kTranslationType2 = 0x00000008ULL,
kTranslationType3 = 0x0000000CULL,
kEvictionHint = 0x00000010ULL,
kAddressLocalityHint = 0x00000020ULL,
};
enum
{
kAddressWidth30 = 0x00000000ULL,
kAddressWidth39 = 0x00000001ULL,
kAddressWidth48 = 0x00000002ULL,
kAddressWidth57 = 0x00000003ULL,
kAddressWidth64 = 0x00000004ULL,
kContextAvail1 = 0x00000008ULL, kDomainIdentifier1 = 0x00000100ULL, };
enum
{
kNotTheDomain = 1ULL << 32,
kTheDomain = 2ULL
};
typedef uint64_t page_entry_t;
enum
{
kReadAccess = 0x00000001ULL,
kWriteAccess = 0x00000002ULL,
kPageAccess = kReadAccess|kWriteAccess,
kPageAvail1 = 0x00000004ULL, kSuperPage = 0x00000080ULL,
kPageAvail2 = 0x00000100ULL, kSnoopBehavior = 0x00000800ULL,
kTransientMapping = 0x4000000000000000ULL,
kPageAvail3 = 0x8000000000000000ULL,
kPageAddrMask = 0x3ffffffffffff000ULL
};
struct vtd_registers_t
{
uint32_t version;
uint32_t res1;
uint64_t capability;
uint64_t extended_capability;
uint32_t global_command;
uint32_t global_status;
uint64_t root_entry_table;
uint64_t context_command;
uint32_t res2;
uint32_t fault_status;
uint32_t fault_event_control;
uint32_t fault_event_data;
uint32_t fault_event_address;
uint32_t fault_event_upper_address;
uint64_t res3[2];
uint64_t advanced_fault;
uint32_t res4;
uint32_t protected_memory_enable;
uint32_t protected_low_memory_base;
uint32_t protected_low_memory_limit;
uint64_t protected_high_memory_base;
uint64_t protected_high_memory_limit;
uint64_t invalidation_queue_head;
uint64_t invalidation_queue_tail;
uint64_t invalidation_queue_address;
uint32_t res5;
uint32_t invalidation_completion_status;
uint32_t invalidation_completion_event_control;
uint32_t invalidation_completion_event_data;
uint32_t invalidation_completion_event_address;
uint32_t invalidation_completion_event_upper_address;
uint64_t res6;
uint64_t interrupt_remapping_table;
};
struct vtd_iotlb_registers_t
{
uint64_t address;
uint64_t command;
};
struct vtd_fault_registers_t
{
uint64_t fault_low;
uint64_t fault_high;
};
typedef char vtd_registers_t_check[(sizeof(vtd_registers_t) == 0xc0) ? 1 : -1];
struct vtd_unit_t
{
ACPI_DMAR_HARDWARE_UNIT * dmar;
volatile vtd_registers_t * regs;
volatile vtd_iotlb_registers_t * iotlb;
volatile vtd_fault_registers_t * faults;
IOMemoryMap * qi_map;
qi_descriptor_t * qi_table;
uint64_t root;
uint64_t msi_address;
uint64_t qi_address;
uint64_t qi_stamp_address;
uint32_t qi_tail;
uint32_t qi_mask;
volatile
uint32_t qi_stamp;
uint32_t msi_data;
uint32_t num_fault;
uint32_t rounding;
uint8_t global:1;
uint8_t caching:1;
uint8_t selective:1;
};
static inline void __mfence(void)
{
__asm__ volatile("mfence");
}
static inline void __clflush(void *ptr)
{
__asm__ volatile("clflush (%0)" : : "r" (ptr));
}
static inline void clflush(uintptr_t addr, unsigned int count, uintptr_t linesize)
{
uintptr_t bound = (addr + count + linesize -1) & ~(linesize - 1);
__mfence();
while (addr < bound)
{
__clflush((void *) (uintptr_t) addr);
addr += linesize;
}
__mfence();
}
static
vtd_unit_t * unit_init(ACPI_DMAR_HARDWARE_UNIT * dmar)
{
vtd_unit_t * unit;
unit = IONew(vtd_unit_t, 1);
if (!unit) return (NULL);
bzero(unit, sizeof(vtd_unit_t));
unit->dmar = dmar;
VTLOG("unit %p Address %llx, Flags %x\n",
dmar, dmar->Address, dmar->Flags);
unit->regs = (typeof unit->regs) ml_io_map(dmar->Address, 0x1000);
uint32_t
offset = (unit->regs->extended_capability >> (8 - 4)) & (((1 << 10) - 1) << 4);
unit->iotlb = (typeof(unit->iotlb)) (((uintptr_t)unit->regs) + offset);
offset = (unit->regs->capability >> (24 - 4)) & (((1 << 10) - 1) << 4);
unit->faults = (typeof(unit->faults)) (((uintptr_t)unit->regs) + offset);
unit->num_fault = (1 + ((unit->regs->capability >> 40) & ((1 << 8) - 1)));
unit->selective = (1 & (unit->regs->capability >> 39));
unit->rounding = (0x3f & (unit->regs->capability >> 48));
unit->caching = (1 & (unit->regs->capability >> 7));
unit->global = (ACPI_DMAR_INCLUDE_ALL & dmar->Flags);
VTLOG("cap 0x%llx extcap 0x%llx glob %d cache sel %d mode %d iotlb %p nfault[%d] %p\n",
unit->regs->capability, unit->regs->extended_capability,
unit->global, unit->selective, unit->caching,
unit->iotlb, unit->num_fault, unit->faults);
if (unit->caching
|| ((!unit->global) && (!(kIOPCIConfiguratorIGIsMapped & gIOPCIFlags))))
{
IODelete(unit, vtd_unit_t, 1);
unit = NULL;
}
return (unit);
}
static void
unit_faults(vtd_unit_t * unit, bool log)
{
uint32_t idx;
for (idx = 0; idx < unit->num_fault; idx++)
{
uint64_t h, l;
uint32_t faults_pending;
faults_pending = unit->regs->fault_status;
h = unit->faults[idx].fault_high;
l = unit->faults[idx].fault_low;
unit->faults[idx].fault_high = h;
unit->regs->fault_status = faults_pending;
__mfence();
if (log && ((1ULL << 63) & h))
{
char msg[256];
snprintf(msg, sizeof(msg), "vtd[%d] fault: device %d:%d:%d reason 0x%x %c:0x%llx", idx,
(int)(255 & (h >> 8)), (int)(31 & (h >> 3)), (int)(7 & (h >> 0)),
(int)(255 & (h >> (96 - 64))), (h & (1ULL << (126 - 64))) ? 'R' : 'W', l);
IOLog("%s\n", msg);
kprintf("%s\n", msg);
if (kIOPCIConfiguratorPanicOnFault & gIOPCIFlags) panic("%s", msg);
}
}
}
static void
unit_enable(vtd_unit_t * unit)
{
uint32_t command;
VTLOG("unit %p global status 0x%x\n", unit, unit->regs->global_status);
unit->regs->root_entry_table = unit->root;
__mfence();
unit->regs->global_command = (1UL<<30);
__mfence();
while (!((1UL<<30) & unit->regs->global_status)) {}
unit->regs->context_command = (1ULL<<63) | (1ULL<<61);
__mfence();
while ((1ULL<<63) & unit->regs->context_command) {}
unit->iotlb->command = (1ULL<<63) | (1ULL<<60) | (1ULL<<49) | (1ULL<<48);
__mfence();
while ((1ULL<<63) & unit->iotlb->command) {}
unit->qi_tail = 0;
unit->regs->invalidation_queue_head = 0;
unit->regs->invalidation_queue_tail = 0;
unit->regs->invalidation_queue_address = unit->qi_address;
command = 0;
#if ENA_QI
command |= (1UL<<26);
unit->regs->global_command = command;
__mfence();
while (!((1UL<<26) & unit->regs->global_status)) {}
VTLOG("did ena qi p 0x%qx v %p\n", unit->qi_address, unit->qi_table);
#endif
command |= (1UL<<31);
unit->regs->global_command = command;
__mfence();
while (!((1UL<<31) & unit->regs->global_status)) {}
VTLOG("did ena\n");
if (unit->msi_address)
{
unit->regs->invalidation_completion_event_data = unit->msi_data;
unit->regs->invalidation_completion_event_address = unit->msi_address;
unit->regs->invalidation_completion_event_upper_address = (unit->msi_address >> 32);
unit->regs->fault_event_data = unit->msi_data + 1;
unit->regs->fault_event_address = unit->msi_address;
unit->regs->fault_event_upper_address = (unit->msi_address >> 32);
__mfence();
unit_faults(unit, false);
unit->regs->fault_event_control = 0; unit->regs->invalidation_completion_event_control = 0; unit->regs->invalidation_completion_status = 1;
}
}
static void
unit_quiesce(vtd_unit_t * unit)
{
VTLOG("unit %p quiesce\n", unit);
}
static void
unit_invalidate(vtd_unit_t * unit,
uint64_t did, ppnum_t addr, ppnum_t mask, bool leaf)
{
if (unit->selective)
{
unit->iotlb->address = ptoa_64(addr) | (leaf << 6) | mask;
__mfence();
unit->iotlb->command = (1ULL<<63) | (3ULL<<60) | (kTlbDrainReads<<49) | (kTlbDrainWrites<<48) | (did << 32);
}
else unit->iotlb->command = (1ULL<<63) | (1ULL<<60) | (kTlbDrainReads<<49) | (kTlbDrainWrites<<48);
__mfence();
}
static void
unit_invalidate_done(vtd_unit_t * unit)
{
while ((1ULL<<63) & unit->iotlb->command) {}
}
typedef uint32_t vtd_vaddr_t;
union vtd_table_entry
{
struct
{
uint read:1 __attribute__ ((packed));
uint write:1 __attribute__ ((packed));
uint resv:10 __attribute__ ((packed));
uint64_t addr:51 __attribute__ ((packed));
uint used:1 __attribute__ ((packed));
} used;
struct
{
uint access:2 __attribute__ ((packed));
uint next:28 __attribute__ ((packed));
uint prev:28 __attribute__ ((packed));
uint size:5 __attribute__ ((packed));
uint free:1 __attribute__ ((packed));
} free;
uint64_t bits;
};
typedef union vtd_table_entry vtd_table_entry_t;
typedef uint32_t vtd_rbaddr_t;
struct vtd_rblock
{
RB_ENTRY(vtd_rblock) address_link;
RB_ENTRY(vtd_rblock) size_link;
vtd_rbaddr_t start;
vtd_rbaddr_t end;
};
RB_HEAD(vtd_rbaddr_list, vtd_rblock);
RB_HEAD(vtd_rbsize_list, vtd_rblock);
struct vtd_space_stats
{
ppnum_t vsize;
ppnum_t tables;
ppnum_t bused;
ppnum_t rused;
ppnum_t largest_paging;
ppnum_t largest_32b;
ppnum_t inserts;
ppnum_t max_inval[2];
ppnum_t breakups;
ppnum_t merges;
ppnum_t allocs[64];
ppnum_t bcounts[20];
};
typedef struct vtd_space_stats vtd_space_stats_t;
struct vtd_free_queued_t
{
ppnum_t addr;
ppnum_t size;
uint32_t stamp;
};
enum
{
kFreeQCount = 2,
kFreeQElems = 256
};
struct vtd_space
{
#if BSIMPLE
IOSimpleLock * block;
#else
IOLock * block;
#endif
IOLock * rlock;
ppnum_t vsize;
ppnum_t rsize;
size_t table_bitmap_size;
uint8_t * table_bitmap;
IOMemoryMap * table_map;
vtd_table_entry_t * tables[6];
uint32_t cachelinesize;
ppnum_t root_page;
uint8_t max_level;
uint8_t waiting_space;
uint8_t bheads_count;
vtd_table_entry_t * bheads;
vtd_space_stats_t stats;
vtd_free_queued_t free_queue[kFreeQCount][kFreeQElems];
volatile uint32_t free_head[kFreeQCount];
volatile uint32_t free_tail[kFreeQCount];
uint32_t free_mask;
uint32_t stamp;
struct vtd_rbaddr_list rbaddr_list;
struct vtd_rbsize_list rbsize_list;
};
typedef struct vtd_space vtd_space_t;
static vtd_vaddr_t
vtd_log2up(vtd_vaddr_t size)
{
if (1 == size) size = 0;
else size = 32 - __builtin_clz((unsigned int)size - 1);
return (size);
}
static vtd_vaddr_t
vtd_log2down(vtd_vaddr_t size)
{
size = 31 - __builtin_clz((unsigned int)size);
return (size);
}
static void
_vtd_space_nfault(vtd_space_t * bf, vtd_vaddr_t start, vtd_vaddr_t size)
{
vtd_vaddr_t index;
vtd_vaddr_t byte;
uint8_t bit;
vtassert((start + size) < bf->vsize);
size += (start & 511);
size = (size + 511) & ~511;
while (true)
{
index = (start >> 9);
byte = (index >> 3);
bit = (1 << (7 & index));
vtassert(bf->table_bitmap[byte] & bit);
if (size < 512) break;
size -= 512;
start += 512;
}
}
static bool
vtd_space_present(vtd_space_t * bf, vtd_vaddr_t start)
{
vtd_vaddr_t byte;
uint8_t bit;
vtassert(start < bf->vsize);
start >>= 9;
byte = (start >> 3);
bit = (1 << (7 & start));
return (bf->table_bitmap[byte] & bit);
}
static void
vtd_space_fault(vtd_space_t * bf, vtd_vaddr_t start, vtd_vaddr_t size)
{
vtd_vaddr_t index;
vtd_vaddr_t byte;
uint8_t bits, bit;
IOReturn kr;
vtassert((start + size) < bf->vsize);
size += (start & 511);
size = (size + 511) & ~511;
while (true)
{
index = (start >> 9);
byte = (index >> 3);
index &= 7;
bits = bf->table_bitmap[byte];
#if 1
if (0xff == bits)
{
index = (8 - index) * 512;
if (size <= index) break;
size -= index;
start += index;
continue;
}
#endif
bit = (1 << index);
if (!(bits & bit))
{
bf->table_bitmap[byte] = bits | bit;
index = start & ~511;
kr = bf->table_map->wireRange(kIODirectionOutIn, index << 3, page_size);
vtassert(kr == KERN_SUCCESS);
STAT_ADD(bf, tables, 1);
bf->tables[0][index].bits = 0;
ppnum_t lvl0page = pmap_find_phys(kernel_pmap, (uintptr_t) &bf->tables[0][index]);
if (!lvl0page) panic("!lvl0page");
bf->tables[1][index >> 9].bits = ptoa_64(lvl0page) | kPageAccess;
table_flush(&bf->tables[1][index >> 9], sizeof(vtd_table_entry_t), bf->cachelinesize);
}
if (size <= 512) break;
size -= 512;
start += 512;
}
}
static void
vtd_space_set(vtd_space_t * bf, vtd_vaddr_t start, vtd_vaddr_t size,
uint32_t mapOptions, upl_page_info_t * pageList)
{
ppnum_t idx;
uint8_t access = kReadAccess | 0*kWriteAccess;
if (kIODMAMapPhysicallyContiguous & mapOptions) VTLOG("map phys %x, %x\n", pageList[0].phys_addr, size);
if (mapOptions & kIODMAMapWriteAccess) access |= kWriteAccess;
vtassert((start + size) <= bf->vsize);
vtd_space_nfault(bf, start, size);
if (kIODMAMapPhysicallyContiguous & mapOptions)
{
for (idx = 0; idx < size; idx++)
{
bf->tables[0][start + idx].bits = (access | ptoa_64(pageList[0].phys_addr + idx));
}
#if TABLE_CB
table_flush(&bf->tables[0][start], size * sizeof(vtd_table_entry_t), bf->cachelinesize);
#endif
}
else
{
#if TABLE_CB
ppnum_t j;
for (idx = 0; size >= 8; size -= 8, idx += 8)
{
j = 0;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr)); j++;
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr));
__mfence();
__clflush((void *) &bf->tables[0][start + idx].bits);
}
if (size)
{
for (j = 0; j < size; j++)
{
bf->tables[0][start + idx + j].bits = (access | ptoa_64(pageList[idx + j].phys_addr));
}
__mfence();
__clflush((void *) &bf->tables[0][start + idx].bits);
}
#else
for (idx = 0; idx < size; idx++)
{
bf->tables[0][start + idx].bits = (access | ptoa_64(pageList[idx].phys_addr));
}
#endif
}
__mfence();
}
#include "balloc.c"
#include "rballoc.c"
class AppleVTD : public IOMapper
{
OSDeclareDefaultStructors(AppleVTD);
public:
IOSimpleLock * fHWLock;
const OSData * fDMARData;
IOWorkLoop * fWorkLoop;
IOInterruptEventSource * fIntES;
IOInterruptEventSource * fFaultES;
IOTimerEventSource * fTimerES;
enum { kMaxUnits = 8 };
vtd_unit_t * units[kMaxUnits];
uint32_t fTreeBits;
uint32_t fMaxRoundSize;
uint32_t fCacheLineSize;
IOMemoryMap * fTableMap;
IOMemoryMap * fContextTableMap;
ppnum_t fRootEntryPage;
vtd_space_t * fSpace;
static void install(IOWorkLoop * wl, uint32_t flags,
IOService * provider, const OSData * data);
bool init(IOWorkLoop * wl, const OSData * data);
virtual void free();
virtual bool initHardware(IOService *provider);
vtd_space_t * space_create(uint32_t cachelinesize, uint32_t treebits, ppnum_t vsize,
uint32_t buddybits, ppnum_t rsize);
vtd_vaddr_t space_alloc(vtd_space_t * bf, vtd_vaddr_t size,
uint32_t mapOptions, const IODMAMapSpecification * mapSpecification,
upl_page_info_t * pageList);
void space_free(vtd_space_t * bf, vtd_vaddr_t addr, vtd_vaddr_t size);
void space_alloc_fixed(vtd_space_t * bf, vtd_vaddr_t addr, vtd_vaddr_t size);
IOReturn handleInterrupt(IOInterruptEventSource * source, int count);
IOReturn handleFault(IOInterruptEventSource * source, int count);
IOReturn timer(OSObject * owner, IOTimerEventSource * sender);
virtual IOReturn callPlatformFunction(const OSSymbol * functionName,
bool waitForFunction,
void * param1, void * param2,
void * param3, void * param4);
void iovmInvalidateSync(ppnum_t addr, IOItemCount pages);
void checkFree(uint32_t queue);
virtual ppnum_t iovmAlloc(IOItemCount pages);
virtual void iovmFree(ppnum_t addr, IOItemCount pages);
virtual void iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page);
virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
ppnum_t *pageList, IOItemCount pageCount);
virtual void iovmInsert(ppnum_t addr, IOItemCount offset,
upl_page_info_t *pageList, IOItemCount pageCount);
virtual ppnum_t iovmMapMemory(
OSObject * memory, ppnum_t offsetPage,
ppnum_t pageCount,
uint32_t options,
upl_page_info_t * pageList,
const IODMAMapSpecification * mapSpecification);
virtual addr64_t mapAddr(IOPhysicalAddress addr);
};
OSDefineMetaClassAndStructors(AppleVTD, IOMapper);
#define super IOMapper
void
AppleVTD::install(IOWorkLoop * wl, uint32_t flags,
IOService * provider, const OSData * data)
{
AppleVTD * mapper = 0;
bool ok = false;
if (!IOService::getPlatform()->getProperty(kIOPlatformMapperPresentKey)) return;
VTLOG("DMAR %p\n", data);
if (data)
{
mapper = new AppleVTD;
if (mapper)
{
if (mapper->init(wl, data) && mapper->attach(provider))
{
ok = mapper->start(provider);
if (!ok) mapper->detach(provider);
}
mapper->release();
}
}
if (!ok)
{
IOService::getPlatform()->removeProperty(kIOPlatformMapperPresentKey);
IOMapper::setMapperRequired(false);
}
}
bool
AppleVTD::init(IOWorkLoop * wl, const OSData * data)
{
uint32_t unitIdx;
if (!super::init()) return (false);
data->retain();
fDMARData = data;
wl->retain();
fWorkLoop = wl;
fCacheLineSize = cpuid_info()->cache_linesize;
ACPI_TABLE_DMAR * dmar = (typeof(dmar)) data->getBytesNoCopy();
ACPI_DMAR_HEADER * dmarEnd = (typeof(dmarEnd))(((uintptr_t) dmar) + data->getLength());
ACPI_DMAR_HEADER * hdr = (typeof(hdr)) (dmar + 1);
ACPI_DMAR_HARDWARE_UNIT * unit;
VTLOG("DMAR Width %x, Flags %x\n", dmar->Width, dmar->Flags);
for (unitIdx = 0; hdr < dmarEnd;
hdr = (typeof(hdr))(((uintptr_t) hdr) + hdr->Length))
{
switch (hdr->Type)
{
case ACPI_DMAR_TYPE_HARDWARE_UNIT:
unit = (typeof(unit)) hdr;
if ((units[unitIdx] = unit_init(unit))) unitIdx++;
break;
}
}
return (unitIdx != 0);
}
void AppleVTD::free()
{
super::free();
}
vtd_space_t *
AppleVTD::space_create(uint32_t cachelinesize,
uint32_t treebits, ppnum_t vsize, uint32_t buddybits, ppnum_t rsize)
{
IOBufferMemoryDescriptor * md;
IOReturn kr = kIOReturnSuccess;
vtd_space_t * bf;
uint32_t count;
mach_vm_size_t alloc;
uint32_t level;
uint32_t bit;
vtassert(vsize >= (1U << buddybits));
vtassert(vsize > rsize);
vtassert(buddybits > (9 + 3));
vtassert(treebits > 12);
bf = IONew(vtd_space_t, 1);
if (!bf) return (NULL);
bzero(bf, sizeof(vtd_space_t));
bf->rlock = IOLockAlloc();
#if BSIMPLE
bf->block = fHWLock;
#else
bf->block = IOLockAlloc();
#endif
bf->cachelinesize = cachelinesize;
treebits -= 12;
vsize = (vsize + 511) & ~511;
bf->vsize = vsize;
bf->table_bitmap_size = ((vsize / 512) + 7) / 8;
bf->table_bitmap = IONew(uint8_t, bf->table_bitmap_size);
if (!bf->table_bitmap) return (NULL);
bzero(bf->table_bitmap, bf->table_bitmap_size);
alloc = 0;
level = 0;
bit = 0;
while (bit < treebits)
{
count = (vsize >> bit);
if (!count) count = 1;
alloc += round_page_64(count * sizeof(vtd_table_entry_t));
bit += 9;
level++;
}
bf->max_level = level - 1;
VTLOG("level %d, bmd...0x%llx\n", bf->max_level, alloc);
md = IOBufferMemoryDescriptor::inTaskWithOptions(TASK_NULL,
kIOMemoryPageable |
#if !TABLE_CB
kIOMapWriteCombineCache |
#endif
kIOMemoryMapperNone,
alloc, page_size);
VTLOG("bmd %p\n", md);
vtassert(md);
if (!md) return (NULL);
bf->table_map = md->map();
vtassert(bf->table_map);
md->release();
vtassert(bf->table_map);
if (!bf->table_map) return (NULL);
vtd_table_entry_t * table;
table = (typeof(table)) bf->table_map->getVirtualAddress();
vtd_table_entry_t * prior = NULL;
vtd_table_entry_t * next = table;
mach_vm_size_t offset;
uint32_t idx;
level = 0;
bit = 0;
while (bit < treebits)
{
count = (vsize >> bit);
if (!count) count = 1;
vtassert(level < arrayCount(bf->tables));
vtassert(level <= bf->max_level);
bf->tables[level] = next;
if (level == 1)
{
offset = ((next - table) * sizeof(vtd_table_entry_t));
VTLOG("wire [%llx, %llx]\n", offset, alloc);
kr = bf->table_map->wireRange(kIODirectionOutIn, offset, alloc - offset);
vtassert(KERN_SUCCESS == kr);
STAT_ADD(bf, tables, atop_64(alloc - offset));
if (KERN_SUCCESS != kr)
{
bf->table_map->release();
return (NULL);
}
}
else if (level >= 2)
{
for (idx = 0; idx < count; idx++)
{
ppnum_t lvl2page = pmap_find_phys(kernel_pmap, (uintptr_t) &prior[idx << 9]);
if (!lvl2page) panic("!lvl2page");
VTLOG("lvl2 %p[%x] = %p\n", next, idx, &prior[idx << 9]);
next[idx].bits = (kPageAccess | ptoa_64(lvl2page));
}
}
prior = next;
next = next + ((count + 511) & ~511);
bit += 9;
level++;
}
table_flush(&bf->tables[1][0], alloc - offset, bf->cachelinesize);
#if !TABLE_CB
IOSetProcessorCacheMode(kernel_task, (IOVirtualAddress) &bf->tables[0][0], page_size, kIOCopybackCache);
#endif
VTLOG("tables %p, %p, %p, %p, %p, %p\n", bf->tables[0], bf->tables[1], bf->tables[2],
bf->tables[3], bf->tables[4], bf->tables[5]);
bf->root_page = pmap_find_phys(kernel_pmap, (uintptr_t) bf->tables[bf->max_level]);
if (!bf->root_page) panic("!root_page");
VTLOG("tree root 0x%llx\n", ptoa_64(bf->root_page));
vtd_ballocator_init(bf, buddybits);
bf->rsize = rsize;
vtd_rballocator_init(bf, rsize, vsize - rsize);
VTLOG("bsize 0x%x, bsafe 0x%x, breserve 0x%x, rsize 0x%x\n",
(1<<kBPagesLog2), kBPagesSafe, kBPagesReserve, bf->rsize);
STAT_ADD(bf, vsize, vsize);
OSData *
data = OSData::withBytesNoCopy(&bf->stats, sizeof(bf->stats));
if (data)
{
setProperty("stats", data);
data->release();
}
bf->stamp = 0x100;
bf->free_mask = (kFreeQElems - 1);
return (bf);
}
vtd_baddr_t
AppleVTD::space_alloc(vtd_space_t * bf, vtd_baddr_t size,
uint32_t mapOptions, const IODMAMapSpecification * mapSpecification,
upl_page_info_t * pageList)
{
vtd_vaddr_t addr;
vtd_vaddr_t align = 1;
vtd_baddr_t largethresh;
bool uselarge;
uint32_t list;
if ((kIODMAMapPagingPath & mapOptions) && (size > bf->stats.largest_paging))
bf->stats.largest_paging = size;
list = vtd_log2up(size);
if (mapSpecification)
{
if (mapSpecification->numAddressBits
&& (mapSpecification->numAddressBits <= 32)
&& (size > bf->stats.largest_32b)) bf->stats.largest_32b = size;
if (mapSpecification->alignment > page_size) align = atop_64(mapSpecification->alignment);
}
if (bf->stats.bused >= kBPagesReserve)
{
largethresh = 1;
}
else if (bf->stats.bused >= kBPagesSafe)
{
largethresh = kLargeThresh2;
}
else
{
largethresh = kLargeThresh;
}
if (!(kIODMAMapPagingPath & mapOptions)
&& (size >= largethresh)
&& mapSpecification
&& mapSpecification->numAddressBits
&& ((1ULL << (mapSpecification->numAddressBits - 12)) >= bf->vsize))
{
uselarge = true;
}
else
{
uselarge = false;
if (align > size) size = align;
}
#if 0
IOSimpleLockLock(fHWLock);
checkFree(uselarge);
IOSimpleLockUnlock(fHWLock);
#endif
do
{
if (uselarge)
{
IOLockLock(bf->rlock);
addr = vtd_rballoc(bf, size, align, fMaxRoundSize, mapOptions, pageList);
STAT_ADD(bf, allocs[list], 1);
if (addr)
{
STAT_ADD(bf, rused, size);
vtd_space_fault(bf, addr, size);
}
IOLockUnlock(bf->rlock);
if (addr && pageList) vtd_space_set(bf, addr, size, mapOptions, pageList);
}
else
{
BLOCK(bf->block);
addr = vtd_balloc(bf, size, mapOptions, pageList);
STAT_ADD(bf, allocs[list], 1);
if (addr) STAT_ADD(bf, bused, (1 << list));
BUNLOCK(bf->block);
}
if (addr) break;
if (!uselarge && (size >= (1 << (kBPagesLog2 - 2)))) break;
IOLockLock(bf->rlock);
bf->waiting_space = true;
IOLockSleep(bf->rlock, &bf->waiting_space, THREAD_UNINT);
IOLockUnlock(bf->rlock);
VTLOG("AppleVTD: waiting space (%d, bused %d, rused %d)\n",
size, bf->stats.bused, bf->stats.rused);
}
while (true);
return (addr);
}
void
AppleVTD::space_free(vtd_space_t * bf, vtd_baddr_t addr, vtd_baddr_t size)
{
uint32_t list;
vtassert(addr);
vtassert((addr + size) <= bf->vsize);
if (addr >= bf->rsize)
{
IOLockLock(bf->rlock);
vtd_rbfree(bf, addr, size, fMaxRoundSize);
STAT_ADD(bf, rused, -size);
IOLockUnlock(bf->rlock);
}
else
{
list = vtd_log2up(size);
BLOCK(bf->block);
vtd_bfree(bf, addr, size);
STAT_ADD(bf, bused, -(1 << list));
BUNLOCK(bf->block);
}
if (bf->waiting_space)
{
IOLockLock(bf->rlock);
bf->waiting_space = false;
IOLockWakeup(bf->rlock, &bf->waiting_space, false);
IOLockUnlock(bf->rlock);
}
}
void
AppleVTD::space_alloc_fixed(vtd_space_t * bf, vtd_baddr_t addr, vtd_baddr_t size)
{
vtd_balloc_fixed(bf, addr, size);
vtd_rballoc_fixed(bf, addr, size);
vtd_space_fault(bf, addr, size);
}
static page_entry_t
vtd_tree_read(page_entry_t root, uint32_t width, addr64_t addr)
{
page_entry_t entry = root;
page_entry_t table;
uint32_t index;
uint32_t level = 0;
while (width > 12)
{
width -= 9;
index = (addr >> (width - 3)) & (511 << 3);
table = entry & kPageAddrMask;
entry = ml_phys_read_double_64(table + index);
if (!(kPageAccess & entry))
break;
level++;
}
return (entry);
}
bool
AppleVTD::initHardware(IOService *provider)
{
uint32_t idx;
vtd_unit_t * unit;
fIsSystem = true;
uint64_t context_width;
fTreeBits = 0;
unit = units[0];
for (context_width = kAddressWidth30;
(context_width <= kAddressWidth64);
context_width++)
{
if ((0x100 << context_width) & unit->regs->capability)
{
fTreeBits = (30 + 9 * context_width); break;
}
}
for (idx = 0; (unit = units[idx]); idx++)
{
if (!((0x100 << context_width) & unit->regs->capability))
panic("!tree bits %d on unit %d", fTreeBits, idx);
if (unit->selective && ((unit->rounding > fMaxRoundSize)))
fMaxRoundSize = unit->rounding;
}
VTLOG("context_width %lld, treebits %d, round %d\n",
context_width, fTreeBits, fMaxRoundSize);
if (!fMaxRoundSize) return (false);
if ((48 == fTreeBits) && (9 == fMaxRoundSize)) return (false);
fHWLock = IOSimpleLockAlloc();
fSpace = space_create(fCacheLineSize, fTreeBits, kVPages, kBPagesLog2, kRPages);
if (!fSpace) return (false);
space_alloc_fixed(fSpace, atop_64(0xfee00000), atop_64(0xfef00000-0xfee00000));
vtd_space_fault(fSpace, atop_64(0xfee00000), 1);
fSpace->tables[0][atop_64(0xfee00000)].bits = 0xfee00000 | kPageAccess;
ACPI_TABLE_DMAR * dmar = (typeof(dmar)) fDMARData->getBytesNoCopy();
ACPI_DMAR_HEADER * dmarEnd = (typeof(dmarEnd))(((uintptr_t) dmar) + fDMARData->getLength());
ACPI_DMAR_HEADER * hdr = (typeof(hdr)) (dmar + 1);
ACPI_DMAR_RESERVED_MEMORY * mem;
for (; hdr < dmarEnd;
hdr = (typeof(hdr))(((uintptr_t) hdr) + hdr->Length))
{
uint64_t addr;
uint32_t count;
switch (hdr->Type)
{
case ACPI_DMAR_TYPE_RESERVED_MEMORY:
mem = (typeof(mem)) hdr;
VTLOG("ACPI_DMAR_TYPE_RESERVED_MEMORY 0x%llx, 0x%llx\n",
mem->BaseAddress, mem->EndAddress);
addr = mem->BaseAddress;
count = atop_32(mem->EndAddress - addr);
space_alloc_fixed(fSpace, atop_64(addr), count);
for (; count; addr += page_size, count--)
{
fSpace->tables[0][atop_64(addr)].bits = (addr | kPageAccess);
}
break;
}
}
IOReturn kr;
IOBufferMemoryDescriptor *
md = IOBufferMemoryDescriptor::inTaskWithOptions(TASK_NULL,
kIOMemoryPageable |
kIOMapWriteCombineCache |
kIOMemoryMapperNone,
2 * page_size, page_size);
vtassert(md);
if (!md) return (kIOReturnNoMemory);
kr = md->prepare(kIODirectionOutIn);
vtassert(KERN_SUCCESS == kr);
fContextTableMap = md->map();
vtassert(fContextTableMap);
md->release();
context_entry_t * context_entry_table = (typeof(context_entry_table)) fContextTableMap->getVirtualAddress();
for (idx = 0; idx < 256; idx++)
{
context_entry_table[idx].address_space_root = ptoa_64(fSpace->root_page)
| kEntryPresent
| kTranslationType0;
context_entry_table[idx].context_entry = context_width
| kTheDomain*kDomainIdentifier1;
if (!(kIOPCIConfiguratorIGIsMapped & gIOPCIFlags))
{
if (idx == ((2<<3)|0)) context_entry_table[idx].address_space_root &= ~kEntryPresent;
}
}
ppnum_t context_page = pmap_find_phys(kernel_pmap, (uintptr_t) &context_entry_table[0]);
if (!context_page) panic("!context_page");
root_entry_t * root_entry_table = (typeof(root_entry_table)) (fContextTableMap->getAddress() + page_size);
for (idx = 0; idx < 256; idx++)
{
root_entry_table[idx].context_entry_ptr = ptoa_64(context_page)
| kEntryPresent;
root_entry_table[idx].resv = 0;
}
fRootEntryPage = pmap_find_phys(kernel_pmap, (uintptr_t) &root_entry_table[0]);
if (!fRootEntryPage) panic("!fRootEntryPage");
for (idx = 0; (unit = units[idx]); idx++)
{
unit->root = ptoa_64(fRootEntryPage);
}
for (idx = 0; (unit = units[idx]); idx++)
{
md = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,
kIOMemoryHostPhysicallyContiguous |
kIOMapWriteCombineCache |
kIOMemoryMapperNone,
kQIPageCount * page_size, page_size);
vtassert(md);
if (!md) return (kIOReturnNoMemory);
kr = md->prepare(kIODirectionOutIn);
vtassert(KERN_SUCCESS == kr);
unit->qi_map = md->map();
vtassert(unit->qi_map);
unit->qi_mask = (kQIPageCount * 256) - 1;
unit->qi_table = (typeof(unit->qi_table)) (unit->qi_map->getAddress());
unit->qi_address = vtd_log2down(kQIPageCount)
| md->getPhysicalSegment(0, NULL, kIOMemoryMapperNone);
ppnum_t stamp_page = pmap_find_phys(kernel_pmap, (uintptr_t) &unit->qi_stamp);
vtassert(stamp_page);
unit->qi_stamp_address = ptoa_64(stamp_page) | (page_mask & ((uintptr_t) &unit->qi_stamp));
md->release();
}
IOReturn ret;
uint64_t msiAddress;
uint32_t msiData;
ret = gIOPCIMessagedInterruptController->allocateDeviceInterrupts(
this, 2, 0, &msiAddress, &msiData);
if (kIOReturnSuccess == ret)
{
fIntES = IOInterruptEventSource::interruptEventSource(
this,
OSMemberFunctionCast(IOInterruptEventSource::Action,
this, &AppleVTD::handleInterrupt),
this, 0);
if (fIntES) fWorkLoop->addEventSource(fIntES);
fFaultES = IOInterruptEventSource::interruptEventSource(
this,
OSMemberFunctionCast(IOInterruptEventSource::Action,
this, &AppleVTD::handleFault),
this, 1);
if (fFaultES) fWorkLoop->addEventSource(fFaultES);
}
fTimerES = IOTimerEventSource::timerEventSource(this,
OSMemberFunctionCast(IOTimerEventSource::Action,
this, &AppleVTD::timer));
if (fTimerES) fWorkLoop->addEventSource(fTimerES);
if (!fIntES || !fFaultES) msiData = msiAddress = 0;
__mfence();
for (idx = 0; (unit = units[idx]); idx++)
{
unit->msi_data = msiData & 0xff;
unit->msi_address = msiAddress;
unit_enable(unit);
}
if (fIntES) fIntES->enable();
if (fFaultES) fFaultES->enable();
setProperty(kIOPlatformQuiesceActionKey, INT32_MAX - 1000, 64);
setProperty(kIOPlatformActiveActionKey, INT32_MAX - 1000, 64);
registerService();
return (true);
}
IOReturn
AppleVTD::handleInterrupt(IOInterruptEventSource * source, int count)
{
uint32_t idx;
vtd_unit_t * unit;
IOSimpleLockLock(fHWLock);
for (idx = 0; idx < kFreeQCount; idx++) checkFree(idx);
for (idx = 0; (unit = units[idx]); idx++)
{
unit->regs->invalidation_completion_status = 1;
}
IOSimpleLockUnlock(fHWLock);
return (kIOReturnSuccess);
}
IOReturn
AppleVTD::handleFault(IOInterruptEventSource * source, int count)
{
uint32_t idx;
vtd_unit_t * unit;
for (idx = 0; (unit = units[idx]); idx++) unit_faults(unit, true || (idx != 0));
return (kIOReturnSuccess);
}
IOReturn
AppleVTD::timer(OSObject * owner, IOTimerEventSource * sender)
{
uint32_t idx;
IOSimpleLockLock(fHWLock);
for (idx = 0; idx < kFreeQCount; idx++) checkFree(idx);
IOSimpleLockUnlock(fHWLock);
fTimerES->setTimeoutMS(10);
return (kIOReturnSuccess);
}
IOReturn
AppleVTD::callPlatformFunction(const OSSymbol * functionName,
bool waitForFunction,
void * param1, void * param2,
void * param3, void * param4)
{
if (functionName)
{
uint32_t idx;
vtd_unit_t * unit;
if (functionName->isEqualTo(gIOPlatformActiveActionKey))
{
for (idx = 0; (unit = units[idx]); idx++)
{
unit_enable(unit);
}
return (kIOReturnSuccess);
}
else if (functionName->isEqualTo(gIOPlatformQuiesceActionKey))
{
for (idx = 0; (unit = units[idx]); idx++)
{
unit_quiesce(unit);
}
return (kIOReturnSuccess);
}
}
return (super::callPlatformFunction(functionName, waitForFunction,
param1, param2, param3, param4));
}
ppnum_t
AppleVTD::iovmMapMemory(
OSObject * memory, ppnum_t offsetPage,
ppnum_t pageCount,
uint32_t mapOptions,
upl_page_info_t * pageList,
const IODMAMapSpecification * mapSpecification)
{
vtd_vaddr_t base;
base = space_alloc(fSpace, pageCount, mapOptions, mapSpecification, pageList);
vtassert((base + pageCount) <= fSpace->vsize);
#if KP
VTLOG("iovmMapMemory: (0x%x)=0x%x\n", length, (int)base);
#endif
return (base);
}
ppnum_t
AppleVTD::iovmAlloc(IOItemCount pages)
{
ppnum_t result;
result = space_alloc(fSpace, pages, 0, NULL, NULL);
#if KP
VTLOG("iovmAlloc: 0x%x=0x%x\n", (int)pages, (int)result );
#endif
return (result);
}
void
AppleVTD::iovmInvalidateSync(ppnum_t addr, IOItemCount pages)
{
vtd_unit_t * unit;
unsigned int leaf;
unsigned int idx;
uint32_t wait;
ppnum_t unitAddr[kMaxUnits];
IOItemCount unitPages[kMaxUnits];
bool more;
for (idx = 0; (unit = units[idx]); idx++)
{
unitAddr[idx] = addr;
unitPages[idx] = pages;
}
leaf = true;
do
{
more = false;
wait = 0;
for (idx = 0; (unit = units[idx]); idx++)
{
if (unitPages[idx])
{
wait |= (1 << idx);
unit_invalidate(unit, kTheDomain, unitAddr[idx], unit->rounding, leaf);
if (!unit->selective
|| (unitPages[idx] <= (1U << unit->rounding)))
{
unitPages[idx] = 0;
}
else
{
more = true;
unitPages[idx] -= (1U << unit->rounding);
unitAddr[idx] += (1U << unit->rounding);
}
}
}
for (idx = 0; (unit = units[idx]); idx++)
{
if (wait & (1U << idx)) unit_invalidate_done(unit);
}
}
while (more);
}
void
AppleVTD::iovmFree(ppnum_t addr, IOItemCount pages)
{
vtd_unit_t * unit;
unsigned int leaf, isLarge;
unsigned int unitIdx;
uint32_t did = kTheDomain;
ppnum_t unitAddr;
IOItemCount unitPages;
uint32_t idx;
uint32_t next;
uint32_t count;
uint64_t stamp;
#if KP
VTLOG("iovmFree: 0x%x,0x%x\n", (int)pages, addr);
#endif
vtassert((addr + pages) <= fSpace->vsize);
vtd_space_nfault(fSpace, addr, pages);
bzero(&fSpace->tables[0][addr], pages * sizeof(vtd_table_entry_t));
table_flush(&fSpace->tables[0][addr], pages * sizeof(vtd_table_entry_t), fCacheLineSize);
#if !ENA_QI
IOSimpleLockLock(fHWLock);
iovmInvalidateSync(addr, pages);
IOSimpleLockUnlock(fHWLock);
space_free(fSpace, addr, pages);
return;
#else
leaf = true;
isLarge = (addr >= fSpace->rsize);
IOSimpleLockLock(fHWLock);
#if 0
int32_t freeCount;
freeCount = fSpace->free_tail[isLarge] - fSpace->free_head[isLarge];
if (freeCount < 0) freeCount = kFreeQElems - freeCount;
if (freeCount >= 8)
#endif
{
checkFree(isLarge);
}
stamp = ++fSpace->stamp;
idx = fSpace->free_tail[isLarge];
next = (idx + 1) & fSpace->free_mask;
if (next == fSpace->free_head[isLarge]) panic("qfull");
fSpace->free_queue[isLarge][idx].addr = addr;
fSpace->free_queue[isLarge][idx].size = pages;
fSpace->free_queue[isLarge][idx].stamp = stamp;
fSpace->free_tail[isLarge] = next;
for (unitIdx = 0; (unit = units[unitIdx]); unitIdx++)
{
unitAddr = addr;
unitPages = pages;
idx = unit->qi_tail;
count = 0;
while (unitPages)
{
next = (idx + 1) & unit->qi_mask;
while ((next << 4) == unit->regs->invalidation_queue_head) {}
if (unit->selective)
{
uint32_t mask = unit->rounding;
if (unitPages < (1U << unit->rounding)) mask = vtd_log2up(unitPages);
unit->qi_table[idx].command = (did<<16) | (kTlbDrainReads<<7) | (kTlbDrainWrites<<6) | (3<<4) | (2);
unit->qi_table[idx].address = ptoa_64(unitAddr) | (leaf << 6) | mask;
}
else
{
unit->qi_table[idx].command = (kTlbDrainReads<<7) | (kTlbDrainWrites<<6) | (1<<4) | (2);
}
if (!unit->selective
|| (unitPages <= (1U << unit->rounding)))
{
unitPages = 0;
}
else
{
unitPages -= (1U << unit->rounding);
unitAddr += (1U << unit->rounding);
count++;
if (!(count & (unit->qi_mask >> 5)))
{
__mfence();
unit->regs->invalidation_queue_tail = (next << 4);
}
}
idx = next;
}
{
next = (idx + 1) & unit->qi_mask;
while ((next << 4) == unit->regs->invalidation_queue_head) {}
uint64_t command = (stamp<<32) | (1<<5) | (5);
unit->qi_table[idx].command = command;
unit->qi_table[idx].address = unit->qi_stamp_address;
}
__mfence();
unit->regs->invalidation_queue_tail = (next << 4);
unit->qi_tail = next;
}
IOSimpleLockUnlock(fHWLock);
#endif
}
#define stampPassed(a,b) (((int32_t)((a)-(b))) >= 0)
void
AppleVTD::checkFree(uint32_t isLarge)
{
vtd_unit_t * unit;
uint32_t unitIdx;
uint32_t idx;
uint32_t next;
ppnum_t addr, size, count;
bool ok;
count = 0;
idx = fSpace->free_head[isLarge];
do
{
if (idx == fSpace->free_tail[isLarge]) break;
for (unitIdx = 0, ok = true; ok && (unit = units[unitIdx]); unitIdx++)
{
ok &= stampPassed(unit->qi_stamp, fSpace->free_queue[isLarge][idx].stamp);
}
if (ok)
{
next = (idx + 1) & fSpace->free_mask;
addr = fSpace->free_queue[isLarge][idx].addr;
size = fSpace->free_queue[isLarge][idx].size;
#if BSIMPLE
if (!isLarge)
{
vtd_bfree(fSpace, addr, size);
STAT_ADD(fSpace, bused, -size);
idx = next;
}
else
#endif
{
fSpace->free_head[isLarge] = next;
IOSimpleLockUnlock(fHWLock);
space_free(fSpace, addr, size);
IOSimpleLockLock(fHWLock);
idx = fSpace->free_head[isLarge];
}
count++;
}
}
while (ok);
#if BSIMPLE
fSpace->free_head[isLarge] = idx;
#endif
if (count > fSpace->stats.max_inval[isLarge]) fSpace->stats.max_inval[isLarge] = count;
}
addr64_t
AppleVTD::mapAddr(IOPhysicalAddress addr)
{
ppnum_t page = atop_64(addr);
page_entry_t entry;
if (page >= fSpace->vsize) return (addr);
if (!vtd_space_present(fSpace, page)) return (addr);
entry = fSpace->tables[0][page].bits;
#if KP
VTLOG("mapAddr: 0x%x=0x%llx\n", (int)addr, entry);
#endif
if (kPageAccess & entry)
return (trunc_page_64(entry) | (addr & page_mask));
else
return (addr);
return (0);
}
void
AppleVTD::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page)
{
addr += offset;
vtassert(addr < fSpace->vsize);
vtd_space_nfault(fSpace, addr, 1);
fSpace->tables[0][addr].bits = ptoa_64(page) | kPageAccess;
table_flush(&fSpace->tables[0][addr], sizeof(vtd_table_entry_t), fCacheLineSize);
STAT_ADD(fSpace, inserts, 1);
}
void
AppleVTD::iovmInsert(ppnum_t addr, IOItemCount offset,
ppnum_t *pageList, IOItemCount pageCount)
{
ppnum_t idx;
addr += offset;
vtassert((addr + pageCount) <= fSpace->vsize);
vtd_space_nfault(fSpace, addr, pageCount);
for (idx = 0; idx < pageCount; idx++)
{
fSpace->tables[0][addr + idx].bits = ptoa_64(pageList[idx]) | kPageAccess;
}
table_flush(&fSpace->tables[0][addr], pageCount * sizeof(vtd_table_entry_t), fCacheLineSize);
STAT_ADD(fSpace, inserts, pageCount);
}
void
AppleVTD::iovmInsert(ppnum_t addr, IOItemCount offset,
upl_page_info_t *pageList, IOItemCount pageCount)
{
ppnum_t idx;
addr += offset;
vtassert((addr + pageCount) <= fSpace->vsize);
vtd_space_nfault(fSpace, addr, pageCount);
for (idx = 0; idx < pageCount; idx++)
{
fSpace->tables[0][addr + idx].bits = ptoa_64(pageList[idx].phys_addr) | kPageAccess;
}
table_flush(&fSpace->tables[0][addr], pageCount * sizeof(vtd_table_entry_t), fCacheLineSize);
STAT_ADD(fSpace, inserts, pageCount);
}
#endif