IOMbufMemoryCursor.cpp [plain text]
#include <sys/cdefs.h>
__BEGIN_DECLS
#include <IOKit/assert.h>
#include <sys/param.h>
#include <sys/mbuf.h>
struct mbuf * m_getpackets(int num_needed, int num_with_pkthdrs, int how);
__END_DECLS
#include <IOKit/network/IOMbufMemoryCursor.h>
#include <IOKit/IOLib.h>
#ifndef MIN
#define MIN(a,b) (((a)<(b))?(a):(b))
#endif
#define next_page(x) trunc_page_32(x + PAGE_SIZE)
#if 0
#define ERROR_LOG(args...) IOLog(args)
#else
#define ERROR_LOG(args...)
#endif
OSDefineMetaClassAndStructors(IOMbufMemoryCursor, IOMemoryCursor)
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 0);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 1);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 2);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 3);
OSDefineMetaClassAndStructors(IOMbufNaturalMemoryCursor, IOMbufMemoryCursor)
OSDefineMetaClassAndStructors(IOMbufBigMemoryCursor, IOMbufMemoryCursor)
OSDefineMetaClassAndStructors(IOMbufLittleMemoryCursor, IOMbufMemoryCursor)
#ifdef __ppc__
OSDefineMetaClassAndStructors(IOMbufDBDMAMemoryCursor, IOMbufMemoryCursor)
#endif
#define super IOMemoryCursor
bool IOMbufMemoryCursor::initWithSpecification(OutputSegmentFunc outSeg,
UInt32 maxSegmentSize,
UInt32 maxTransferSize,
UInt32 align)
{
return false;
}
bool IOMbufMemoryCursor::initWithSpecification(OutputSegmentFunc inOutSeg,
UInt32 inMaxSegmentSize,
UInt32 inMaxNumSegments)
{
if (!super::initWithSpecification(inOutSeg, inMaxSegmentSize, 0, 1))
return false;
#if 0
assert(inMaxSegmentSize >= PAGE_SIZE);
if (inMaxSegmentSize < PAGE_SIZE)
return false;
#else
if (!inMaxSegmentSize)
return false;
#endif
maxSegmentSize = MIN(maxSegmentSize, PAGE_SIZE);
maxNumSegments = inMaxNumSegments;
coalesceCount = 0;
return true;
}
#define BCOPY(s, d, l) do { bcopy((void *) s, (void *) d, l); } while(0)
static inline void coalesceSegments(struct mbuf *srcm, struct mbuf *dstm)
{
vm_offset_t src, dst;
SInt32 srcLen, dstLen;
struct mbuf *temp;
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
for (;;) {
if (srcLen < dstLen) {
BCOPY(src, dst, srcLen);
dst += srcLen;
dstLen -= srcLen;
temp = srcm->m_next; assert(temp);
srcm = temp;
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
}
else if (srcLen > dstLen) {
BCOPY(src, dst, dstLen);
src += dstLen;
srcLen -= dstLen;
temp = dstm->m_next; assert(temp);
dstm = temp;
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
}
else {
BCOPY(src, dst, srcLen);
srcm = srcm->m_next;
if (!dstm->m_next)
break;
dstm = dstm->m_next;
assert(srcm);
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
}
}
}
static const UInt32 kMBufDataCacheSize = 16;
static inline bool analyseSegments(
struct mbuf *packet,
const UInt32 mbufsInCache,
const UInt32 segsPerMBuf[],
SInt32 numSegs,
const UInt32 maxSegs)
{
struct mbuf *newPacket; struct mbuf *out; SInt32 outSize; SInt32 outSegs; SInt32 doneSegs; SInt32 outLen;
struct mbuf *in = packet; UInt32 inIndex = 0;
MGET(newPacket, M_DONTWAIT, MT_DATA);
if (!newPacket) {
ERROR_LOG("analyseSegments: MGET() 1 error\n");
return false;
}
out = newPacket;
outSize = MLEN;
doneSegs = outSegs = outLen = 0;
numSegs -= maxSegs;
do {
vm_offset_t vmo;
outLen += in->m_len;
while (outLen > outSize) {
if (outSize != MCLBYTES) {
MCLGET(out, M_DONTWAIT);
if ( !(out->m_flags & M_EXT) ) {
ERROR_LOG("analyseSegments: MCLGET() error\n");
goto bombAnalysis;
}
outSize = MCLBYTES;
continue;
}
vmo = mtod(out, vm_offset_t);
out->m_len = MCLBYTES;
doneSegs += (round_page_32(vmo + MCLBYTES) - trunc_page_32(vmo))
/ PAGE_SIZE;
if (doneSegs + 1 > (int) maxSegs) {
ERROR_LOG("analyseSegments: maxSegs limit 1 reached! %ld %ld\n",
doneSegs, maxSegs);
goto bombAnalysis;
}
MGET(out->m_next, M_DONTWAIT, MT_DATA);
if (!out->m_next) {
ERROR_LOG("analyseSegments: MGET() error\n");
goto bombAnalysis;
}
out = out->m_next;
outSize = MLEN;
outLen -= MCLBYTES;
}
vmo = mtod(out, vm_offset_t);
outSegs = (round_page_32(vmo + outLen) - trunc_page_32(vmo)) / PAGE_SIZE;
if (doneSegs + outSegs > (int) maxSegs) {
ERROR_LOG("analyseSegments: maxSegs limit 2 reached! %ld %ld %ld\n",
doneSegs, outSegs, maxSegs);
goto bombAnalysis;
}
if (inIndex < mbufsInCache)
numSegs -= segsPerMBuf[inIndex]; else {
int thisLen = 0, mbufLen;
vmo = mtod(in, vm_offset_t);
for (mbufLen = in->m_len; mbufLen; mbufLen -= thisLen) {
thisLen = MIN(next_page(vmo), vmo + mbufLen) - vmo;
vmo += thisLen;
numSegs--;
}
}
in = in->m_next;
inIndex++;
} while (in && ((numSegs + doneSegs + outSegs) > 0));
if ( (int) (numSegs + doneSegs + outSegs) <= 0) {
out->m_len = outLen;
coalesceSegments(packet, newPacket);
struct mbuf *m = packet->m_next;
while (m != in)
m = m_free(m);
packet->m_len = 0;
packet->m_next = newPacket;
newPacket->m_next = in;
return true;
}
bombAnalysis:
m_freem(newPacket);
return false;
}
UInt32 IOMbufMemoryCursor::genPhysicalSegments(struct mbuf *packet, void *vector,
UInt32 maxSegs, bool doCoalesce)
{
bool doneCoalesce = false;
if (!packet || !(packet->m_flags & M_PKTHDR))
return 0;
if (!maxSegs)
{
maxSegs = maxNumSegments;
if (!maxSegs) return 0;
}
if ( packet->m_next == 0 )
{
vm_offset_t src;
struct IOPhysicalSegment physSeg;
src = mtod(packet, vm_offset_t);
if ( trunc_page_32(src) == trunc_page_32(src + packet->m_len - 1) )
{
physSeg.location = (IOPhysicalAddress) mcl_to_paddr((char *)src);
if ( physSeg.location )
{
physSeg.length = packet->m_len;
(*outSeg)(physSeg, vector, 0);
return 1;
}
maxSegs = 1;
if ( doCoalesce == false ) return 0;
}
}
if ( doCoalesce == true && maxSegs == 1 )
{
vm_offset_t src;
vm_offset_t dst;
struct mbuf *m;
struct mbuf *mnext;
struct mbuf *out;
UInt32 len = 0;
struct IOPhysicalSegment physSeg;
if ( packet->m_pkthdr.len > MCLBYTES ) return 0;
m = packet;
out = m_getpackets( 1, 0, M_DONTWAIT );
if ( out == 0 ) return 0;
dst = mtod(out, vm_offset_t);
do
{
src = mtod(m, vm_offset_t);
BCOPY( src, dst, m->m_len );
dst += m->m_len;
len += m->m_len;
} while ( (m = m->m_next) != 0 );
out->m_len = len;
dst = mtod(out, vm_offset_t);
physSeg.location = (IOPhysicalAddress) mcl_to_paddr((char *)dst);
if (!physSeg.location)
{
m_free(out);
return 0;
}
physSeg.length = out->m_len;
(*outSeg)(physSeg, vector, 0);
m = packet->m_next;
while (m != 0)
{
mnext = m->m_next;
m_free(m);
m = mnext;
}
packet->m_len = 0;
packet->m_next = out;
out->m_next = 0;
return 1;
}
UInt32 segsPerMBuf[kMBufDataCacheSize];
tryAgain:
UInt32 curMBufIndex = 0;
UInt32 curSegIndex = 0;
UInt32 lastSegCount = 0;
struct mbuf *m = packet;
do {
vm_size_t mbufLen, thisLen = 0;
vm_offset_t src;
for (mbufLen = m->m_len, src = mtod(m, vm_offset_t);
mbufLen;
src += thisLen, mbufLen -= thisLen)
{
thisLen = MIN(mbufLen, maxSegmentSize);
thisLen = MIN(next_page(src), src + thisLen) - src;
if (curSegIndex < maxSegs) {
struct IOPhysicalSegment physSeg;
physSeg.location = (IOPhysicalAddress) mcl_to_paddr((char *)src);
if ( physSeg.location == 0 )
{
return doCoalesce ?
genPhysicalSegments(packet, vector, 1, true) : 0;
}
physSeg.length = thisLen;
(*outSeg)(physSeg, vector, curSegIndex);
}
curSegIndex++;
}
if (curMBufIndex < kMBufDataCacheSize) {
segsPerMBuf[curMBufIndex] = curSegIndex - lastSegCount;
lastSegCount = curSegIndex;
}
curMBufIndex++;
m = m->m_next;
} while (m);
if (curSegIndex <= maxSegs)
return curSegIndex;
if (!doCoalesce)
return 0;
if (!doneCoalesce
&& (UInt) packet->m_pkthdr.len <= maxSegs * maxSegmentSize) {
bool analysisRet;
analysisRet = analyseSegments(packet,
MIN(curMBufIndex, kMBufDataCacheSize),
segsPerMBuf,
curSegIndex, maxSegs);
if (analysisRet) {
doneCoalesce = true;
coalesceCount++;
goto tryAgain;
}
}
assert(!doneCoalesce); packetTooBigErrors++;
return 0;
}
UInt32 IOMbufMemoryCursor::getAndResetCoalesceCount()
{
UInt32 cnt = coalesceCount; coalesceCount = 0; return cnt;
}
IOMbufBigMemoryCursor *
IOMbufBigMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufBigMemoryCursor *me = new IOMbufBigMemoryCursor;
if (me && !me->initWithSpecification(&bigOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
UInt32
IOMbufBigMemoryCursor::getPhysicalSegments(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
UInt32
IOMbufBigMemoryCursor::getPhysicalSegmentsWithCoalesce(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
IOMbufNaturalMemoryCursor *
IOMbufNaturalMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufNaturalMemoryCursor *me = new IOMbufNaturalMemoryCursor;
if (me && !me->initWithSpecification(&naturalOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
UInt32
IOMbufNaturalMemoryCursor::getPhysicalSegments(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
UInt32
IOMbufNaturalMemoryCursor::getPhysicalSegmentsWithCoalesce(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
IOMbufLittleMemoryCursor *
IOMbufLittleMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufLittleMemoryCursor *me = new IOMbufLittleMemoryCursor;
if (me && !me->initWithSpecification(&littleOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
UInt32
IOMbufLittleMemoryCursor::getPhysicalSegments(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
UInt32
IOMbufLittleMemoryCursor::getPhysicalSegmentsWithCoalesce(struct mbuf *packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
#ifdef __ppc__
IOMbufDBDMAMemoryCursor *
IOMbufDBDMAMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufDBDMAMemoryCursor *me = new IOMbufDBDMAMemoryCursor;
if (me && !me->initWithSpecification(&dbdmaOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
UInt32
IOMbufDBDMAMemoryCursor::getPhysicalSegments(struct mbuf *packet,
struct IODBDMADescriptor *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
UInt32
IOMbufDBDMAMemoryCursor::getPhysicalSegmentsWithCoalesce(struct mbuf *packet,
struct IODBDMADescriptor *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
#endif