IOMbufMemoryCursor.cpp [plain text]
#include <sys/cdefs.h>
__BEGIN_DECLS
#include <IOKit/assert.h>
#include <sys/param.h>
#include <sys/mbuf.h>
struct mbuf * m_getpackets(int num_needed, int num_with_pkthdrs, int how);
__END_DECLS
#include <IOKit/network/IOMbufMemoryCursor.h>
#include <IOKit/IOLib.h>
#ifndef MIN
#define MIN(a,b) (((a)<(b))?(a):(b))
#endif
#define next_page(x) trunc_page(x + PAGE_SIZE)
#if 0
#define ERROR_LOG(args...) IOLog(args)
#else
#define ERROR_LOG(args...)
#endif
OSDefineMetaClassAndStructors(IOMbufMemoryCursor, IOMemoryCursor)
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 0);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 1);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 2);
OSMetaClassDefineReservedUnused( IOMbufMemoryCursor, 3);
OSDefineMetaClassAndStructors(IOMbufNaturalMemoryCursor, IOMbufMemoryCursor)
OSDefineMetaClassAndStructors(IOMbufBigMemoryCursor, IOMbufMemoryCursor)
OSDefineMetaClassAndStructors(IOMbufLittleMemoryCursor, IOMbufMemoryCursor)
#ifdef __ppc__
OSDefineMetaClassAndStructors(IOMbufDBDMAMemoryCursor, IOMbufMemoryCursor)
#endif
#define super IOMemoryCursor
bool IOMbufMemoryCursor::initWithSpecification(OutputSegmentFunc outSeg,
UInt32 maxSegmentSize,
UInt32 maxTransferSize,
UInt32 align)
{
return false;
}
bool IOMbufMemoryCursor::initWithSpecification(OutputSegmentFunc inOutSeg,
UInt32 inMaxSegmentSize,
UInt32 inMaxNumSegments)
{
if (!super::initWithSpecification(inOutSeg, inMaxSegmentSize, 0, 1))
return false;
#if 0
assert(inMaxSegmentSize >= PAGE_SIZE);
if (inMaxSegmentSize < PAGE_SIZE)
return false;
#else
if (!inMaxSegmentSize)
return false;
#endif
maxSegmentSize = MIN(maxSegmentSize, PAGE_SIZE);
maxNumSegments = inMaxNumSegments;
coalesceCount = 0;
return true;
}
#define BCOPY(s, d, l) do { bcopy((void *) s, (void *) d, l); } while(0)
static inline void coalesceSegments(mbuf_t srcm, mbuf_t dstm)
{
uintptr_t src, dst;
SInt32 srcLen, dstLen;
mbuf_t temp;
mbuf_t head = srcm;
srcLen = mbuf_len( srcm );
src = (uintptr_t) mbuf_data(srcm);
dstLen = mbuf_len( dstm );
dst = (uintptr_t) mbuf_data( dstm );
for (;;) {
if (srcLen < dstLen) {
BCOPY(src, dst, srcLen);
dst += srcLen;
dstLen -= srcLen;
temp = mbuf_next( srcm ); assert(temp);
if(srcm != head)
mbuf_free(srcm);
srcm = temp;
srcLen = mbuf_len( srcm );
src = (uintptr_t)mbuf_data(srcm);
}
else if (srcLen > dstLen) {
BCOPY(src, dst, dstLen);
src += dstLen;
srcLen -= dstLen;
temp = mbuf_next( dstm ); assert(temp);
dstm = temp;
dstLen = mbuf_len( dstm );
dst = (uintptr_t)mbuf_data( dstm );
}
else {
BCOPY(src, dst, srcLen);
temp = mbuf_next( srcm );
if(srcm != head)
mbuf_free(srcm);
srcm = temp;
if (! mbuf_next ( dstm ))
{
mbuf_setnext(dstm, srcm);
break;
}
dstm = mbuf_next ( dstm );
assert(srcm);
dstLen = mbuf_len ( dstm );
dst = (uintptr_t)mbuf_data( dstm );
srcLen = mbuf_len( srcm );
src = (uintptr_t)mbuf_data( srcm );
}
}
}
static const UInt32 kMBufDataCacheSize = 16;
static inline bool analyseSegments(
mbuf_t packet,
const UInt32 mbufsInCache,
const UInt32 segsPerMBuf[],
SInt32 numSegs,
const UInt32 maxSegs)
{
mbuf_t newPacket; mbuf_t out; SInt32 outSize; SInt32 outSegs; SInt32 doneSegs; SInt32 outLen;
mbuf_t in = packet; UInt32 inIndex = 0;
const uint32_t c_mlen = mbuf_get_mlen();
if(mbuf_get(MBUF_DONTWAIT, MT_DATA, &newPacket))
{
ERROR_LOG("analyseSegments: MGET() 1 error\n");
return false;
}
out = newPacket;
outSize = c_mlen;
doneSegs = outSegs = outLen = 0;
numSegs -= maxSegs;
do {
uintptr_t vmo;
outLen += mbuf_len(in);
while (outLen > outSize) {
if (outSize != MCLBYTES) {
if(mbuf_mclget(MBUF_DONTWAIT, MT_DATA, &out) || !(mbuf_flags(out) & MBUF_EXT) )
{
ERROR_LOG("analyseSegments: MCLGET() error\n");
goto bombAnalysis;
}
outSize = MCLBYTES;
continue;
}
vmo = (uintptr_t)mbuf_data(out);
mbuf_setlen(out, MCLBYTES);
doneSegs += (round_page(vmo + MCLBYTES) - trunc_page(vmo))
/ PAGE_SIZE;
if (doneSegs + 1 > (int) maxSegs) {
ERROR_LOG("analyseSegments: maxSegs limit 1 reached! %ld %ld\n",
doneSegs, maxSegs);
goto bombAnalysis;
}
mbuf_t tempmbuf;
if(mbuf_get(MBUF_DONTWAIT, MT_DATA, &tempmbuf))
{
ERROR_LOG("analyseSegments: MGET() error\n");
goto bombAnalysis;
}
mbuf_setnext(out, tempmbuf);
out = tempmbuf;
outSize = c_mlen;
outLen -= MCLBYTES;
}
vmo = (uintptr_t)mbuf_data(out);
outSegs = (round_page(vmo + outLen) - trunc_page(vmo)) / PAGE_SIZE;
if (doneSegs + outSegs > (int) maxSegs) {
ERROR_LOG("analyseSegments: maxSegs limit 2 reached! %ld %ld %ld\n",
doneSegs, outSegs, maxSegs);
goto bombAnalysis;
}
if (inIndex < mbufsInCache)
numSegs -= segsPerMBuf[inIndex]; else {
int thisLen = 0, mbufLen;
vmo = (uintptr_t)mbuf_data(in);
for (mbufLen = mbuf_len(in); mbufLen; mbufLen -= thisLen) {
thisLen = MIN(next_page(vmo), vmo + mbufLen) - vmo;
vmo += thisLen;
numSegs--;
}
}
in = mbuf_next(in);
inIndex++;
} while (in && ((numSegs + doneSegs + outSegs) > 0));
if ( (int) (numSegs + doneSegs + outSegs) <= 0) {
mbuf_setlen(out, outLen);
coalesceSegments(packet, newPacket);
mbuf_setlen(packet , 0 );
mbuf_setnext(packet, newPacket);
return true;
}
bombAnalysis:
mbuf_freem(newPacket);
return false;
}
UInt32 IOMbufMemoryCursor::genPhysicalSegments(mbuf_t packet, void *vector,
UInt32 maxSegs, bool doCoalesce)
{
bool doneCoalesce = false;
if (!packet || !(mbuf_flags(packet) & MBUF_PKTHDR))
return 0;
if (!maxSegs)
{
maxSegs = maxNumSegments;
if (!maxSegs) return 0;
}
if ( mbuf_next(packet) == 0 )
{
uintptr_t src;
struct IOPhysicalSegment physSeg;
src = (uintptr_t)mbuf_data(packet);
if ( trunc_page(src) == trunc_page(src + mbuf_len(packet) - 1) )
{
physSeg.location = (IOPhysicalAddress) mbuf_data_to_physical((char *)src);
if ( physSeg.location )
{
physSeg.length = mbuf_len(packet);
(*outSeg)(physSeg, vector, 0);
return 1;
}
maxSegs = 1;
if ( doCoalesce == false ) return 0;
}
}
if ( doCoalesce == true && maxSegs == 1 )
{
uintptr_t src;
uintptr_t dst;
mbuf_t m;
mbuf_t mnext;
mbuf_t out;
UInt32 len = 0;
struct IOPhysicalSegment physSeg;
if ( mbuf_pkthdr_len(packet) > MCLBYTES ) return 0;
m = packet;
if (mbuf_getpacket( MBUF_DONTWAIT, &out ))
return 0;
mbuf_setflags( out, mbuf_flags( out ) & ~MBUF_PKTHDR );
dst = (uintptr_t)mbuf_data(out);
do
{
src = (uintptr_t)mbuf_data(m);
BCOPY( src, dst, mbuf_len(m) );
dst += mbuf_len(m);
len += mbuf_len(m);
} while ( (m = mbuf_next(m)) != 0 );
mbuf_setlen(out , len);
dst = (uintptr_t)mbuf_data(out);
physSeg.location = (IOPhysicalAddress) mbuf_data_to_physical((char *)dst);
if (!physSeg.location)
{
mbuf_free(out);
return 0;
}
physSeg.length = mbuf_len(out);
(*outSeg)(physSeg, vector, 0);
m = mbuf_next(packet);
while (m != 0)
{
mnext = mbuf_next(m);
mbuf_free(m);
m = mnext;
}
mbuf_setlen(packet , 0);
mbuf_setnext(packet , out);
mbuf_setnext(out , 0);
return 1;
}
UInt32 segsPerMBuf[kMBufDataCacheSize];
tryAgain:
UInt32 curMBufIndex = 0;
UInt32 curSegIndex = 0;
UInt32 lastSegCount = 0;
mbuf_t m = packet;
do {
vm_size_t mbufLen, thisLen = 0;
uintptr_t src;
for (mbufLen = mbuf_len(m), src = (uintptr_t)mbuf_data(m);
mbufLen;
src += thisLen, mbufLen -= thisLen)
{
thisLen = MIN(mbufLen, maxSegmentSize);
thisLen = MIN(next_page(src), src + thisLen) - src;
if (curSegIndex < maxSegs) {
struct IOPhysicalSegment physSeg;
physSeg.location = (IOPhysicalAddress) mbuf_data_to_physical((char *)src);
if ( physSeg.location == 0 )
{
return doCoalesce ?
genPhysicalSegments(packet, vector, 1, true) : 0;
}
physSeg.length = thisLen;
(*outSeg)(physSeg, vector, curSegIndex);
}
curSegIndex++;
}
if (curMBufIndex < kMBufDataCacheSize) {
segsPerMBuf[curMBufIndex] = curSegIndex - lastSegCount;
lastSegCount = curSegIndex;
}
curMBufIndex++;
m = mbuf_next(m);
} while (m);
if (curSegIndex <= maxSegs)
return curSegIndex;
if (!doCoalesce)
return 0;
if (!doneCoalesce
&& (UInt) mbuf_pkthdr_len(packet) <= maxSegs * maxSegmentSize) {
bool analysisRet;
analysisRet = analyseSegments(packet,
MIN(curMBufIndex, kMBufDataCacheSize),
segsPerMBuf,
curSegIndex, maxSegs);
if (analysisRet) {
doneCoalesce = true;
coalesceCount++;
goto tryAgain;
}
}
assert(!doneCoalesce); packetTooBigErrors++;
return 0;
}
UInt32 IOMbufMemoryCursor::getAndResetCoalesceCount()
{
UInt32 cnt = coalesceCount; coalesceCount = 0; return cnt;
}
IOMbufBigMemoryCursor *
IOMbufBigMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufBigMemoryCursor *me = new IOMbufBigMemoryCursor;
if (me && !me->initWithSpecification(&bigOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
extern "C" UInt32 _ZN21IOMbufBigMemoryCursor19getPhysicalSegmentsEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufBigMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegments((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufBigMemoryCursor::getPhysicalSegments(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
extern "C" UInt32 _ZN21IOMbufBigMemoryCursor31getPhysicalSegmentsWithCoalesceEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufBigMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegmentsWithCoalesce((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufBigMemoryCursor::getPhysicalSegmentsWithCoalesce(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
IOMbufNaturalMemoryCursor *
IOMbufNaturalMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufNaturalMemoryCursor *me = new IOMbufNaturalMemoryCursor;
if (me && !me->initWithSpecification(&naturalOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
extern "C" UInt32 _ZN25IOMbufNaturalMemoryCursor19getPhysicalSegmentsEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufNaturalMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegments((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufNaturalMemoryCursor::getPhysicalSegments(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
extern "C" UInt32 _ZN25IOMbufNaturalMemoryCursor31getPhysicalSegmentsWithCoalesceEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufNaturalMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegmentsWithCoalesce((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufNaturalMemoryCursor::getPhysicalSegmentsWithCoalesce(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
IOMbufLittleMemoryCursor *
IOMbufLittleMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufLittleMemoryCursor *me = new IOMbufLittleMemoryCursor;
if (me && !me->initWithSpecification(&littleOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
extern "C" UInt32 _ZN24IOMbufLittleMemoryCursor19getPhysicalSegmentsEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufLittleMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegments((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufLittleMemoryCursor::getPhysicalSegments(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
extern "C" UInt32 _ZN24IOMbufLittleMemoryCursor31getPhysicalSegmentsWithCoalesceEP4mbufPN14IOMemoryCursor15PhysicalSegmentEm(
IOMbufLittleMemoryCursor *self, void *packet, struct IOPhysicalSegment *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegmentsWithCoalesce((mbuf_t)packet, vector,numVectorSegments);
}
UInt32
IOMbufLittleMemoryCursor::getPhysicalSegmentsWithCoalesce(mbuf_t packet,
struct IOPhysicalSegment *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
#ifdef __ppc__
IOMbufDBDMAMemoryCursor *
IOMbufDBDMAMemoryCursor::withSpecification(UInt32 maxSegSize, UInt32 maxNumSegs)
{
IOMbufDBDMAMemoryCursor *me = new IOMbufDBDMAMemoryCursor;
if (me && !me->initWithSpecification(&dbdmaOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
extern "C" UInt32 _ZN23IOMbufDBDMAMemoryCursor19getPhysicalSegmentsEP4mbufP17IODBDMADescriptorm(
IOMbufDBDMAMemoryCursor *self, void *packet, struct IODBDMADescriptor *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegments((mbuf_t)packet, vector, numVectorSegments);
}
UInt32
IOMbufDBDMAMemoryCursor::getPhysicalSegments(mbuf_t packet,
struct IODBDMADescriptor *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, false);
}
extern "C" UInt32 _ZN23IOMbufDBDMAMemoryCursor31getPhysicalSegmentsWithCoalesceEP4mbufP17IODBDMADescriptorm(
IOMbufDBDMAMemoryCursor *self, void *packet, struct IODBDMADescriptor *vector, UInt32 numVectorSegments)
{
return self->getPhysicalSegmentsWithCoalesce((mbuf_t)packet, vector, numVectorSegments);
}
UInt32
IOMbufDBDMAMemoryCursor::getPhysicalSegmentsWithCoalesce(mbuf_t packet,
struct IODBDMADescriptor *vector,
UInt32 numVectorSegments)
{
return genPhysicalSegments(packet, vector, numVectorSegments, true);
}
#endif