#include "clang/Lex/PTHLexer.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/FileSystemStatCache.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/OnDiskHashTable.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/PTHManager.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Token.h"
#include "llvm/ADT/OwningPtr.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/system_error.h"
using namespace clang;
using namespace clang::io;
#define DISK_TOKEN_SIZE (1+1+2+4+4)
PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
const unsigned char *ppcond, PTHManager &PM)
: PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
}
bool PTHLexer::Lex(Token& Tok) {
const unsigned char *CurPtrShadow = CurPtr;
unsigned Word0 = ReadLE32(CurPtrShadow);
uint32_t IdentifierID = ReadLE32(CurPtrShadow);
uint32_t FileOffset = ReadLE32(CurPtrShadow);
tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
uint32_t Len = Word0 >> 16;
CurPtr = CurPtrShadow;
Tok.startToken();
Tok.setKind(TKind);
Tok.setFlag(TFlags);
assert(!LexingRawMode);
Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
Tok.setLength(Len);
if (Tok.isLiteral()) {
Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
}
else if (IdentifierID) {
MIOpt.ReadToken();
IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
Tok.setIdentifierInfo(II);
Tok.setKind(II->getTokenID());
if (II->isHandleIdentifierCase())
return PP->HandleIdentifier(Tok);
return true;
}
if (TKind == tok::eof) {
EofToken = Tok;
assert(!ParsingPreprocessorDirective);
assert(!LexingRawMode);
return LexEndOfFile(Tok);
}
if (TKind == tok::hash && Tok.isAtStartOfLine()) {
LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
assert(!LexingRawMode);
PP->HandleDirective(Tok);
return false;
}
if (TKind == tok::eod) {
assert(ParsingPreprocessorDirective);
ParsingPreprocessorDirective = false;
return true;
}
MIOpt.ReadToken();
return true;
}
bool PTHLexer::LexEndOfFile(Token &Result) {
if (ParsingPreprocessorDirective) {
ParsingPreprocessorDirective = false; return true; }
assert(!LexingRawMode);
while (!ConditionalStack.empty()) {
if (PP->getCodeCompletionFileLoc() != FileStartLoc)
PP->Diag(ConditionalStack.back().IfLoc,
diag::err_pp_unterminated_conditional);
ConditionalStack.pop_back();
}
return PP->HandleEndOfFile(Result);
}
void PTHLexer::getEOF(Token& Tok) {
assert(EofToken.is(tok::eof));
Tok = EofToken;
}
void PTHLexer::DiscardToEndOfLine() {
assert(ParsingPreprocessorDirective && ParsingFilename == false &&
"Must be in a preprocessing directive!");
ParsingPreprocessorDirective = false;
const unsigned char* p = CurPtr;
while (1) {
tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
if (x == tok::eof) break;
Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
if (y & Token::StartOfLine) break;
p += DISK_TOKEN_SIZE;
}
CurPtr = p;
}
bool PTHLexer::SkipBlock() {
assert(CurPPCondPtr && "No cached PP conditional information.");
assert(LastHashTokPtr && "No known '#' token.");
const unsigned char* HashEntryI = 0;
uint32_t TableIdx;
do {
uint32_t Offset = ReadLE32(CurPPCondPtr);
TableIdx = ReadLE32(CurPPCondPtr);
HashEntryI = TokBuf + Offset;
if (HashEntryI < LastHashTokPtr && TableIdx) {
const unsigned char* NextPPCondPtr =
PPCond + TableIdx*(sizeof(uint32_t)*2);
assert(NextPPCondPtr >= CurPPCondPtr);
const unsigned char* HashEntryJ = TokBuf + ReadLE32(NextPPCondPtr);
if (HashEntryJ <= LastHashTokPtr) {
HashEntryI = HashEntryJ;
TableIdx = ReadLE32(NextPPCondPtr);
CurPPCondPtr = NextPPCondPtr;
}
}
}
while (HashEntryI < LastHashTokPtr);
assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
assert(TableIdx && "No jumping from #endifs.");
const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
assert(NextPPCondPtr >= CurPPCondPtr);
CurPPCondPtr = NextPPCondPtr;
HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
uint32_t NextIdx = ReadLE32(NextPPCondPtr);
bool isEndif = NextIdx == 0;
if (CurPtr > HashEntryI) {
assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
if (isEndif)
CurPtr += DISK_TOKEN_SIZE*2;
else
LastHashTokPtr = HashEntryI;
return isEndif;
}
CurPtr = HashEntryI;
LastHashTokPtr = CurPtr;
assert(((tok::TokenKind)*CurPtr) == tok::hash);
CurPtr += DISK_TOKEN_SIZE;
if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
return isEndif;
}
SourceLocation PTHLexer::getSourceLocation() {
const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
uint32_t Offset = ReadLE32(OffsetPtr);
return FileStartLoc.getLocWithOffset(Offset);
}
namespace {
class PTHFileData {
const uint32_t TokenOff;
const uint32_t PPCondOff;
public:
PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
: TokenOff(tokenOff), PPCondOff(ppCondOff) {}
uint32_t getTokenOffset() const { return TokenOff; }
uint32_t getPPCondOffset() const { return PPCondOff; }
};
class PTHFileLookupCommonTrait {
public:
typedef std::pair<unsigned char, const char*> internal_key_type;
static unsigned ComputeHash(internal_key_type x) {
return llvm::HashString(x.second);
}
static std::pair<unsigned, unsigned>
ReadKeyDataLength(const unsigned char*& d) {
unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
unsigned dataLen = (unsigned) *(d++);
return std::make_pair(keyLen, dataLen);
}
static internal_key_type ReadKey(const unsigned char* d, unsigned) {
unsigned char k = *(d++); return std::make_pair(k, (const char*) d);
}
};
class PTHFileLookupTrait : public PTHFileLookupCommonTrait {
public:
typedef const FileEntry* external_key_type;
typedef PTHFileData data_type;
static internal_key_type GetInternalKey(const FileEntry* FE) {
return std::make_pair((unsigned char) 0x1, FE->getName());
}
static bool EqualKey(internal_key_type a, internal_key_type b) {
return a.first == b.first && strcmp(a.second, b.second) == 0;
}
static PTHFileData ReadData(const internal_key_type& k,
const unsigned char* d, unsigned) {
assert(k.first == 0x1 && "Only file lookups can match!");
uint32_t x = ::ReadUnalignedLE32(d);
uint32_t y = ::ReadUnalignedLE32(d);
return PTHFileData(x, y);
}
};
class PTHStringLookupTrait {
public:
typedef uint32_t
data_type;
typedef const std::pair<const char*, unsigned>
external_key_type;
typedef external_key_type internal_key_type;
static bool EqualKey(const internal_key_type& a,
const internal_key_type& b) {
return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
: false;
}
static unsigned ComputeHash(const internal_key_type& a) {
return llvm::HashString(StringRef(a.first, a.second));
}
static const internal_key_type&
GetInternalKey(const external_key_type& x) { return x; }
static std::pair<unsigned, unsigned>
ReadKeyDataLength(const unsigned char*& d) {
return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
}
static std::pair<const char*, unsigned>
ReadKey(const unsigned char* d, unsigned n) {
assert(n >= 2 && d[n-1] == '\0');
return std::make_pair((const char*) d, n-1);
}
static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
unsigned) {
return ::ReadUnalignedLE32(d);
}
};
}
typedef OnDiskChainedHashTable<PTHFileLookupTrait> PTHFileLookup;
typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
const unsigned char* idDataTable,
IdentifierInfo** perIDCache,
void* stringIdLookup, unsigned numIds,
const unsigned char* spellingBase,
const char* originalSourceFile)
: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
NumIds(numIds), PP(0), SpellingBase(spellingBase),
OriginalSourceFile(originalSourceFile) {}
PTHManager::~PTHManager() {
delete Buf;
delete (PTHFileLookup*) FileLookup;
delete (PTHStringIdLookup*) StringIdLookup;
free(PerIDCache);
}
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
}
PTHManager *PTHManager::Create(const std::string &file,
DiagnosticsEngine &Diags) {
OwningPtr<llvm::MemoryBuffer> File;
if (llvm::MemoryBuffer::getFile(file, File)) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
unsigned Version = ReadLE32(p);
if (Version < PTHManager::Version) {
InvalidPTH(Diags,
Version < PTHManager::Version
? "PTH file uses an older PTH format that is no longer supported"
: "PTH file uses a newer PTH format that cannot be read");
return 0;
}
const unsigned char *PrologueOffset = p;
if (PrologueOffset >= BufEnd) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
if (!(FileTable > BufBeg && FileTable < BufEnd)) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0; }
OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
if (FL->isEmpty())
InvalidPTH(Diags, "PTH file contains no cached source data");
const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
if (!(IData >= BufBeg && IData < BufEnd)) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
BufBeg));
const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
Diags.Report(diag::err_invalid_pth_file) << file;
return 0;
}
uint32_t NumIds = ReadLE32(IData);
IdentifierInfo** PerIDCache = 0;
if (NumIds) {
PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
if (!PerIDCache) {
InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
return 0;
}
}
const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
unsigned len = ReadUnalignedLE16(originalSourceBase);
if (!len) originalSourceBase = 0;
return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
SL.take(), NumIds, spellingBase,
(const char*) originalSourceBase);
}
IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
const unsigned char* IDData =
(const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
assert(IDData < (const unsigned char*)Buf->getBufferEnd());
std::pair<IdentifierInfo,const unsigned char*> *Mem =
Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
Mem->second = IDData;
assert(IDData[0] != '\0');
IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
PerIDCache[PersistentID] = II;
assert(II->getNameStart() && II->getNameStart()[0] != '\0');
return II;
}
IdentifierInfo* PTHManager::get(StringRef Name) {
PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
assert(Name.empty() || Name.back() != '\0');
PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
Name.size()));
if (I == SL.end()) return 0;
assert(*I > 0);
return GetIdentifierInfo(*I-1);
}
PTHLexer *PTHManager::CreateLexer(FileID FID) {
const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
if (!FE)
return 0;
PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
PTHFileLookup::iterator I = PFL.find(FE);
if (I == PFL.end()) return 0;
const PTHFileData& FileData = *I;
const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
const unsigned char* data = BufStart + FileData.getTokenOffset();
const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
uint32_t Len = ReadLE32(ppcond);
if (Len == 0) ppcond = 0;
assert(PP && "No preprocessor set yet!");
return new PTHLexer(*PP, FID, data, ppcond, *this);
}
namespace {
class PTHStatData {
public:
const bool HasData;
uint64_t Size;
time_t ModTime;
llvm::sys::fs::UniqueID UniqueID;
bool IsDirectory;
PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
bool IsDirectory)
: HasData(true), Size(Size), ModTime(ModTime), UniqueID(UniqueID),
IsDirectory(IsDirectory) {}
PTHStatData() : HasData(false) {}
};
class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
public:
typedef const char* external_key_type; typedef PTHStatData data_type;
static internal_key_type GetInternalKey(const char *path) {
return std::make_pair((unsigned char) 0x0, path);
}
static bool EqualKey(internal_key_type a, internal_key_type b) {
return strcmp(a.second, b.second) == 0;
}
static data_type ReadData(const internal_key_type& k, const unsigned char* d,
unsigned) {
if (k.first ) {
bool IsDirectory = true;
if (k.first == 0x1 ) {
IsDirectory = false;
d += 4 * 2; }
uint64_t File = ReadUnalignedLE64(d);
uint64_t Device = ReadUnalignedLE64(d);
llvm::sys::fs::UniqueID UniqueID(File, Device);
time_t ModTime = ReadUnalignedLE64(d);
uint64_t Size = ReadUnalignedLE64(d);
return data_type(Size, ModTime, UniqueID, IsDirectory);
}
return data_type();
}
};
class PTHStatCache : public FileSystemStatCache {
typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
CacheTy Cache;
public:
PTHStatCache(PTHFileLookup &FL) :
Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
FL.getBase()) {}
~PTHStatCache() {}
LookupResult getStat(const char *Path, FileData &Data, bool isFile,
vfs::File **F, vfs::FileSystem &FS) {
CacheTy::iterator I = Cache.find(Path);
if (I == Cache.end())
return statChained(Path, Data, isFile, F, FS);
const PTHStatData &D = *I;
if (!D.HasData)
return CacheMissing;
Data.Name = Path;
Data.Size = D.Size;
Data.ModTime = D.ModTime;
Data.UniqueID = D.UniqueID;
Data.IsDirectory = D.IsDirectory;
Data.IsNamedPipe = false;
Data.InPCH = true;
return CacheExists;
}
};
}
FileSystemStatCache *PTHManager::createStatCache() {
return new PTHStatCache(*((PTHFileLookup*) FileLookup));
}