diff options
Diffstat (limited to 'include')
-rw-r--r-- | include/llvm/Bitcode/BitstreamReader.h | 93 | ||||
-rw-r--r-- | include/llvm/Bitcode/ReaderWriter.h | 45 | ||||
-rw-r--r-- | include/llvm/MC/MCDisassembler.h | 2 | ||||
-rw-r--r-- | include/llvm/Support/DataStream.h | 38 | ||||
-rw-r--r-- | include/llvm/Support/MemoryObject.h | 6 | ||||
-rw-r--r-- | include/llvm/Support/StreamableMemoryObject.h | 181 |
6 files changed, 309 insertions, 56 deletions
diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h index e943e3c..b7c52f4 100644 --- a/include/llvm/Bitcode/BitstreamReader.h +++ b/include/llvm/Bitcode/BitstreamReader.h @@ -15,10 +15,12 @@ #ifndef BITSTREAM_READER_H #define BITSTREAM_READER_H +#include "llvm/ADT/OwningPtr.h" #include "llvm/Bitcode/BitCodes.h" #include <climits> #include <string> #include <vector> +#include "llvm/Support/StreamableMemoryObject.h" namespace llvm { @@ -36,9 +38,7 @@ public: std::vector<std::pair<unsigned, std::string> > RecordNames; }; private: - /// FirstChar/LastChar - This remembers the first and last bytes of the - /// stream. - const unsigned char *FirstChar, *LastChar; + OwningPtr<StreamableMemoryObject> BitcodeBytes; std::vector<BlockInfo> BlockInfoRecords; @@ -47,10 +47,10 @@ private: /// uses this. bool IgnoreBlockInfoNames; - BitstreamReader(const BitstreamReader&); // NOT IMPLEMENTED - void operator=(const BitstreamReader&); // NOT IMPLEMENTED + BitstreamReader(const BitstreamReader&); // DO NOT IMPLEMENT + void operator=(const BitstreamReader&); // DO NOT IMPLEMENT public: - BitstreamReader() : FirstChar(0), LastChar(0), IgnoreBlockInfoNames(true) { + BitstreamReader() : IgnoreBlockInfoNames(true) { } BitstreamReader(const unsigned char *Start, const unsigned char *End) { @@ -58,12 +58,17 @@ public: init(Start, End); } + BitstreamReader(StreamableMemoryObject *bytes) { + BitcodeBytes.reset(bytes); + } + void init(const unsigned char *Start, const unsigned char *End) { - FirstChar = Start; - LastChar = End; assert(((End-Start) & 3) == 0 &&"Bitcode stream not a multiple of 4 bytes"); + BitcodeBytes.reset(getNonStreamedMemoryObject(Start, End)); } + StreamableMemoryObject &getBitcodeBytes() { return *BitcodeBytes; } + ~BitstreamReader() { // Free the BlockInfoRecords. while (!BlockInfoRecords.empty()) { @@ -75,9 +80,6 @@ public: BlockInfoRecords.pop_back(); } } - - const unsigned char *getFirstChar() const { return FirstChar; } - const unsigned char *getLastChar() const { return LastChar; } /// CollectBlockInfoNames - This is called by clients that want block/record /// name information. @@ -122,7 +124,7 @@ public: class BitstreamCursor { friend class Deserializer; BitstreamReader *BitStream; - const unsigned char *NextChar; + size_t NextChar; /// CurWord - This is the current data we have pulled from the stream but have /// not returned to the client. @@ -156,8 +158,7 @@ public: } explicit BitstreamCursor(BitstreamReader &R) : BitStream(&R) { - NextChar = R.getFirstChar(); - assert(NextChar && "Bitstream not initialized yet"); + NextChar = 0; CurWord = 0; BitsInCurWord = 0; CurCodeSize = 2; @@ -167,8 +168,7 @@ public: freeState(); BitStream = &R; - NextChar = R.getFirstChar(); - assert(NextChar && "Bitstream not initialized yet"); + NextChar = 0; CurWord = 0; BitsInCurWord = 0; CurCodeSize = 2; @@ -225,13 +225,38 @@ public: /// GetAbbrevIDWidth - Return the number of bits used to encode an abbrev #. unsigned GetAbbrevIDWidth() const { return CurCodeSize; } - bool AtEndOfStream() const { - return NextChar == BitStream->getLastChar() && BitsInCurWord == 0; + bool isEndPos(size_t pos) { + return BitStream->getBitcodeBytes().isObjectEnd(static_cast<uint64_t>(pos)); + } + + bool canSkipToPos(size_t pos) const { + // pos can be skipped to if it is a valid address or one byte past the end. + return pos == 0 || BitStream->getBitcodeBytes().isValidAddress( + static_cast<uint64_t>(pos - 1)); + } + + unsigned char getByte(size_t pos) { + uint8_t byte = -1; + BitStream->getBitcodeBytes().readByte(pos, &byte); + return byte; + } + + uint32_t getWord(size_t pos) { + uint32_t word = -1; + BitStream->getBitcodeBytes().readBytes(pos, + sizeof(word), + reinterpret_cast<uint8_t *>(&word), + NULL); + return word; + } + + bool AtEndOfStream() { + return isEndPos(NextChar) && BitsInCurWord == 0; } /// GetCurrentBitNo - Return the bit # of the bit we are reading. uint64_t GetCurrentBitNo() const { - return (NextChar-BitStream->getFirstChar())*CHAR_BIT - BitsInCurWord; + return NextChar*CHAR_BIT - BitsInCurWord; } BitstreamReader *getBitStreamReader() { @@ -246,12 +271,10 @@ public: void JumpToBit(uint64_t BitNo) { uintptr_t ByteNo = uintptr_t(BitNo/8) & ~3; uintptr_t WordBitNo = uintptr_t(BitNo) & 31; - assert(ByteNo <= (uintptr_t)(BitStream->getLastChar()- - BitStream->getFirstChar()) && - "Invalid location"); + assert(canSkipToPos(ByteNo) && "Invalid location"); // Move the cursor to the right word. - NextChar = BitStream->getFirstChar()+ByteNo; + NextChar = ByteNo; BitsInCurWord = 0; CurWord = 0; @@ -272,7 +295,7 @@ public: } // If we run out of data, stop at the end of the stream. - if (NextChar == BitStream->getLastChar()) { + if (isEndPos(NextChar)) { CurWord = 0; BitsInCurWord = 0; return 0; @@ -281,8 +304,7 @@ public: unsigned R = CurWord; // Read the next word from the stream. - CurWord = (NextChar[0] << 0) | (NextChar[1] << 8) | - (NextChar[2] << 16) | (NextChar[3] << 24); + CurWord = getWord(NextChar); NextChar += 4; // Extract NumBits-BitsInCurWord from what we just read. @@ -376,9 +398,8 @@ public: // Check that the block wasn't partially defined, and that the offset isn't // bogus. - const unsigned char *const SkipTo = NextChar + NumWords*4; - if (AtEndOfStream() || SkipTo > BitStream->getLastChar() || - SkipTo < BitStream->getFirstChar()) + size_t SkipTo = NextChar + NumWords*4; + if (AtEndOfStream() || !canSkipToPos(SkipTo)) return true; NextChar = SkipTo; @@ -409,8 +430,7 @@ public: if (NumWordsP) *NumWordsP = NumWords; // Validate that this block is sane. - if (CurCodeSize == 0 || AtEndOfStream() || - NextChar+NumWords*4 > BitStream->getLastChar()) + if (CurCodeSize == 0 || AtEndOfStream()) return true; return false; @@ -512,24 +532,25 @@ public: SkipToWord(); // 32-bit alignment // Figure out where the end of this blob will be including tail padding. - const unsigned char *NewEnd = NextChar+((NumElts+3)&~3); + size_t NewEnd = NextChar+((NumElts+3)&~3); // If this would read off the end of the bitcode file, just set the // record to empty and return. - if (NewEnd > BitStream->getLastChar()) { + if (!canSkipToPos(NewEnd)) { Vals.append(NumElts, 0); - NextChar = BitStream->getLastChar(); + NextChar = BitStream->getBitcodeBytes().getExtent(); break; } // Otherwise, read the number of bytes. If we can return a reference to // the data, do so to avoid copying it. if (BlobStart) { - *BlobStart = (const char*)NextChar; + *BlobStart = (const char*)BitStream->getBitcodeBytes().getPointer( + NextChar, NumElts); *BlobLen = NumElts; } else { for (; NumElts; ++NextChar, --NumElts) - Vals.push_back(*NextChar); + Vals.push_back(getByte(NextChar)); } // Skip over tail padding. NextChar = NewEnd; diff --git a/include/llvm/Bitcode/ReaderWriter.h b/include/llvm/Bitcode/ReaderWriter.h index fa754c0..ce975cd 100644 --- a/include/llvm/Bitcode/ReaderWriter.h +++ b/include/llvm/Bitcode/ReaderWriter.h @@ -17,35 +17,45 @@ #include <string> namespace llvm { - class Module; - class MemoryBuffer; - class ModulePass; class BitstreamWriter; + class MemoryBuffer; + class DataStreamer; class LLVMContext; + class Module; + class ModulePass; class raw_ostream; - + /// getLazyBitcodeModule - Read the header of the specified bitcode buffer /// and prepare for lazy deserialization of function bodies. If successful, /// this takes ownership of 'buffer' and returns a non-null pointer. On /// error, this returns null, *does not* take ownership of Buffer, and fills /// in *ErrMsg with an error description if ErrMsg is non-null. Module *getLazyBitcodeModule(MemoryBuffer *Buffer, - LLVMContext& Context, + LLVMContext &Context, std::string *ErrMsg = 0); + /// getStreamedBitcodeModule - Read the header of the specified stream + /// and prepare for lazy deserialization and streaming of function bodies. + /// On error, this returns null, and fills in *ErrMsg with an error + /// description if ErrMsg is non-null. + Module *getStreamedBitcodeModule(const std::string &name, + DataStreamer *streamer, + LLVMContext &Context, + std::string *ErrMsg = 0); + /// getBitcodeTargetTriple - Read the header of the specified bitcode /// buffer and extract just the triple information. If successful, /// this returns a string and *does not* take ownership /// of 'buffer'. On error, this returns "", and fills in *ErrMsg /// if ErrMsg is non-null. std::string getBitcodeTargetTriple(MemoryBuffer *Buffer, - LLVMContext& Context, + LLVMContext &Context, std::string *ErrMsg = 0); /// ParseBitcodeFile - Read the specified bitcode file, returning the module. /// If an error occurs, this returns null and fills in *ErrMsg if it is /// non-null. This method *never* takes ownership of Buffer. - Module *ParseBitcodeFile(MemoryBuffer *Buffer, LLVMContext& Context, + Module *ParseBitcodeFile(MemoryBuffer *Buffer, LLVMContext &Context, std::string *ErrMsg = 0); /// WriteBitcodeToFile - Write the specified module to the specified @@ -60,8 +70,8 @@ namespace llvm { /// createBitcodeWriterPass - Create and return a pass that writes the module /// to the specified ostream. ModulePass *createBitcodeWriterPass(raw_ostream &Str); - - + + /// isBitcodeWrapper - Return true if the given bytes are the magic bytes /// for an LLVM IR bitcode wrapper. /// @@ -109,21 +119,24 @@ namespace llvm { /// uint32_t BitcodeSize; // Size of traditional bitcode file. /// ... potentially other gunk ... /// }; - /// + /// /// This function is called when we find a file with a matching magic number. /// In this case, skip down to the subsection of the file that is actually a /// BC file. - static inline bool SkipBitcodeWrapperHeader(unsigned char *&BufPtr, - unsigned char *&BufEnd) { + /// If 'VerifyBufferSize' is true, check that the buffer is large enough to + /// contain the whole bitcode file. + static inline bool SkipBitcodeWrapperHeader(const unsigned char *&BufPtr, + const unsigned char *&BufEnd, + bool VerifyBufferSize) { enum { KnownHeaderSize = 4*4, // Size of header we read. OffsetField = 2*4, // Offset in bytes to Offset field. SizeField = 3*4 // Offset in bytes to Size field. }; - + // Must contain the header! if (BufEnd-BufPtr < KnownHeaderSize) return true; - + unsigned Offset = ( BufPtr[OffsetField ] | (BufPtr[OffsetField+1] << 8) | (BufPtr[OffsetField+2] << 16) | @@ -132,9 +145,9 @@ namespace llvm { (BufPtr[SizeField +1] << 8) | (BufPtr[SizeField +2] << 16) | (BufPtr[SizeField +3] << 24)); - + // Verify that Offset+Size fits in the file. - if (Offset+Size > unsigned(BufEnd-BufPtr)) + if (VerifyBufferSize && Offset+Size > unsigned(BufEnd-BufPtr)) return true; BufPtr += Offset; BufEnd = BufPtr+Size; diff --git a/include/llvm/MC/MCDisassembler.h b/include/llvm/MC/MCDisassembler.h index 454277d..8022cc1 100644 --- a/include/llvm/MC/MCDisassembler.h +++ b/include/llvm/MC/MCDisassembler.h @@ -79,7 +79,7 @@ public: /// MCDisassembler::Fail if the instruction was invalid. virtual DecodeStatus getInstruction(MCInst& instr, uint64_t& size, - const MemoryObject ®ion, + MemoryObject ®ion, uint64_t address, raw_ostream &vStream, raw_ostream &cStream) const = 0; diff --git a/include/llvm/Support/DataStream.h b/include/llvm/Support/DataStream.h new file mode 100644 index 0000000..620dbb8 --- /dev/null +++ b/include/llvm/Support/DataStream.h @@ -0,0 +1,38 @@ +//===---- llvm/Support/DataStream.h - Lazy bitcode streaming -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This header defines DataStreamer, which fetches bytes of data from +// a stream source. It provides support for streaming (lazy reading) of +// data, e.g. bitcode +// +//===----------------------------------------------------------------------===// + + +#ifndef LLVM_SUPPORT_DATASTREAM_H_ +#define LLVM_SUPPORT_DATASTREAM_H_ + +#include <string> + +namespace llvm { + +class DataStreamer { +public: + /// Fetch bytes [start-end) from the stream, and write them to the + /// buffer pointed to by buf. Returns the number of bytes actually written. + virtual size_t GetBytes(unsigned char *buf, size_t len) = 0; + + virtual ~DataStreamer(); +}; + +DataStreamer *getDataFileStreamer(const std::string &Filename, + std::string *Err); + +} + +#endif // LLVM_SUPPORT_DATASTREAM_H_ diff --git a/include/llvm/Support/MemoryObject.h b/include/llvm/Support/MemoryObject.h index dec0f13..5ae813f 100644 --- a/include/llvm/Support/MemoryObject.h +++ b/include/llvm/Support/MemoryObject.h @@ -34,7 +34,7 @@ public: /// is getBase() + getExtent() - 1). /// /// @result - The size of the region. - virtual uint64_t getExtent() const = 0; + virtual uint64_t getExtent() = 0; /// readByte - Tries to read a single byte from the region. /// @@ -42,7 +42,7 @@ public: /// @param ptr - A pointer to a byte to be filled in. Must be non-NULL. /// @result - 0 if successful; -1 if not. Failure may be due to a /// bounds violation or an implementation-specific error. - virtual int readByte(uint64_t address, uint8_t* ptr) const = 0; + virtual int readByte(uint64_t address, uint8_t* ptr) = 0; /// readBytes - Tries to read a contiguous range of bytes from the /// region, up to the end of the region. @@ -61,7 +61,7 @@ public: virtual int readBytes(uint64_t address, uint64_t size, uint8_t* buf, - uint64_t* copied) const; + uint64_t* copied); }; } diff --git a/include/llvm/Support/StreamableMemoryObject.h b/include/llvm/Support/StreamableMemoryObject.h new file mode 100644 index 0000000..b0e202c --- /dev/null +++ b/include/llvm/Support/StreamableMemoryObject.h @@ -0,0 +1,181 @@ +//===- StreamableMemoryObject.h - Streamable data interface - -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + + +#ifndef STREAMABLEMEMORYOBJECT_H_ +#define STREAMABLEMEMORYOBJECT_H_ + +#include "llvm/ADT/OwningPtr.h" +#include "llvm/Support/MemoryObject.h" +#include "llvm/Support/DataStream.h" +#include <vector> + +namespace llvm { + +/// StreamableMemoryObject - Interface to data which might be streamed. +/// Streamability has 2 important implications/restrictions. First, the data +/// might not yet exist in memory when the request is made. This just means +/// that readByte/readBytes might have to block or do some work to get it. +/// More significantly, the exact size of the object might not be known until +/// it has all been fetched. This means that to return the right result, +/// getExtent must also wait for all the data to arrive; therefore it should +/// not be called on objects which are actually streamed (this would defeat +/// the purpose of streaming). Instead, isValidAddress and isObjectEnd can be +/// used to test addresses without knowing the exact size of the stream. +/// Finally, getPointer can be used instead of readBytes to avoid extra copying. +class StreamableMemoryObject : public MemoryObject { + public: + /// Destructor - Override as necessary. + virtual ~StreamableMemoryObject(); + + /// getBase - Returns the lowest valid address in the region. + /// + /// @result - The lowest valid address. + virtual uint64_t getBase() const = 0; + + /// getExtent - Returns the size of the region in bytes. (The region is + /// contiguous, so the highest valid address of the region + /// is getBase() + getExtent() - 1). + /// May block until all bytes in the stream have been read + /// + /// @result - The size of the region. + virtual uint64_t getExtent() = 0; + + /// readByte - Tries to read a single byte from the region. + /// May block until (address - base) bytes have been read + /// @param address - The address of the byte, in the same space as getBase(). + /// @param ptr - A pointer to a byte to be filled in. Must be non-NULL. + /// @result - 0 if successful; -1 if not. Failure may be due to a + /// bounds violation or an implementation-specific error. + virtual int readByte(uint64_t address, uint8_t* ptr) = 0; + + /// readBytes - Tries to read a contiguous range of bytes from the + /// region, up to the end of the region. + /// May block until (address - base + size) bytes have + /// been read. Additionally, StreamableMemoryObjects will + /// not do partial reads - if size bytes cannot be read, + /// readBytes will fail. + /// + /// @param address - The address of the first byte, in the same space as + /// getBase(). + /// @param size - The maximum number of bytes to copy. + /// @param buf - A pointer to a buffer to be filled in. Must be non-NULL + /// and large enough to hold size bytes. + /// @param copied - A pointer to a nunber that is filled in with the number + /// of bytes actually read. May be NULL. + /// @result - 0 if successful; -1 if not. Failure may be due to a + /// bounds violation or an implementation-specific error. + virtual int readBytes(uint64_t address, + uint64_t size, + uint8_t* buf, + uint64_t* copied) = 0; + + /// getPointer - Ensures that the requested data is in memory, and returns + /// A pointer to it. More efficient than using readBytes if the + /// data is already in memory. + /// May block until (address - base + size) bytes have been read + /// @param address - address of the byte, in the same space as getBase() + /// @param size - amount of data that must be available on return + /// @result - valid pointer to the requested data + virtual const uint8_t *getPointer(uint64_t address, uint64_t size) = 0; + + /// isValidAddress - Returns true if the address is within the object + /// (i.e. between base and base + extent - 1 inclusive) + /// May block until (address - base) bytes have been read + /// @param address - address of the byte, in the same space as getBase() + /// @result - true if the address may be read with readByte() + virtual bool isValidAddress(uint64_t address) = 0; + + /// isObjectEnd - Returns true if the address is one past the end of the + /// object (i.e. if it is equal to base + extent) + /// May block until (address - base) bytes have been read + /// @param address - address of the byte, in the same space as getBase() + /// @result - true if the address is equal to base + extent + virtual bool isObjectEnd(uint64_t address) = 0; +}; + +/// StreamingMemoryObject - interface to data which is actually streamed from +/// a DataStreamer. In addition to inherited members, it has the +/// dropLeadingBytes and setKnownObjectSize methods which are not applicable +/// to non-streamed objects. +class StreamingMemoryObject : public StreamableMemoryObject { +public: + StreamingMemoryObject(DataStreamer *streamer); + virtual uint64_t getBase() const { return 0; } + virtual uint64_t getExtent(); + virtual int readByte(uint64_t address, uint8_t* ptr); + virtual int readBytes(uint64_t address, + uint64_t size, + uint8_t* buf, + uint64_t* copied); + virtual const uint8_t *getPointer(uint64_t address, uint64_t size) { + // This could be fixed by ensuring the bytes are fetched and making a copy, + // requiring that the bitcode size be known, or otherwise ensuring that + // the memory doesn't go away/get reallocated, but it's + // not currently necessary. Users that need the pointer don't stream. + assert(0 && "getPointer in streaming memory objects not allowed"); + return NULL; + } + virtual bool isValidAddress(uint64_t address); + virtual bool isObjectEnd(uint64_t address); + + /// Drop s bytes from the front of the stream, pushing the positions of the + /// remaining bytes down by s. This is used to skip past the bitcode header, + /// since we don't know a priori if it's present, and we can't put bytes + /// back into the stream once we've read them. + bool dropLeadingBytes(size_t s); + + /// If the data object size is known in advance, many of the operations can + /// be made more efficient, so this method should be called before reading + /// starts (although it can be called anytime). + void setKnownObjectSize(size_t size); + +private: + const static uint32_t kChunkSize = 4096 * 4; + std::vector<unsigned char> Bytes; + OwningPtr<DataStreamer> Streamer; + size_t BytesRead; // Bytes read from stream + size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header) + size_t ObjectSize; // 0 if unknown, set if wrapper was seen or EOF reached + bool EOFReached; + + // Fetch enough bytes such that Pos can be read or EOF is reached + // (i.e. BytesRead > Pos). Return true if Pos can be read. + // Unlike most of the functions in BitcodeReader, returns true on success. + // Most of the requests will be small, but we fetch at kChunkSize bytes + // at a time to avoid making too many potentially expensive GetBytes calls + bool fetchToPos(size_t Pos) { + if (EOFReached) return Pos < ObjectSize; + while (Pos >= BytesRead) { + Bytes.resize(BytesRead + kChunkSize); + size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped], + kChunkSize); + BytesRead += bytes; + if (bytes < kChunkSize) { + if (ObjectSize && BytesRead < Pos) + assert(0 && "Unexpected short read fetching bitcode"); + if (BytesRead <= Pos) { // reached EOF/ran out of bytes + ObjectSize = BytesRead; + EOFReached = true; + return false; + } + } + } + return true; + } + + StreamingMemoryObject(const StreamingMemoryObject&); // DO NOT IMPLEMENT + void operator=(const StreamingMemoryObject&); // DO NOT IMPLEMENT +}; + +StreamableMemoryObject *getNonStreamedMemoryObject( + const unsigned char *Start, const unsigned char *End); + +} +#endif // STREAMABLEMEMORYOBJECT_H_ |