SerialIo.h

#ifndef ROSE_BinaryAnalysis_SerialIo_H
#define ROSE_BinaryAnalysis_SerialIo_H
#include <featureTests.h>
#ifdef ROSE_ENABLE_BINARY_ANALYSIS
 
// Define this if you need to debug SerialIo -- it causes everything to run in the calling thread and avoid catching exceptions.
//#define ROSE_DEBUG_SERIAL_IO
 
#if defined(BOOST_WINDOWS)
    // Lacks POSIX file system, so we can't monitor the I/O progress
    #undef ROSE_SUPPORTS_SERIAL_IO
#elif !defined(ROSE_HAVE_BOOST_SERIALIZATION_LIB)
    // Lacks Boost's serialization library, which is how we convert objects to bytes and vice versa
    #undef ROSE_SUPPORTS_SERIAL_IO
#elif defined(__clang__)
    #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
#elif defined(__GNUC__)
    #if __GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNU_C_PATCHLEVEL__ <= 40204
        // GCC <= 4.2.4 gets segfaults compiling this file
        #undef ROSE_SUPPORTS_SERIAL_IO
    #else
        #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
    #endif
#else
    #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
#endif
 
// These have to be included early, before the definitions of the ROSE classes that are serialized
#ifdef ROSE_SUPPORTS_SERIAL_IO
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/archive/xml_oarchive.hpp>
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/stream.hpp>
#endif
 
#include <Rose/Progress.h>
#include <Rose/Exception.h>
 
#include <Sawyer/Message.h>
#include <Sawyer/ProgressBar.h>
#include <Sawyer/Synchronization.h>
 
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/thread.hpp>
 
namespace Rose {
namespace BinaryAnalysis {
 
// SerialIo
 
class SerialIo: public Sawyer::SharedObject {
public:
    using Ptr = SerialIoPtr;
 
    enum Format {
        BINARY,         
        TEXT,           
        XML             
    };
 
    enum Savable {
        NO_OBJECT           = 0x00000000, 
        PARTITIONER         = 0x00000001, 
        AST                 = 0x00000002, 
        END_OF_DATA         = 0x0000fffe, 
        ERROR               = 0x0000ffff, 
        USER_DEFINED        = 0x00010000, 
        USER_DEFINED_LAST   = 0xffffffff  
    };
 
    class Exception: public Rose::Exception {
    public:
        explicit Exception(const std::string &s): Rose::Exception(s) {}
        ~Exception() throw() {}
    };
 
private:
    mutable SAWYER_THREAD_TRAITS::Mutex mutex_; // protects the following data members
    Format format_;
    Progress::Ptr progress_;
    bool isOpen_;
    Savable objectType_;
 
protected:
    Sawyer::ProgressBar<size_t> progressBar_;
 
    // We use low-level file descriptors under an std::stream I/O interface in order to provide progress reports. This
    // allows one thread to be reading from or writing to the file and another thread to monitor the file position to
    // report progress. The C++ standard library doesn't have an API for wrapping file descriptors in a stream interface,
    // so we use Boost.
    int fd_;
 
protected:
    SerialIo()
        : format_(BINARY), progress_(Progress::instance()), isOpen_(false), objectType_(NO_OBJECT),
          progressBar_(mlog[Sawyer::Message::MARCH]), fd_(-1) {
        init();
        progressBar_.suffix(" bytes");
    }
 
public:
    static Sawyer::Message::Facility mlog;
 
    virtual ~SerialIo();
 
    static Savable userSavable(unsigned offset);
 
    Format format() const;
    void format(Format);
    Progress::Ptr progress() const;
    void progress(const Progress::Ptr&);
    virtual void open(const boost::filesystem::path&) = 0;
 
    virtual void close() = 0;
 
    bool isOpen() const;
 
    Savable objectType() const;
 
protected:
    // Set or clear the isOpen flag.
    void setIsOpen(bool b);
 
    // Set object type to ERROR to indicate that a read was unsuccessful
    void objectType(Savable);
 
private:
    void init();
};
 
 
// SerialOutput
 
class SerialOutput: public SerialIo {
public:
    using Ptr = SerialOutputPtr;
 
private:
#ifdef ROSE_SUPPORTS_SERIAL_IO
    boost::iostreams::file_descriptor_sink device_;
    boost::iostreams::stream<boost::iostreams::file_descriptor_sink> file_;
    boost::archive::binary_oarchive *binary_archive_;
    boost::archive::text_oarchive *text_archive_;
    boost::archive::xml_oarchive *xml_archive_;
#endif
 
protected:
#ifdef ROSE_SUPPORTS_SERIAL_IO
    SerialOutput(): binary_archive_(NULL), text_archive_(NULL), xml_archive_(NULL) {}
#else
    SerialOutput() {}
#endif
 
public:
    ~SerialOutput();
    void open(const boost::filesystem::path &fileName) override;
    void close() override;
 
    static Ptr instance() { return Ptr(new SerialOutput); }
 
    void savePartitioner(const Partitioner2::PartitionerConstPtr&);
 
    void saveAst(SgAsmNode*);
    void saveAst(SgBinaryComposite*);
private:
    // The saveAstHelper is what actually gets called for the functions above. It doesn't descriminate between nodes that
    // support serialization and those that don't, which is why it's private. Use only the public functions because they'll
    // give you a nice compiler error if you try to save an Ast node type that isn't supported.
    void saveAstHelper(SgNode*);
public:
 
    template<class T>
    void saveObject(Savable objectTypeId, const T &object) {
        if (!isOpen())
            throw Exception("cannot save object when no file is open");
        if (ERROR == objectType())
            throw Exception("cannot save object because stream is in error state");
 
#ifndef ROSE_SUPPORTS_SERIAL_IO
        throw Exception("binary state files are not supported in this configuration");
#elif defined(ROSE_DEBUG_SERIAL_IO)
        std::string errorMessage;
        asyncSave(objectTypeId, object, &errorMessage);
#else
        // A different thread saves the object while this thread updates the progress
        std::string errorMessage;
        boost::thread worker(startWorker<T>, this, objectTypeId, &object, &errorMessage);
        boost::chrono::milliseconds timeout((unsigned)(1000 * Sawyer::ProgressBarSettings::minimumUpdateInterval()));
        progressBar_.prefix("writing");
        while (!worker.try_join_for(timeout)) {
            off_t cur = ::lseek(fd_, 0, SEEK_CUR);
            if (-1 == cur) {
                ++progressBar_; // so a spinner moves
            } else {
                progressBar_.value(cur);
                if (Progress::Ptr p = progress())
                    p->update(Progress::Report(cur, NAN));
            }
        }
        if (!errorMessage.empty())
            throw Exception(errorMessage);
#endif
    }
 
private:
    template<class T>
    static void startWorker(SerialOutput *saver, Savable objectTypeId, const T *object, std::string *errorMessage) {
        ASSERT_not_null(object);
        saver->asyncSave(objectTypeId, *object, errorMessage);
    }
 
    // This might run in its own thread.
    template<class T>
    void asyncSave(Savable objectTypeId, const T &object, std::string *errorMessage) {
        ASSERT_not_null(errorMessage);
#ifndef ROSE_SUPPORTS_SERIAL_IO
        ASSERT_not_reachable("not supported in this configuration");
#else
#if !defined(ROSE_DEBUG_SERIAL_IO)
        try {
#endif
            std::string roseVersion = ROSE_PACKAGE_VERSION;
            objectType(ERROR);
            switch (format()) {
                case BINARY:
                    ASSERT_not_null(binary_archive_);
                    *binary_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
                    *binary_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
                    *binary_archive_ <<BOOST_SERIALIZATION_NVP(object);
                    break;
                case TEXT:
                    ASSERT_not_null(text_archive_);
                    *text_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
                    *text_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
                    *text_archive_ <<BOOST_SERIALIZATION_NVP(object);
                    break;
                case XML:
                    ASSERT_not_null(xml_archive_);
                    *xml_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
                    *xml_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
                    *xml_archive_ <<BOOST_SERIALIZATION_NVP(object);
                    break;
            }
            objectType(objectTypeId);
#if !defined(ROSE_DEBUG_SERIAL_IO)
        } catch (std::exception &e) {
            *errorMessage = e.what();
        } catch (...) {
            *errorMessage = "failed to write object to output stream";
        }
#endif
#endif
    }
};
 
 
// SerialInput
 
class SerialInput: public SerialIo {
public:
    using Ptr = SerialInputPtr;
 
private:
#ifdef ROSE_SUPPORTS_SERIAL_IO
    size_t fileSize_;
    boost::iostreams::file_descriptor_source device_;
    boost::iostreams::stream<boost::iostreams::file_descriptor_source> file_;
    boost::archive::binary_iarchive *binary_archive_;
    boost::archive::text_iarchive *text_archive_;
    boost::archive::xml_iarchive *xml_archive_;
#endif
 
protected:
#ifdef ROSE_SUPPORTS_SERIAL_IO
    SerialInput(): fileSize_(0), binary_archive_(NULL), text_archive_(NULL), xml_archive_(NULL) {}
#else
    SerialInput() {}
#endif
 
public:
    ~SerialInput();
    void open(const boost::filesystem::path &fileName) override;
    void close() override;
 
    static Ptr instance() { return Ptr(new SerialInput); }
 
    Savable nextObjectType();
 
    Partitioner2::PartitionerPtr loadPartitioner();
 
    SgNode* loadAst();
 
    template<class T>
    T loadObject(Savable objectTypeId) {
        T object;
        loadObject<T>(objectTypeId, object);
        return object;
    }
    template<class T>
    void loadObject(Savable objectTypeId, T &object) {
        if (!isOpen())
            throw Exception("cannot load object when no file is open");
 
#ifndef ROSE_SUPPORTS_SERIAL_IO
        throw Exception("binary state files are not supported in this configuration");
#else
        if (ERROR == objectType())
            throw Exception("cannot read object because stream is in error state");
        if (objectType() != objectTypeId) {
            throw Exception("unexpected object type (expected " + boost::lexical_cast<std::string>(objectTypeId) +
                            " but read " + boost::lexical_cast<std::string>(objectType()) + ")");
        }
        objectType(ERROR); // in case of exception
        std::string errorMessage;
#ifdef ROSE_DEBUG_SERIAL_IO
        asyncLoad(object, &errorMessage);
#else
        boost::thread worker(startWorker<T>, this, &object, &errorMessage);
        boost::chrono::milliseconds timeout((unsigned)(1000 * Sawyer::ProgressBarSettings::minimumUpdateInterval()));
        progressBar_.prefix("reading");
        while (!worker.try_join_for(timeout)) {
            if (fileSize_ > 0) {
                off_t cur = ::lseek(fd_, 0, SEEK_CUR);
                if (cur != -1) {
                    progressBar_.value(cur);
                    if (Progress::Ptr p = progress())
                        p->update(Progress::Report(cur, fileSize_));
                }
            } else {
                ++progressBar_; // so the spinner moves
            }
        }
        if (!errorMessage.empty())
            throw Exception(errorMessage);
#endif
        advanceObjectType();
#endif
    }
private:
    template<class T>
    static void startWorker(SerialInput *loader, T *object, std::string *errorMessage) {
        loader->asyncLoad(*object, errorMessage);
    }
 
    // Might run in its own thread
    template<class T>
    void asyncLoad(T &object, std::string *errorMessage) {
        ASSERT_not_null(errorMessage);
#ifndef ROSE_SUPPORTS_SERIAL_IO
        ASSERT_not_reachable("not supported in this configuration");
#else
#if !defined(ROSE_DEBUG_SERIAL_IO)
        try {
#endif
            std::string roseVersion;
            switch (format()) {
                case BINARY:
                    ASSERT_not_null(binary_archive_);
                    *binary_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
                    checkCompatibility(roseVersion);
                    *binary_archive_ >>BOOST_SERIALIZATION_NVP(object);
                    break;
                case TEXT:
                    ASSERT_not_null(text_archive_);
                    *text_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
                    checkCompatibility(roseVersion);
                    *text_archive_ >>BOOST_SERIALIZATION_NVP(object);
                    break;
                case XML:
                    ASSERT_not_null(xml_archive_);
                    *xml_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
                    checkCompatibility(roseVersion);
                    *xml_archive_ >>BOOST_SERIALIZATION_NVP(object);
                    break;
            }
#if !defined(ROSE_DEBUG_SERIAL_IO)
        } catch (const std::exception &e) {
            *errorMessage = e.what();
        } catch (...) {
            *errorMessage = "failed to read object from input stream";
        }
#endif
#endif
    }
 
    void checkCompatibility(const std::string &fileVersion);
 
protected:
    // Read the next object type from the input stream
    void advanceObjectType();
};
 
} // namespace
} // namespace
 
#endif
#endif