ROSE  0.11.101.0
SerialIo.h
1 #ifndef ROSE_BinaryAnalysis_SerialIo_H
2 #define ROSE_BinaryAnalysis_SerialIo_H
3 #include <featureTests.h>
4 #ifdef ROSE_ENABLE_BINARY_ANALYSIS
5 
6 // Define this if you need to debug SerialIo -- it causes everything to run in the calling thread and avoid catching exceptions.
7 //#define ROSE_DEBUG_SERIAL_IO
8 
9 #if defined(BOOST_WINDOWS)
10  // Lacks POSIX file system, so we can't monitor the I/O progress
11  #undef ROSE_SUPPORTS_SERIAL_IO
12 #elif !defined(ROSE_HAVE_BOOST_SERIALIZATION_LIB)
13  // Lacks Boost's serialization library, which is how we convert objects to bytes and vice versa
14  #undef ROSE_SUPPORTS_SERIAL_IO
15 #elif defined(__clang__)
16  #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
17 #elif defined(__GNUC__)
18  #if __GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNU_C_PATCHLEVEL__ <= 40204
19  // GCC <= 4.2.4 gets segfaults compiling this file
20  #undef ROSE_SUPPORTS_SERIAL_IO
21  #else
22  #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
23  #endif
24 #else
25  #define ROSE_SUPPORTS_SERIAL_IO /*supported*/
26 #endif
27 
28 // These have to be included early, before the definitions of the ROSE classes that are serialized
29 #ifdef ROSE_SUPPORTS_SERIAL_IO
30 #include <boost/archive/binary_iarchive.hpp>
31 #include <boost/archive/binary_oarchive.hpp>
32 #include <boost/archive/text_iarchive.hpp>
33 #include <boost/archive/text_oarchive.hpp>
34 #include <boost/archive/xml_iarchive.hpp>
35 #include <boost/archive/xml_oarchive.hpp>
36 #include <boost/iostreams/device/file_descriptor.hpp>
37 #include <boost/iostreams/stream.hpp>
38 #endif
39 
40 #include <Rose/Progress.h>
41 #include <Rose/Exception.h>
42 #include <boost/filesystem.hpp>
43 #include <boost/lexical_cast.hpp>
44 #include <boost/thread.hpp>
45 #include <Sawyer/Message.h>
46 #include <Sawyer/ProgressBar.h>
47 #include <Sawyer/Synchronization.h>
48 
49 namespace Rose {
50 namespace BinaryAnalysis {
51 
52 namespace Partitioner2 {
53 class Partitioner;
54 }
55 
57 // SerialIo
59 
117 public:
119  using Ptr = SerialIoPtr;
120 
122  enum Format {
130  };
131 
133  enum Savable {
134  NO_OBJECT = 0x00000000,
135  PARTITIONER = 0x00000001,
136  AST = 0x00000002,
137  END_OF_DATA = 0x0000fffe,
138  ERROR = 0x0000ffff,
139  USER_DEFINED = 0x00010000,
140  USER_DEFINED_LAST = 0xffffffff
141  };
142 
144  class Exception: public Rose::Exception {
145  public:
147  explicit Exception(const std::string &s): Rose::Exception(s) {}
148  ~Exception() throw() {}
149  };
150 
151 private:
152  mutable SAWYER_THREAD_TRAITS::Mutex mutex_; // protects the following data members
153  Format format_;
154  Progress::Ptr progress_;
155  bool isOpen_;
156  Savable objectType_;
157 
158 protected:
159  Sawyer::ProgressBar<size_t> progressBar_;
160 
161  // We use low-level file descriptors under an std::stream I/O interface in order to provide progress reports. This
162  // allows one thread to be reading from or writing to the file and another thread to monitor the file position to
163  // report progress. The C++ standard library doesn't have an API for wrapping file descriptors in a stream interface,
164  // so we use Boost.
165  int fd_;
166 
167 protected:
168  SerialIo()
169  : format_(BINARY), progress_(Progress::instance()), isOpen_(false), objectType_(NO_OBJECT),
170  progressBar_(mlog[Sawyer::Message::MARCH]), fd_(-1) {
171  init();
172  progressBar_.suffix(" bytes");
173  }
174 
175 public:
178 
186  virtual ~SerialIo();
187 
191  static Savable userSavable(unsigned offset);
192 
201  Format format() const;
202  void format(Format);
213  Progress::Ptr progress() const;
214  void progress(const Progress::Ptr&);
225  virtual void open(const boost::filesystem::path&) = 0;
226 
238  virtual void close() = 0;
239 
245  bool isOpen() const;
246 
251  Savable objectType() const;
252 
253 protected:
254  // Set or clear the isOpen flag.
255  void setIsOpen(bool b);
256 
257  // Set object type to ERROR to indicate that a read was unsuccessful
258  void objectType(Savable);
259 
260 private:
261  void init();
262 };
263 
264 
266 // SerialOutput
268 
272 class SerialOutput: public SerialIo {
273 public:
276 
277 private:
278 #ifdef ROSE_SUPPORTS_SERIAL_IO
279  boost::iostreams::file_descriptor_sink device_;
280  boost::iostreams::stream<boost::iostreams::file_descriptor_sink> file_;
281  boost::archive::binary_oarchive *binary_archive_;
282  boost::archive::text_oarchive *text_archive_;
283  boost::archive::xml_oarchive *xml_archive_;
284 #endif
285 
286 protected:
287 #ifdef ROSE_SUPPORTS_SERIAL_IO
288  SerialOutput(): binary_archive_(NULL), text_archive_(NULL), xml_archive_(NULL) {}
289 #else
290  SerialOutput() {}
291 #endif
292 
293 public:
294  ~SerialOutput();
295  void open(const boost::filesystem::path &fileName) override;
296  void close() override;
297 
302  static Ptr instance() { return Ptr(new SerialOutput); }
303 
314 
326  void saveAst(SgAsmNode*);
327  void saveAst(SgBinaryComposite*);
328 private:
329  // The saveAstHelper is what actually gets called for the functions above. It doesn't descriminate between nodes that
330  // support serialization and those that don't, which is why it's private. Use only the public functions because they'll
331  // give you a nice compiler error if you try to save an Ast node type that isn't supported.
332  void saveAstHelper(SgNode*);
333 public:
334 
349  template<class T>
350  void saveObject(Savable objectTypeId, const T &object) {
351  if (!isOpen())
352  throw Exception("cannot save object when no file is open");
353  if (ERROR == objectType())
354  throw Exception("cannot save object because stream is in error state");
355 
356 #ifndef ROSE_SUPPORTS_SERIAL_IO
357  throw Exception("binary state files are not supported in this configuration");
358 #elif defined(ROSE_DEBUG_SERIAL_IO)
359  std::string errorMessage;
360  asyncSave(objectTypeId, object, &errorMessage);
361 #else
362  // A different thread saves the object while this thread updates the progress
363  std::string errorMessage;
364  boost::thread worker(startWorker<T>, this, objectTypeId, &object, &errorMessage);
365  boost::chrono::milliseconds timeout((unsigned)(1000 * Sawyer::ProgressBarSettings::minimumUpdateInterval()));
366  progressBar_.prefix("writing");
367  while (!worker.try_join_for(timeout)) {
368  off_t cur = ::lseek(fd_, 0, SEEK_CUR);
369  if (-1 == cur) {
370  ++progressBar_; // so a spinner moves
371  } else {
372  progressBar_.value(cur);
373  if (Progress::Ptr p = progress())
374  p->update(Progress::Report(cur, NAN));
375  }
376  }
377  if (!errorMessage.empty())
378  throw Exception(errorMessage);
379 #endif
380  }
381 
382 private:
383  template<class T>
384  static void startWorker(SerialOutput *saver, Savable objectTypeId, const T *object, std::string *errorMessage) {
385  ASSERT_not_null(object);
386  saver->asyncSave(objectTypeId, *object, errorMessage);
387  }
388 
389  // This might run in its own thread.
390  template<class T>
391  void asyncSave(Savable objectTypeId, const T &object, std::string *errorMessage) {
392  ASSERT_not_null(errorMessage);
393 #ifndef ROSE_SUPPORTS_SERIAL_IO
394  ASSERT_not_reachable("not supported in this configuration");
395 #else
396 #if !defined(ROSE_DEBUG_SERIAL_IO)
397  try {
398 #endif
399  std::string roseVersion = ROSE_PACKAGE_VERSION;
400  objectType(ERROR);
401  switch (format()) {
402  case BINARY:
403  ASSERT_not_null(binary_archive_);
404  *binary_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
405  *binary_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
406  *binary_archive_ <<BOOST_SERIALIZATION_NVP(object);
407  break;
408  case TEXT:
409  ASSERT_not_null(text_archive_);
410  *text_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
411  *text_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
412  *text_archive_ <<BOOST_SERIALIZATION_NVP(object);
413  break;
414  case XML:
415  ASSERT_not_null(xml_archive_);
416  *xml_archive_ <<BOOST_SERIALIZATION_NVP(objectTypeId);
417  *xml_archive_ <<BOOST_SERIALIZATION_NVP(roseVersion);
418  *xml_archive_ <<BOOST_SERIALIZATION_NVP(object);
419  break;
420  }
421  objectType(objectTypeId);
422 #if !defined(ROSE_DEBUG_SERIAL_IO)
423  } catch (const Exception &e) {
424  *errorMessage = e.what();
425  } catch (...) {
426  *errorMessage = "failed to write object to output stream";
427  }
428 #endif
429 #endif
430  }
431 };
432 
433 
435 // SerialInput
437 
441 class SerialInput: public SerialIo {
442 public:
445 
446 private:
447 #ifdef ROSE_SUPPORTS_SERIAL_IO
448  size_t fileSize_;
449  boost::iostreams::file_descriptor_source device_;
450  boost::iostreams::stream<boost::iostreams::file_descriptor_source> file_;
451  boost::archive::binary_iarchive *binary_archive_;
452  boost::archive::text_iarchive *text_archive_;
453  boost::archive::xml_iarchive *xml_archive_;
454 #endif
455 
456 protected:
457 #ifdef ROSE_SUPPORTS_SERIAL_IO
458  SerialInput(): fileSize_(0), binary_archive_(NULL), text_archive_(NULL), xml_archive_(NULL) {}
459 #else
460  SerialInput() {}
461 #endif
462 
463 public:
464  ~SerialInput();
465  void open(const boost::filesystem::path &fileName) override;
466  void close() override;
467 
472  static Ptr instance() { return Ptr(new SerialInput); }
473 
482 
490 
498  SgNode* loadAst();
499 
506  template<class T>
507  T loadObject(Savable objectTypeId) {
508  T object;
509  loadObject<T>(objectTypeId, object);
510  return object;
511  }
512  template<class T>
513  void loadObject(Savable objectTypeId, T &object) {
514  if (!isOpen())
515  throw Exception("cannot load object when no file is open");
516 
517 #ifndef ROSE_SUPPORTS_SERIAL_IO
518  throw Exception("binary state files are not supported in this configuration");
519 #else
520  if (ERROR == objectType())
521  throw Exception("cannot read object because stream is in error state");
522  if (objectType() != objectTypeId) {
523  throw Exception("unexpected object type (expected " + boost::lexical_cast<std::string>(objectTypeId) +
524  " but read " + boost::lexical_cast<std::string>(objectType()) + ")");
525  }
526  objectType(ERROR); // in case of exception
527  std::string errorMessage;
528 #ifdef ROSE_DEBUG_SERIAL_IO
529  asyncLoad(object, &errorMessage);
530 #else
531  boost::thread worker(startWorker<T>, this, &object, &errorMessage);
532  boost::chrono::milliseconds timeout((unsigned)(1000 * Sawyer::ProgressBarSettings::minimumUpdateInterval()));
533  progressBar_.prefix("reading");
534  while (!worker.try_join_for(timeout)) {
535  if (fileSize_ > 0) {
536  off_t cur = ::lseek(fd_, 0, SEEK_CUR);
537  if (cur != -1) {
538  progressBar_.value(cur);
539  if (Progress::Ptr p = progress())
540  p->update(Progress::Report(cur, fileSize_));
541  }
542  } else {
543  ++progressBar_; // so the spinner moves
544  }
545  }
546  if (!errorMessage.empty())
547  throw Exception(errorMessage);
548 #endif
549  advanceObjectType();
550 #endif
551  }
554 private:
555  template<class T>
556  static void startWorker(SerialInput *loader, T *object, std::string *errorMessage) {
557  loader->asyncLoad(*object, errorMessage);
558  }
559 
560  // Might run in its own thread
561  template<class T>
562  void asyncLoad(T &object, std::string *errorMessage) {
563  ASSERT_not_null(errorMessage);
564 #ifndef ROSE_SUPPORTS_SERIAL_IO
565  ASSERT_not_reachable("not supported in this configuration");
566 #else
567 #if !defined(ROSE_DEBUG_SERIAL_IO)
568  try {
569 #endif
570  std::string roseVersion;
571  switch (format()) {
572  case BINARY:
573  ASSERT_not_null(binary_archive_);
574  *binary_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
575  checkCompatibility(roseVersion);
576  *binary_archive_ >>BOOST_SERIALIZATION_NVP(object);
577  break;
578  case TEXT:
579  ASSERT_not_null(text_archive_);
580  *text_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
581  checkCompatibility(roseVersion);
582  *text_archive_ >>BOOST_SERIALIZATION_NVP(object);
583  break;
584  case XML:
585  ASSERT_not_null(xml_archive_);
586  *xml_archive_ >>BOOST_SERIALIZATION_NVP(roseVersion);
587  checkCompatibility(roseVersion);
588  *xml_archive_ >>BOOST_SERIALIZATION_NVP(object);
589  break;
590  }
591 #if !defined(ROSE_DEBUG_SERIAL_IO)
592  } catch (const Exception &e) {
593  *errorMessage = e.what();
594  } catch (...) {
595  *errorMessage = "failed to read object from input stream";
596  }
597 #endif
598 #endif
599  }
600 
601  void checkCompatibility(const std::string &fileVersion);
602 
603 protected:
604  // Read the next object type from the input stream
605  void advanceObjectType();
606 };
607 
608 } // namespace
609 } // namespace
610 
611 #endif
612 #endif
Savable
Types of objects that can be saved.
Definition: SerialIo.h:133
A single progress report.
Definition: Progress.h:179
double minimumUpdateInterval()
Minimum time between updates.
Base class for all binary analysis IR nodes.
Textual binary state files use a custom format (Boost serialization format) that stores the data as A...
Definition: SerialIo.h:125
void savePartitioner(const Partitioner2::Partitioner &)
Save a binary analysis partitioner.
static Savable userSavable(unsigned offset)
Create a new Savable enum constant.
Collection of streams.
Definition: Message.h:1606
First user-defined object number.
Definition: SerialIo.h:139
void saveObject(Savable objectTypeId, const T &object)
Save an object to the output stream.
Definition: SerialIo.h:350
Abstract syntax tree.
Definition: SerialIo.h:136
Format format() const
Property: File format.
Output binary analysis state.
Definition: SerialIo.h:272
Rose::BinaryAnalysis::Partitioner2::Partitioner.
Definition: SerialIo.h:135
void saveAst(SgAsmNode *)
Save a binary AST.
void open(const boost::filesystem::path &fileName) override
Attach a file.
void close() override
Detach a file.
static Sawyer::Message::Facility mlog
Message facility.
Definition: SerialIo.h:177
Main namespace for the ROSE library.
Base class for binary state input and output.
Definition: SerialIo.h:116
Progress::Ptr progress() const
Property: Progress reporter.
SgNode * loadAst()
Load an AST from the input stream.
Reference-counting intrusive smart pointer.
Definition: SharedPointer.h:68
SerialInputPtr Ptr
Reference counting pointer.
Definition: SerialIo.h:444
Name space for the entire library.
Definition: FeasiblePath.h:773
Object type for newly-initialized serializers.
Definition: SerialIo.h:134
Savable objectType() const
Type ID for next object.
Exception(const std::string &s)
Construct an exception with an error message.
Definition: SerialIo.h:147
virtual void open(const boost::filesystem::path &)=0
Attach a file.
Marks that the stream has encountered an error condition.
Definition: SerialIo.h:138
static Ptr instance()
Factory method to create a new instance.
Definition: SerialIo.h:302
void close() override
Detach a file.
The states are stored as XML, which is a very verbose and slow format.
Definition: SerialIo.h:128
This class represents the base class for all IR nodes within Sage III.
Definition: Cxx_Grammar.h:9433
static Ptr instance()
Factory method to create a new instance.
Definition: SerialIo.h:472
ValueType value() const
Value for the progress bar.
Definition: ProgressBar.h:159
SerialOutputPtr Ptr
Reference counting pointer.
Definition: SerialIo.h:275
T loadObject(Savable objectTypeId)
Load an object from the input stream.
Definition: SerialIo.h:507
Partitioner2::Partitioner loadPartitioner()
Load a partitioner from the input stream.
Binary analysis.
bool isOpen() const
Whether a file is attached.
const std::string & prefix() const
String to show before the beginning of the bar.
Definition: ProgressBar.h:290
void loadObject(Savable objectTypeId, T &object)
Load an object from the input stream.
Definition: SerialIo.h:513
virtual ~SerialIo()
Destructor.
Suffix suffix()
Property: suffix.
Definition: ProgressBar.h:190
Base class for reference counted objects.
Definition: SharedObject.h:64
void open(const boost::filesystem::path &fileName) override
Attach a file.
A general, thread-safe way to report progress made on some task.
Definition: Progress.h:165
Marks the end of the data stream.
Definition: SerialIo.h:137
Errors thrown by this API.
Definition: SerialIo.h:144
Format
Format of the state file.
Definition: SerialIo.h:122
Partitions instructions into basic blocks and functions.
Definition: Partitioner.h:294
Base class for all ROSE exceptions.
Definition: Rose/Exception.h:9
Binary state files are smaller and faster than the other formats, but are not portable across archite...
Definition: SerialIo.h:123
Last user-defined object number.
Definition: SerialIo.h:140
virtual void close()=0
Detach a file.
Input binary analysis state.
Definition: SerialIo.h:441
Savable nextObjectType()
Type of next object in the input stream.