ROSE  0.9.9.139
Lexer.h
1 // WARNING: Changes to this file must be contributed back to Sawyer or else they will
2 // be clobbered by the next update from Sawyer. The Sawyer repository is at
3 // https://github.com/matzke1/sawyer.
4 
5 
6 
7 
8 #ifndef Sawyer_Lexer_H
9 #define Sawyer_Lexer_H
10 
11 #include <Sawyer/AllocatingBuffer.h>
12 #include <Sawyer/LineVector.h>
13 #include <Sawyer/Optional.h>
14 #include <Sawyer/Sawyer.h>
15 
16 #include <boost/filesystem.hpp>
17 
18 namespace Sawyer {
19 
20 namespace Lexer {
21 
27 template<typename T>
28 class SAWYER_EXPORT Token {
29 public:
30  typedef T TokenEnum;
31 
32 private:
33  Optional<TokenEnum> type_; // empty means EOF
34  size_t begin_; // beginning character position in the input
35  size_t end_; // ending (exclusive) character position in the input
36 
37 public:
39  Token(): begin_(0), end_(0) {}
40 
47  Token(TokenEnum type, size_t begin, size_t end)
48  : type_(type), begin_(begin), end_(end) {
49  ASSERT_require(end >= begin);
50  }
51 
56  bool isEof() const {
57  if (type_)
58  return false;
59  return true;
60  }
61 
65  TokenEnum type() const {
66  return *type_;
67  }
68 
72  size_t begin() const {
73  return begin_;
74  }
75 
81  size_t end() const {
82  return end_;
83  }
84 };
85 
90 template<class T>
91 class SAWYER_EXPORT TokenStream {
92 public:
93  typedef T Token;
94 
95 private:
96  std::string name_; // name of stream (e.g., a file name)
97  Container::LineVector content_; // line-oriented character contents of the stream
98  size_t at_; // cursor position in stream
99  std::vector<Token> tokens_; // current token at [0] plus lookahead
100 
101 public:
102  virtual ~TokenStream() {}
103 
105  explicit TokenStream(const boost::filesystem::path &fileName)
106  : name_(fileName.string()), content_(fileName.string()), at_(0) {}
107 
112  explicit TokenStream(const std::string &inputString)
113  : name_("string"), content_(Container::AllocatingBuffer<size_t, char>::instance(inputString)), at_(0) {}
114 
119  : name_("string"), content_(buffer), at_(0) {}
120 
122  const std::string& name() const {
123  return name_;
124  }
125 
129  const Token& current() {
130  return (*this)[0];
131  }
132 
136  bool atEof() {
137  return current().isEof();
138  }
139 
144  const Token& operator[](size_t lookahead) {
145  static const Token eof_;
146  while (lookahead >= tokens_.size()) {
147  if (!tokens_.empty() && tokens_.back().isEof())
148  return eof_;
149  tokens_.push_back(scanNextToken(content_, at_/*in,out*/));
150  }
151  return tokens_[lookahead];
152  }
153 
158  void consume(size_t n = 1) {
159  const Token &t = current();
160  if (t.isEof()) {
161  // void
162  } else if (n >= tokens_.size()) {
163  tokens_.clear();
164  } else {
165  tokens_.erase(tokens_.begin(), tokens_.begin() + n);
166  }
167  }
168 
182  std::string lexeme(const Token &t) {
183  return std::string(content_.characters(t.begin()), t.end() - t.begin());
184  }
185  std::string lexeme() {
186  return lexeme(current());
187  }
196  bool isa(const Token &t, typename Token::TokenEnum type) {
197  return !t.isEof() && t.type() == type;
198  }
199 
200  bool isa(typename Token::TokenEnum type) {
201  return isa(current(), type);
202  }
213  bool match(const Token &t, const char *s) {
214  ASSERT_not_null(s);
215  size_t n1 = t.end() - t.begin();
216  size_t n2 = strlen(s);
217  if (n1 != n2)
218  return false;
219  const char *lexeme = content_.characters(t.begin());
220  return 0 == strncmp(lexeme, s, n1);
221  }
222  bool match(const char *s) {
223  return match(current(), s);
224  }
231  std::pair<size_t, size_t> location(size_t position) {
232  return content_.location(position);
233  }
234 
236  std::pair<size_t, size_t> locationEof() {
237  size_t nChars = content_.nCharacters();
238  return nChars > 0 ? content_.location(nChars-1) : content_.location(0);
239  }
240 
242  std::string lineString(size_t lineIdx) {
243  return content_.lineString(lineIdx);
244  }
245 
253  virtual Token scanNextToken(const Container::LineVector &content, size_t &at /*in,out*/) = 0;
254 };
255 
256 } // namespace
257 } // namespace
258 
259 #endif
260 
TokenStream(const std::string &inputString)
Create a token stream from a string.
Definition: Lexer.h:112
A buffer of characters indexed by line number.
Definition: LineVector.h:23
TokenStream(const Container::Buffer< size_t, char >::Ptr &buffer)
Create a token stream from a buffer.
Definition: Lexer.h:118
const Token & operator[](size_t lookahead)
Return the current or future token.
Definition: Lexer.h:144
std::pair< size_t, size_t > location(size_t charIndex) const
Convert a character index to a line and column index.
size_t end() const
Token lexeme ending position.
Definition: Lexer.h:81
bool isEof() const
Whether this is an EOF token.
Definition: Lexer.h:56
std::string lineString(size_t lineIdx) const
Line as a string.
size_t nCharacters() const
Number of characters.
Definition: LineVector.h:73
Represents one token of input.
Definition: Lexer.h:28
Reference-counting smart pointer.
Definition: SharedPointer.h:34
Name space for the entire library.
Definition: Access.h:11
std::string lineString(size_t lineIdx)
Return the entire string for some line index.
Definition: Lexer.h:242
Token()
Construct an EOF token.
Definition: Lexer.h:39
bool match(const Token &t, const char *s)
Determine whether a token matches a string.
Definition: Lexer.h:213
bool match(const char *s)
Determine whether a token matches a string.
Definition: Lexer.h:222
size_t begin() const
Token lexeme starting position.
Definition: Lexer.h:72
std::pair< size_t, size_t > location(size_t position)
Return the line number and offset for an input position.
Definition: Lexer.h:231
TokenEnum type() const
Returns the token.
Definition: Lexer.h:65
std::string lexeme()
Return the lexeme for a token.
Definition: Lexer.h:185
std::pair< size_t, size_t > locationEof()
Returns the last line index and character offset.
Definition: Lexer.h:236
bool atEof()
Returns true if the stream is at the end.
Definition: Lexer.h:136
TokenStream(const boost::filesystem::path &fileName)
Create a token stream from the contents of a file.
Definition: Lexer.h:105
std::string lexeme(const Token &t)
Return the lexeme for a token.
Definition: Lexer.h:182
bool isa(const Token &t, typename Token::TokenEnum type)
Determine whether token is a specific type.
Definition: Lexer.h:196
Token(TokenEnum type, size_t begin, size_t end)
Construct a token.
Definition: Lexer.h:47
void consume(size_t n=1)
Consume some tokens.
Definition: Lexer.h:158
bool isa(typename Token::TokenEnum type)
Determine whether token is a specific type.
Definition: Lexer.h:200
const char * characters(size_t charIdx) const
Characters at file offset.
An ordered list of tokens scanned from input.
Definition: Lexer.h:91
const Token & current()
Return the current token.
Definition: Lexer.h:129
const std::string & name() const
Property: Name of stream.
Definition: Lexer.h:122