ROSE  0.11.145.0
general_token_defs.h
1 /* DQ (9/24/2007): This is part of Rama's Lexer */
2 #ifndef ROSE_LEX_TOKEN_DEFINITIONS
3 #define ROSE_LEX_TOKEN_DEFINITIONS 1
4 
6 
7 // These have been moved from the preproc.ll file so that they can
8 // be seen by other parst of ROSE (e.g. in the token processing).
9 namespace ROSE_token_ids
10  {
11  // whitespace token IDs
12  enum whitespace
13  {
14  // DQ (9/2/2015): Removed comma to avoid Intel v14 compiler warning.
15  T_NOTKNOWN
16  // The following are non UNIX whitespace
17  };
18 
19 #if 0
20  //I brazenly copied from the interface of WAVE for these enums
21  enum ROSE_C_CXX_keywords
22  {
23  ROSE_ASM /* WAVE equivalent is T_ASM */
24  ROSE_AUTO /* WAVE equivalent is T_AUTO */
25  ROSE_BOOL /* WAVE equivalent is T_BOOL */
26  ROSE_BREAK /* WAVE equivalent is T_BREAK */
27  ROSE_CASE /* WAVE equivalent is T_CASE */
28  ROSE_CATCH /* WAVE equivalent is T_CATCH */
29  ROSE_CHAR /* WAVE equivalent is T_CHAR */
30  ROSE_CLASS /* WAVE equivalent is T_CLASS */
31  ROSE_CONST /* WAVE equivalent is T_CONST */
32  ROSE_CONSTCAST /* WAVE equivalent is T_CONSTCAST */
33  ROSE_CONTINUE /* WAVE equivalent is T_CONTINUE */
34  ROSE_DEFAULT /* WAVE equivalent is T_DEFAULT */
35  ROSE_DEFINED /* WAVE equivalent is T_DEFINED */
36  ROSE_DELETE /* WAVE equivalent is T_DELETE */
37  ROSE_DO /* WAVE equivalent is T_DO */
38  ROSE_DOUBLE /* WAVE equivalent is T_DOUBLE */
39  ROSE_DYNAMICCAST /* WAVE equivalent is T_DYNAMICCAST */
40  ROSE_ELSE /* WAVE equivalent is T_ELSE */
41  ROSE_ENUM /* WAVE equivalent is T_ENUM */
42  ROSE_EXPLICIT /* WAVE equivalent is T_EXPLICIT */
43  ROSE_EXPORT /* WAVE equivalent is T_EXPORT */
44  ROSE_EXTERN /* WAVE equivalent is T_EXTERN */
45  ROSE_FLOAT /* WAVE equivalent is T_FLOAT */
46  ROSE_FOR /* WAVE equivalent is T_FOR */
47  ROSE_FRIEND /* WAVE equivalent is T_FRIEND */
48  ROSE_GOTO /* WAVE equivalent is T_GOTO */
49  ROSE_IF /* WAVE equivalent is T_IF */
50  ROSE_INLINE /* WAVE equivalent is T_INLINE */
51  ROSE_INT /* WAVE equivalent is T_INT */
52  ROSE_LONG /* WAVE equivalent is T_LONG */
53  ROSE_MUTABLE /* WAVE equivalent is T_MUTABLE */
54  ROSE_NAMESPACE /* WAVE equivalent is T_NAMESPACE */
55  ROSE_NEW /* WAVE equivalent is T_NEW */
56  ROSE_OPERATOR /* WAVE equivalent is T_OPERATOR */
57  ROSE_PRIVATE /* WAVE equivalent is T_PRIVATE */
58  ROSE_PROTECTED /* WAVE equivalent is T_PROTECTED */
59  ROSE_PUBLIC /* WAVE equivalent is T_PUBLIC */
60  ROSE_REGISTER /* WAVE equivalent is T_REGISTER */
61  ROSE_REINTERPRETCAST /* WAVE equivalent is T_REINTERPRETCAST */
62  ROSE_RETURN /* WAVE equivalent is T_RETURN */
63  ROSE_SHORT /* WAVE equivalent is T_SHORT */
64  ROSE_SIGNED /* WAVE equivalent is T_SIGNED */
65  ROSE_SIZEOF /* WAVE equivalent is T_SIZEOF */
66  ROSE_STATIC /* WAVE equivalent is T_STATIC */
67  ROSE_STATICCAST /* WAVE equivalent is T_STATICCAST */
68  ROSE_STRUCT /* WAVE equivalent is T_STRUCT */
69  ROSE_SWITCH /* WAVE equivalent is T_SWITCH */
70  ROSE_TEMPLATE /* WAVE equivalent is T_TEMPLATE */
71  ROSE_THIS /* WAVE equivalent is T_THIS */
72  ROSE_THROW /* WAVE equivalent is T_THROW */
73  ROSE_TRY /* WAVE equivalent is T_TRY */
74  ROSE_TYPEDEF /* WAVE equivalent is T_TYPEDEF */
75  ROSE_TYPEID /* WAVE equivalent is T_TYPEID */
76  ROSE_TYPENAME /* WAVE equivalent is T_TYPENAME */
77  ROSE_UNION /* WAVE equivalent is T_UNION */
78  ROSE_UNSIGNED /* WAVE equivalent is T_UNSIGNED */
79  ROSE_USING /* WAVE equivalent is T_USING */
80  ROSE_VIRTUAL /* WAVE equivalent is T_VIRTUAL */
81  ROSE_VOID /* WAVE equivalent is T_VOID */
82  ROSE_VOLATILE /* WAVE equivalent is T_VOLATILE */
83  ROSE_WCHART /* WAVE equivalent is T_WCHART */
84  ROSE_WHILE /* WAVE equivalent is T_WHILE */
85  };
86 #endif
87 
88  enum ROSE_C_CXX_keywords
89  {
90  C_CXX_ASM = 0 + 500,
91  C_CXX_AUTO = 1 + 500,
92  C_CXX_BOOL = 2 + 500,
93  C_CXX_BREAK = 3 + 500,
94  C_CXX_CASE = 4 + 500,
95  C_CXX_CATCH = 5 + 500,
96  C_CXX_CHAR = 6 + 500,
97  C_CXX_CLASS = 7 + 500,
98  C_CXX_CONST = 8 + 500,
99  C_CXX_CONSTCAST = 9 + 500,
100  C_CXX_CONTINUE = 10 + 500,
101  C_CXX_DEFAULT = 11 + 500,
102  C_CXX_DEFINED = 12 + 500,
103  C_CXX_DELETE = 13 + 500,
104  C_CXX_DO = 14 + 500,
105  C_CXX_DOUBLE = 15 + 500,
106  C_CXX_DYNAMICCAST = 16 + 500,
107  C_CXX_ELSE = 17 + 500,
108  C_CXX_ENUM = 18 + 500,
109  C_CXX_EXPLICIT = 19 + 500,
110  C_CXX_EXPORT = 20 + 500,
111  C_CXX_EXTERN = 21 + 500,
112  C_CXX_FALSE = 22 + 500,
113  C_CXX_FLOAT = 23 + 500,
114  C_CXX_FOR = 24 + 500,
115  C_CXX_FRIEND = 25 + 500,
116  C_CXX_GOTO = 26 + 500,
117  C_CXX_IF = 27 + 500,
118  C_CXX_INLINE = 28 + 500,
119  C_CXX_INT = 29 + 500,
120  C_CXX_LONG = 30 + 500,
121  C_CXX_MUTABLE = 31 + 500,
122  C_CXX_NAMESPACE = 32 + 500,
123  C_CXX_NEW = 33 + 500,
124  C_CXX_OPERATOR = 34 + 500,
125  C_CXX_PRIVATE = 35 + 500,
126  C_CXX_PROTECTED = 36 + 500,
127  C_CXX_PUBLIC = 37 + 500,
128  C_CXX_REGISTER = 38 + 500,
129  C_CXX_REINTERPRETCAST = 39 + 500,
130  C_CXX_RETURN = 40 + 500,
131  C_CXX_SHORT = 41 + 500,
132  C_CXX_SIGNED = 42 + 500,
133  C_CXX_SIZEOF = 43 + 500,
134  C_CXX_STATIC = 44 + 500,
135  C_CXX_STATICCAST = 45 + 500,
136  C_CXX_STRUCT = 46 + 500,
137  C_CXX_SWITCH = 47 + 500,
138  C_CXX_TEMPLATE = 48 + 500,
139  C_CXX_THIS = 49 + 500,
140  C_CXX_THROW = 50 + 500,
141  C_CXX_TRY = 51 + 500,
142  C_CXX_TRUE = 52 + 500,
143  C_CXX_TYPEDEF = 53 + 500,
144  C_CXX_TYPEID = 54 + 500,
145  C_CXX_TYPENAME = 55 + 500,
146  C_CXX_UNION = 56 + 500,
147  C_CXX_UNSIGNED = 57 + 500,
148  C_CXX_USING = 58 + 500,
149  C_CXX_VIRTUAL = 59 + 500,
150  C_CXX_VOID = 60 + 500,
151  C_CXX_VOLATILE = 61 + 500,
152  C_CXX_WCHART = 62 + 500,
153  C_CXX_WHILE = 63 + 500
154  };
155 
156  //the following are alternate tokens.
157  //Refer to page 12 of C++ STD
158 
159 enum ROSE_C_CXX_alternate_tok
160  {
161  C_CXX_LEFT_CURLY_ALT, // The <% which is the same as {
162  C_CXX_RIGHT_CURLY_ALT, //The %> which is the same as }
163  C_CXX_LEFT_SQUARE_ALT, //The <: which is the same as [
164  C_CXX_RIGHT_SQUARE_ALT,//The :> which is the same as ]
165  C_CXX_HASH_ALT, // The #
166  C_CXX_HASH_HASH_ALT, // The
167  // DQ (9/2/2015): Removed comma to avoid Intel v14 compiler warning.
168  C_CXX_and_ALT // The #
169  };
170 
171 enum ROSE_C_CXX_Additional_Info
172  {
173  C_CXX_COMMENTS = 0 + 100000,
174  C_CXX_STRING_LITERALS = 1 + 100000,
175  C_CXX_IDENTIFIER = 2 + 100000,
176  C_CXX_PREPROCESSING_INFO = 3 + 100000,
177  C_CXX_UNIDENTIFIED_TOKEN = 4 + 100000,
178  // DQ (10/12/2013): Added C_CXX_SYNTAX, C_CXX_WHITESPACE, and C_CXX_PRAGMA
179  C_CXX_SYNTAX = 5 + 100000,
180  C_CXX_WHITESPACE = 6 + 100000,
181  C_CXX_PRAGMA = 7 + 100000,
182  C_CXX_ERROR = 8 + 100000
183  };
184 
185 };
186 
187 
189  {
190  // This is the simplist contect of a Token
191 
192  // value of the token
193  std::string token_lexeme;
194 
195  // A token can also be an already processed CPP directive or comment (PreprocessingInfo)
196  // PreprocessingInfo* token_preprocessingInfo;
197 
198  // The classification for the token (language dependent)
199  int token_id;
200  };
201 
203  {
204  // Source code position information for token
205  int line_num;
206  int column_num;
207  };
208 
209 // At some point I want to replace this with a SgToken
210 // IR node until then this is an intermediate step.
212  {
213  // This is the element in the token stream.
214 
215  // This is the pointer to the token
216  struct token_element * p_tok_elem;
217 
218  // The token stream can also include references to already processed CPP directives or comments (PreprocessingInfo)
219  PreprocessingInfo* p_preprocessingInfo;
220 
221  // Positon of the start of the token
222  struct file_pos_info beginning_fpi;
223 
224  // Position of the end of the token
225  struct file_pos_info ending_fpi;
226  };
227 
228 // At present this is an STL list, but I would like it to be a vector at some point
229 typedef std::list<stream_element*> LexTokenStreamType;
230 typedef LexTokenStreamType* LexTokenStreamTypePointer;
231 
232 // endif for ROSE_LEX_TOKEN_DEFINITIONS
233 #endif
For preprocessing information including source comments, include , if, define, etc.