comparison: src/lexer.cc
src/lexer.cc
- changeset 75
- bf8c57437231
- parent 74
- 007fbadfa7f9
- child 79
- 2425fa6a4f21
equal
deleted
inserted
replaced
| 74:007fbadfa7f9 | 75:bf8c57437231 |
|---|---|
| 1 /* | 1 /* |
| 2 Copyright (c) 2013-2014, Santeri Piippo | 2 Copyright (c) 2014, Santeri Piippo |
| 3 All rights reserved. | 3 All rights reserved. |
| 4 | 4 |
| 5 Redistribution and use in source and binary forms, with or without | 5 Redistribution and use in source and binary forms, with or without |
| 6 modification, are permitted provided that the following conditions are met: | 6 modification, are permitted provided that the following conditions are met: |
| 7 | 7 |
| 55 if (fp == null) | 55 if (fp == null) |
| 56 error ("couldn't open %1 for reading: %2", file_name, strerror (errno)); | 56 error ("couldn't open %1 for reading: %2", file_name, strerror (errno)); |
| 57 | 57 |
| 58 lexer_scanner sc (fp); | 58 lexer_scanner sc (fp); |
| 59 | 59 |
| 60 devf ("Processing tokens...\n"); | |
| 60 while (sc.get_next_token()) | 61 while (sc.get_next_token()) |
| 61 { | 62 { |
| 63 devf (".\n"); | |
| 62 // Preprocessor commands: | 64 // Preprocessor commands: |
| 63 if (sc.get_token_type() == tk_hash) | 65 if (sc.get_token_type() == tk_hash) |
| 64 { | 66 { |
| 65 must_get_next_from_scanner (sc, tk_symbol); | 67 must_get_next_from_scanner (sc, tk_symbol); |
| 66 | 68 |
| 91 devf ("Lexer: added %1 (%2)\n", describe_token_type (tok.type), | 93 devf ("Lexer: added %1 (%2)\n", describe_token_type (tok.type), |
| 92 describe_token (&tok)); | 94 describe_token (&tok)); |
| 93 } | 95 } |
| 94 } | 96 } |
| 95 | 97 |
| 96 devf ("Lexer: File %1 processed.\n", file_name); | 98 devf ("Lexer: File %1 processed (%2 tokens).\n", file_name, m_tokens.size()); |
| 97 m_token_position = m_tokens.begin() - 1; | 99 m_token_position = m_tokens.begin() - 1; |
| 98 } | 100 } |
| 99 | 101 |
| 100 // ============================================================================= | 102 // ============================================================================= |
| 101 // | 103 // |
| 102 bool lexer::get_next (e_token req) | 104 bool lexer::get_next (e_token req) |
| 103 { | 105 { |
| 104 iterator pos = m_token_position; | 106 iterator pos = m_token_position; |
| 105 devf ("Lexer: Requested next token, requirement: %1\n", describe_token_type (req)); | 107 devf ("Lexer: Requested next token, requirement: %1\n", describe_token_type (req)); |
| 106 | 108 |
| 109 if (m_tokens.is_empty()) | |
| 110 { | |
| 111 devf ("Lexer: no tokens! Failed.\n"); | |
| 112 return false; | |
| 113 } | |
| 114 | |
| 107 if (is_at_end()) | 115 if (is_at_end()) |
| 108 { | 116 { |
| 109 devf ("Lexer: at end of tokens. Failed.\n"); | 117 devf ("Lexer: at end of tokens. Failed.\n"); |
| 110 return false; | 118 return false; |
| 111 } | 119 } |
| 112 | 120 |
| 113 m_token_position++; | 121 m_token_position++; |
| 114 | 122 |
| 115 if (req != tk_any && get_token() != req) | 123 if (req != tk_any && get_token_type() != req) |
| 116 { | 124 { |
| 117 devf ("Lexer: Token %1 does not meet the requirement\n", describe_token (get_token())); | 125 devf ("Lexer: Token %1 does not meet the requirement\n", describe_token (get_token())); |
| 118 m_token_position = pos; | 126 m_token_position = pos; |
| 119 return false; | 127 return false; |
| 120 } | 128 } |
| 154 | 162 |
| 155 if (!get_next()) | 163 if (!get_next()) |
| 156 error ("unexpected EOF"); | 164 error ("unexpected EOF"); |
| 157 | 165 |
| 158 for (e_token tok : toks) | 166 for (e_token tok : toks) |
| 159 if (get_token() == tok) | 167 if (get_token_type() == tok) |
| 160 return; | 168 return; |
| 161 | 169 |
| 162 string toknames; | 170 string toknames; |
| 163 | 171 |
| 164 for (const e_token& tok_type : toks) | 172 for (const e_token& tok_type : toks) |
| 179 int lexer::get_one_symbol (const string_list& syms) | 187 int lexer::get_one_symbol (const string_list& syms) |
| 180 { | 188 { |
| 181 if (!get_next()) | 189 if (!get_next()) |
| 182 error ("unexpected EOF"); | 190 error ("unexpected EOF"); |
| 183 | 191 |
| 184 if (get_token() == tk_symbol) | 192 if (get_token_type() == tk_symbol) |
| 185 { | 193 { |
| 186 for (int i = 0; i < syms.size(); ++i) | 194 for (int i = 0; i < syms.size(); ++i) |
| 187 { | 195 { |
| 188 if (syms[i] == get_token()->text) | 196 if (syms[i] == get_token()->text) |
| 189 return i; | 197 return i; |
| 196 | 204 |
| 197 // ============================================================================= | 205 // ============================================================================= |
| 198 // | 206 // |
| 199 void lexer::must_be (e_token tok) | 207 void lexer::must_be (e_token tok) |
| 200 { | 208 { |
| 201 if (get_token() != tok) | 209 print ("pos: %1", m_token_position - m_tokens.begin()); |
| 210 if (get_token_type() != tok) | |
| 202 error ("expected %1, got %2", describe_token_type (tok), | 211 error ("expected %1, got %2", describe_token_type (tok), |
| 203 describe_token (get_token())); | 212 describe_token (get_token())); |
| 204 } | 213 } |
| 205 | 214 |
| 206 // ============================================================================= | 215 // ============================================================================= |
| 261 | 270 |
| 262 iterator oldpos = m_token_position; | 271 iterator oldpos = m_token_position; |
| 263 m_token_position += a; | 272 m_token_position += a; |
| 264 string result = get_token()->text; | 273 string result = get_token()->text; |
| 265 m_token_position = oldpos; | 274 m_token_position = oldpos; |
| 266 } | 275 return result; |
| 276 } |
