61 const token &next_token();
68 #ifdef DEBUG_TOKENIZER 69 const token &previous_token()
const 71 return previous_token_;
106 current_ = in_.get();
107 }
while (current_ ==
'\r');
111 current_ = in_.get();
112 if (current_ ==
'\r')
116 current_ = in_.get();
142 return c < 128 ? char_types_[
c] : 0;
147 return (char_type(c) & TOK_SPACE) == TOK_SPACE;
152 return (char_type(c) & TOK_NUMERIC) == TOK_NUMERIC;
157 return (char_type(c) & (TOK_ALPHA | TOK_NUMERIC)) != TOK_NONE;
166 bool skip_command(
char const *cmd);
171 #ifdef DEBUG_TOKENIZER 172 token previous_token_;
175 char char_types_[128];
const std::string & textdomain() const
bool is_space(int c) const
Abstract baseclass for the tokenizer.
Helper class for buffering a std::istream.
unsigned in
If equal to search_counter, the node is off the list.
int get_start_line() const
bool is_alnum(int c) const
Helper class for buffering a std::istream.
const token & current_token() const
int char_type(unsigned c) const
const std::string & get_file() const