The Battle for Wesnoth  1.17.12+dev
tokenizer.hpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 - 2022
3  by David White <dave@whitevine.net>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 #pragma once
17 
18 #include <string>
19 
20 namespace wfl
21 {
22 namespace tokenizer
23 {
24 
25 typedef std::string::const_iterator iterator;
26 
27 enum class token_type {
28  operator_token, // Cannot simply be named 'operator' since that's a reserved C++ keyword
30  identifier,
31  integer,
32  decimal,
33  lparens,
34  rparens,
35  lsquare,
36  rsquare,
37  comma,
38  semicolon,
39  whitespace,
40  eol,
41  keyword,
42  comment,
43  pointer
44 };
45 
46 struct token {
47 
48  token() :
50  begin(),
51  end(),
52  line_number(1),
53  filename()
54  {
55  }
56 
57  token(iterator& i1, iterator i2, token_type type) :
58  type(type),
59  begin(i1),
60  end(i2),
61  line_number(1),
62  filename()
63  {
64  }
65 
67  iterator begin, end;
69  const std::string* filename;
70 };
71 
72 token get_token(iterator& i1, iterator i2);
73 
75 {
76  token_error() : description_(), formula_() {}
77  token_error(const std::string& dsc, const std::string& formula) : description_(dsc), formula_(formula) {}
78  std::string description_;
79  std::string formula_;
80 };
81 
82 }
83 
84 }
Abstract baseclass for the tokenizer.
Definition: tokenizer.hpp:56
token(iterator &i1, iterator i2, token_type type)
Definition: tokenizer.hpp:57
const std::string * filename
Definition: tokenizer.hpp:69
token get_token(iterator &i1, const iterator i2)
Definition: tokenizer.cpp:44
Definition: contexts.hpp:44
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
token_error(const std::string &dsc, const std::string &formula)
Definition: tokenizer.hpp:77