The Battle for Wesnoth  1.19.4+dev
tokenizer.hpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 - 2024
3  by David White <dave@whitevine.net>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 #pragma once
17 
18 #include <string>
19 
20 namespace wfl
21 {
22 namespace tokenizer
23 {
24 
25 typedef std::string::const_iterator iterator;
26 
27 enum class token_type {
28  operator_token, // Cannot simply be named 'operator' since that's a reserved C++ keyword
30  identifier,
31  integer,
32  decimal,
33  lparens,
34  rparens,
35  lsquare,
36  rsquare,
37  comma,
38  semicolon,
39  whitespace,
40  eol,
41  keyword,
42  comment,
43  pointer
44 };
45 
46 struct token {
47 
48  token() :
50  begin(),
51  end(),
52  line_number(1),
53  filename()
54  {
55  }
56 
58  type(type),
59  begin(i1),
60  end(i2),
61  line_number(1),
62  filename()
63  {
64  }
65 
69  const std::string* filename;
70 };
71 
73 
75 {
77  token_error(const std::string& dsc, const std::string& formula) : description_(dsc), formula_(formula) {}
78  std::string description_;
79  std::string formula_;
80 };
81 
82 }
83 
84 }
class responsible for parsing the provided text into tokens and tracking information about the curren...
Definition: tokenizer.hpp:99
token get_token(iterator &i1, const iterator i2)
Definition: tokenizer.cpp:44
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
Definition: contexts.hpp:43
token_error(const std::string &dsc, const std::string &formula)
Definition: tokenizer.hpp:77
const std::string * filename
Definition: tokenizer.hpp:69
token(iterator &i1, iterator i2, token_type type)
Definition: tokenizer.hpp:57