The Battle for Wesnoth  1.19.0+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2005 - 2024
3  by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Copyright (C) 2003 by David White <dave@whitevine.net>
5  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
6 
7  This program is free software; you can redistribute it and/or modify
8  it under the terms of the GNU General Public License as published by
9  the Free Software Foundation; either version 2 of the License, or
10  (at your option) any later version.
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY.
13 
14  See the COPYING file for more details.
15 */
16 
17 /**
18  * @file
19  * WML preprocessor.
20  */
21 
23 
24 #include "buffered_istream.hpp"
25 #include "config.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static const std::string current_file_str = "CURRENT_FILE";
44 static const std::string current_dir_str = "CURRENT_DIRECTORY";
45 static const std::string left_curly_str = "LEFT_BRACE";
46 static const std::string right_curly_str = "RIGHT_BRACE";
47 
48 // map associating each filename encountered to a number
49 static std::map<std::string, int> file_number_map;
50 
51 static bool encode_filename = true;
52 
53 static std::string preprocessor_error_detail_prefix = "\n ";
54 
55 static const char OUTPUT_SEPARATOR = '\xFE';
56 
57 // get filename associated to this code
58 static std::string get_filename(const std::string& file_code)
59 {
60  if(!encode_filename) {
61  return file_code;
62  }
63 
64  std::stringstream s;
65  s << file_code;
66  int n = 0;
67  s >> std::hex >> n;
68 
69  for(const auto& p : file_number_map) {
70  if(p.second == n) {
71  return p.first;
72  }
73  }
74 
75  return "<unknown>";
76 }
77 
78 // Get code associated to this filename
79 static std::string get_file_code(const std::string& filename)
80 {
81  if(!encode_filename) {
82  return filename;
83  }
84 
85  // Current number of encountered filenames
86  static int current_file_number = 0;
87 
88  int& fnum = file_number_map[utils::escape(filename, " \\")];
89  if(fnum == 0) {
90  fnum = ++current_file_number;
91  }
92 
93  std::ostringstream shex;
94  shex << std::hex << fnum;
95 
96  return shex.str();
97 }
98 
99 // decode the filenames placed in a location
100 static std::string get_location(const std::string& loc)
101 {
102  std::string res;
103  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
104 
105  if(pos.empty()) {
106  return res;
107  }
108 
109  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
110  while(true) {
111  res += get_filename(*(i++));
112 
113  if(i == end) {
114  break;
115  }
116 
117  res += ' ';
118  res += *(i++);
119 
120  if(i == end) {
121  break;
122  }
123 
124  res += ' ';
125  }
126 
127  return res;
128 }
129 
130 
131 // ==================================================================================
132 // PREPROC_DEFINE IMPLEMENTATION
133 // ==================================================================================
134 
136 {
137  return value == v.value && arguments == v.arguments;
138 }
139 
141 {
142  if(location < v.location) {
143  return true;
144  }
145 
146  if(v.location < location) {
147  return false;
148  }
149 
150  return linenum < v.linenum;
151 }
152 
153 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
154 {
155  const std::string key = "argument";
156 
157  writer.open_child(key);
158 
159  writer.write_key_val("name", arg);
160  writer.close_child(key);
161 }
162 
163 void preproc_define::write_argument(config_writer& writer, const std::string& arg, const std::string& default_value) const
164 {
165  const std::string key = "argument";
166 
167  writer.open_child(key);
168 
169  writer.write_key_val("name", arg);
170  writer.write_key_val("default", default_value);
171  writer.close_child(key);
172 }
173 
174 void preproc_define::write(config_writer& writer, const std::string& name) const
175 {
176  const std::string key = "preproc_define";
177  writer.open_child(key);
178 
179  writer.write_key_val("name", name);
180  writer.write_key_val("value", value);
181  writer.write_key_val("textdomain", textdomain);
182  writer.write_key_val("linenum", std::to_string(linenum));
183  writer.write_key_val("location", get_location(location));
184 
185  if(is_deprecated()) {
186  writer.open_child("deprecated");
187  writer.write_key_val("level", int(*deprecation_level));
188  writer.write_key_val("version", deprecation_version.str());
189  writer.write_key_val("message", deprecation_message);
190  writer.close_child("deprecated");
191  }
192 
193  for(const std::string& arg : arguments) {
194  write_argument(writer, arg);
195  }
196 
197  for(const auto& [key, default_value] : optional_arguments) {
198  write_argument(writer, key, default_value);
199  }
200 
201  writer.close_child(key);
202 }
203 
205 {
206  if(cfg.has_attribute("default")) {
207  optional_arguments.emplace(cfg["name"], cfg["default"]);
208  } else {
209  arguments.push_back(cfg["name"]);
210  }
211 }
212 
213 void preproc_define::read(const config& cfg)
214 {
215  value = cfg["value"].str();
216  textdomain = cfg["textdomain"].str();
217  linenum = cfg["linenum"];
218  location = cfg["location"].str();
219 
220  if(auto deprecated = cfg.optional_child("deprecated")) {
221  deprecation_level = DEP_LEVEL(deprecated.value()["level"].to_int());
222  deprecation_version = deprecated.value()["version"].str();
223  deprecation_message = deprecated.value()["message"].str();
224  }
225 
226  for(const config& arg : cfg.child_range("argument")) {
227  read_argument(arg);
228  }
229 }
230 
231 preproc_map::value_type preproc_define::read_pair(const config& cfg)
232 {
233  preproc_define second;
234  second.read(cfg);
235 
236  return preproc_map::value_type(cfg["name"], second);
237 }
238 
239 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
240 {
241  return stream << "value: " << def.value << " arguments: " << def.location;
242 }
243 
244 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
245 {
246  return stream << def.second;
247 }
248 
249 // ==================================================================================
250 // PREPROCESSOR BASE
251 // ==================================================================================
252 
254 
255 /**
256  * Base class for preprocessing an input.
257  */
259 {
261 
262 protected:
263  /**
264  * Sets up a new preprocessor for stream buffer \a t.
265  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
266  *
267  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
268  */
270 
272 
273 public:
274  virtual ~preprocessor()
275  {
276  }
277 
278  /** Allows specifying any actions that need to be called after the constructor completes. */
279  virtual void init()
280  {
281  }
282 
283  /**
284  * Preprocesses and sends some text to the #parent_ buffer.
285  * @return false when the input has no data left.
286  */
287  virtual bool get_chunk() = 0;
288 
290 
291  /** Returns the appropriate parsing mode for this preprocessor. */
292  virtual MODE parse_mode()
293  {
294  return NO_PARSING;
295  }
296 
297 private:
298  std::string old_textdomain_;
299  std::string old_location_;
300 
302 };
303 
304 
305 // ==================================================================================
306 // PREPROCESSOR BUFFER
307 // ==================================================================================
308 
309 /**
310  * Target for sending preprocessed output.
311  * Objects of this class can be plugged into an STL stream.
312  */
313 class preprocessor_streambuf : public std::streambuf
314 {
315 public:
317  : std::streambuf()
318  , out_buffer_("")
319  , buffer_()
321  , defines_(def)
322  , default_defines_()
324  , location_("")
325  , linenum_(0)
326  , quoted_(false)
327  {
328  }
329 
330  /** Decodes the filenames placed in a location. */
331  std::string get_current_file();
332 
333  void error(const std::string&, int);
334  void warning(const std::string&, int);
335 
336  template<typename T, typename... A>
337  void add_preprocessor(A&&... args)
338  {
339  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
340  preprocessor_queue_.back()->init();
341  }
342 
344  {
345  preprocessor_queue_.pop_back();
346  }
347 
348  int depth() const
349  {
350  return preprocessor_queue_.size();
351  }
352 
354  {
355  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
356  }
357 
358 private:
360  : std::streambuf()
361  , out_buffer_("")
362  , buffer_()
364  , defines_(t.defines_)
365  , default_defines_()
367  , location_("")
368  , linenum_(0)
369  , quoted_(t.quoted_)
370  {
371  }
372 
373  /** Inherited from basic_streambuf. */
374  virtual int underflow() override;
375 
377 
378  /** Buffer read by the STL stream. */
379  std::string out_buffer_;
380 
381  /** Buffer filled by the _current_ preprocessor. */
382  std::stringstream buffer_;
383 
384  /** Input preprocessor queue. */
385  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
386 
389 
390  std::string textdomain_;
391  std::string location_;
392 
393  int linenum_;
394 
395  /**
396  * Set to true if one preprocessor for this target started to read a string.
397  * Deeper-nested preprocessors are then forbidden to.
398  */
399  bool quoted_;
400 
401  friend class preprocessor;
402  friend class preprocessor_file;
403  friend class preprocessor_data;
405 };
406 
407 /** Preprocessor constructor. */
409  : parent_(t)
410  , old_textdomain_(t.textdomain_)
411  , old_location_(t.location_)
412  , old_linenum_(t.linenum_)
413 {
414 }
415 
416 /**
417  * Called by an STL stream whenever it has reached the end of #out_buffer_.
418  * Fills #buffer_ by calling the _current_ preprocessor, then copies its
419  * content into #out_buffer_.
420  * @return the first character of #out_buffer_ if any, EOF otherwise.
421  */
423 {
424  unsigned sz = 0;
425  if(char* gp = gptr()) {
426  if(gp < egptr()) {
427  // Sanity check: the internal buffer has not been totally consumed,
428  // should we force the caller to use what remains first?
429  return *gp;
430  }
431 
432  // The buffer has been completely read; fill it again.
433  // Keep part of the previous buffer, to ensure putback capabilities.
434  sz = out_buffer_.size();
435  buffer_.str(std::string());
436 
437  if(sz > 3) {
438  buffer_ << out_buffer_.substr(sz - 3);
439  sz = 3;
440  } else {
441  buffer_ << out_buffer_;
442  }
443  } else {
444  // The internal get-data pointer is null
445  }
446 
447  const int desired_fill_amount = 2000;
448 
449  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
450  // Process files and data chunks until the desired buffer size is reached
451  if(!current()->get_chunk()) {
452  // Drop the current preprocessor item from the queue.
454  }
455  }
456 
457  // Update the internal state and data pointers
458  out_buffer_ = buffer_.str();
459  if(out_buffer_.empty()) {
460  return EOF;
461  }
462 
463  char* begin = &*out_buffer_.begin();
464  unsigned bs = out_buffer_.size();
465 
466  setg(begin, begin + sz, begin + bs);
467 
468  if(sz >= bs) {
469  return EOF;
470  }
471 
472  return static_cast<unsigned char>(*(begin + sz));
473 }
474 
475 /**
476 * Restores the old preprocessing context.
477 * Appends location and domain directives to the buffer, so that the parser
478 * notices these changes.
479 */
481 {
482  preprocessor* current = this->current();
483 
484  if(!current->old_location_.empty()) {
485  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
486  }
487 
489  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
490  }
491 
495 
496  // Drop the preprocessor from the queue.
498 }
499 
501 {
502  unsigned nested_level = 0;
503 
504  preprocessor* pre = nullptr;
505 
506  // Iterate backwards over queue to get the last non-macro preprocessor.
507  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
508  pre = p->get();
509 
510  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
511  break;
512  }
513 
514  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
515  ++nested_level;
516  }
517  }
518 
519  std::string res;
520  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
521 
522  if(pos.size() <= 2 * nested_level) {
523  return res;
524  }
525 
526  return get_filename(pos[2 * nested_level]);
527 }
528 
529 std::string lineno_string(const std::string& lineno)
530 {
531  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
532  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
533  std::string included_from = preprocessor_error_detail_prefix + "included from ";
534  std::string res;
535 
536  while(i != end) {
537  const std::string& line = *(i++);
538 
539  if(!res.empty()) {
540  res += included_from;
541  }
542 
543  if(i != end) {
544  res += get_filename(*(i++));
545  } else {
546  res += "<unknown>";
547  }
548 
549  res += ':' + line;
550  }
551 
552  if(res.empty()) {
553  res = "???";
554  }
555 
556  return res;
557 }
558 
559 void preprocessor_streambuf::error(const std::string& error_type, int l)
560 {
561  std::string position, error;
562  std::ostringstream pos;
563 
564  pos << l << ' ' << location_;
565  position = lineno_string(pos.str());
566 
567  error = error_type + '\n';
568  error += "at " + position;
569 
570  ERR_PREPROC << error;
571 
573 }
574 
575 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
576 {
577  std::string position, warning;
578  std::ostringstream pos;
579 
580  pos << l << ' ' << location_;
581  position = lineno_string(pos.str());
582 
583  warning = warning_type + '\n';
584  warning += "at " + position;
585 
586  WRN_PREPROC << warning;
587 }
588 
589 
590 // ==================================================================================
591 // PREPROCESSOR FILE
592 // ==================================================================================
593 
594 /**
595  * Specialized preprocessor for handling a file or a set of files.
596  * A preprocessor_file object is created when a preprocessor encounters an
597  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
598  */
600 {
601 public:
602  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
603  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
604 
605  virtual void init() override;
606 
607  /**
608  * Inserts and processes the next file in the list of included files.
609  * @return false if there is no next file.
610  */
611  virtual bool get_chunk() override
612  {
613  while(pos_ != end_) {
614  const std::string& name = *(pos_++);
615  unsigned sz = name.size();
616 
617  // Use reverse iterator to optimize testing
618  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
619  continue;
620  }
621 
623  return true;
624  }
625 
626  return false;
627  }
628 
629 private:
630  std::vector<std::string> files_;
631  std::vector<std::string>::const_iterator pos_, end_;
632 
633  const std::string& name_;
634 
636 };
637 
638 
639 // ==================================================================================
640 // PREPROCESSOR DATA
641 // ==================================================================================
642 
643 /**
644  * Specialized preprocessor for handling any kind of input stream.
645  * This is the core of the preprocessor.
646  */
648 {
649  /** Description of a preprocessing chunk. */
650  struct token_desc
651  {
652  enum class token_type {
653  start, // Toplevel
654  process_if, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
655  process_else, // Processing the "else" branch of a ifdef/ifndef
656  skip_if, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
657  skip_else, // Skipping the "else" branch of a ifdef/ifndef
658  string, // Processing a string
659  verbatim, // Processing a verbatim string
660  macro_space, // Processing between chunks of a macro call (skip spaces)
661  macro_chunk, // Processing inside a chunk of a macro call (stop on space or '(')
662  macro_parens // Processing a parenthesized macro argument
663  };
664 
665  token_desc(token_type type, const int stack_pos, const int linenum)
666  : type(type)
668  , linenum(linenum)
669  {
670  }
671 
673 
674  /** Starting position in #strings_ of the delayed text for this chunk. */
676  int linenum;
677  };
678 
679  /**
680  * Manages the lifetime of the @c std::istream pointer we own.
681  *
682  * Since @ref in_ uses the stream as well this object must be created
683  * before @ref in_ and destroyed after @ref in_ is destroyed.
684  */
686 
687  /** Input stream. */
689 
690  std::string directory_;
691 
692  /** Buffer for delayed input processing. */
693  std::vector<std::string> strings_;
694 
695  /** Mapping of macro arguments to their content. */
696  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
697 
698  /** Stack of nested preprocessing chunks. */
699  std::vector<token_desc> tokens_;
700 
701  /**
702  * Set to true whenever input tokens cannot be directly sent to the target
703  * buffer. For instance, this happens with macro arguments. In that case,
704  * the output is redirected toward #strings_ until it can be processed.
705  */
707 
708  /**
709  * Non-zero when the preprocessor has to skip some input text.
710  * Increased whenever entering a conditional branch that is not useful,
711  * e.g. a ifdef that evaluates to false.
712  */
714  int linenum_;
715 
716  /** True iff we are currently parsing a macros content, otherwise false. */
718 
719  std::string read_word();
720  std::string read_line();
721  std::string read_rest_of_line();
722 
723  void skip_spaces();
724  void skip_eol();
726  void pop_token();
727  void put(char);
728  void put(const std::string& /*, int change_line = 0 */);
729  void conditional_skip(bool skip);
730 
731 public:
734  const std::string& history,
735  const std::string& name,
736  int line,
737  const std::string& dir,
738  const std::string& domain,
739  std::unique_ptr<std::map<std::string, std::string>> defines,
740  bool is_define = false);
741 
742  virtual bool get_chunk() override;
743 
744  virtual preprocessor::MODE parse_mode() override
745  {
747  }
748 
753 };
754 
756 {
757  throw std::logic_error("don't compare tokens with characters");
758 }
759 
761 {
762  return rhs == lhs;
763 }
764 
766 {
767  return !(lhs == rhs);
768 }
769 
771 {
772  return rhs != lhs;
773 }
774 
775 /** preprocessor_file constructor. */
776 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
777  : preprocessor(t)
778  , files_()
779  , pos_()
780  , end_()
781  , name_(name)
782  , is_directory_(filesystem::is_directory(name))
783 {
784  if(is_directory_) {
785  filesystem::get_files_in_dir(name, &files_, nullptr,
789  );
790 
791  for(const std::string& fname : files_) {
792  std::size_t cpos = fname.rfind(" ");
793 
794  if(cpos != std::string::npos && cpos >= symbol_index) {
795  std::stringstream ss;
796  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
797  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
798  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
799 
800  // TODO: find a real linenumber
801  parent_.error(ss.str(), -1);
802  }
803  }
804  } else {
805  // Handled in the init() function.
806  }
807 
808  pos_ = files_.begin();
809  end_ = files_.end();
810 }
811 
813 {
814  if(is_directory_) {
815  return;
816  }
817 
819 
820  if(!file_stream->good()) {
821  ERR_PREPROC << "Could not open file " << name_;
822  } else {
823  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
826  );
827  }
828 }
829 
832  const std::string& history,
833  const std::string& name,
834  int linenum,
835  const std::string& directory,
836  const std::string& domain,
837  std::unique_ptr<std::map<std::string, std::string>> defines,
838  bool is_define)
839  : preprocessor(t)
840  , in_scope_(std::move(i))
841  , in_(*in_scope_)
842  , directory_(directory)
843  , strings_()
844  , local_defines_(std::move(defines))
845  , tokens_()
846  , slowpath_(0)
847  , skipping_(0)
848  , linenum_(linenum)
849  , is_define_(is_define)
850 {
851  std::ostringstream s;
852  s << history;
853 
854  if(!name.empty()) {
855  if(!history.empty()) {
856  s << ' ';
857  }
858 
859  s << get_file_code(name);
860  }
861 
862  if(!t.location_.empty()) {
863  s << ' ' << t.linenum_ << ' ' << t.location_;
864  }
865 
866  t.location_ = s.str();
867  t.linenum_ = linenum;
868 
869  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
870 
871  if(t.textdomain_ != domain) {
872  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
873  t.textdomain_ = domain;
874  }
875 
877 }
878 
880 {
881  tokens_.emplace_back(t, strings_.size(), linenum_);
882 
884  // Macro expansions do not have any associated storage at start.
885  return;
887  /* Quoted strings are always inlined in the parent token. So
888  * they need neither storage nor metadata, unless the parent
889  * token is a macro expansion.
890  */
891  token_desc::token_type& outer_type = tokens_[tokens_.size() - 2].type;
892  if(outer_type != token_desc::token_type::macro_space) {
893  return;
894  }
895 
897  tokens_.back().stack_pos = strings_.size() + 1;
898  }
899 
900  std::ostringstream s;
901  if(!skipping_ && slowpath_) {
902  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
903  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
904  }
905 
906  strings_.push_back(s.str());
907 }
908 
910 {
911  token_desc::token_type inner_type = tokens_.back().type;
912  unsigned stack_pos = tokens_.back().stack_pos;
913 
914  tokens_.pop_back();
915 
916  token_desc::token_type& outer_type = tokens_.back().type;
917 
918  if(inner_type == token_desc::token_type::macro_parens) {
919  // Parenthesized macro arguments are left on the stack.
920  assert(outer_type == token_desc::token_type::macro_space);
921  return;
922  }
923 
924  if(inner_type == token_desc::token_type::string || inner_type == token_desc::token_type::verbatim) {
925  // Quoted strings are always inlined.
926  assert(stack_pos == strings_.size());
927  return;
928  }
929 
930  if(outer_type == token_desc::token_type::macro_space) {
931  /* A macro expansion does not have any associated storage.
932  * Instead, storage of the inner token is not discarded
933  * but kept as a new macro argument. But if the inner token
934  * was a macro expansion, it is about to be appended, so
935  * prepare for it.
936  */
938  strings_.erase(strings_.begin() + stack_pos, strings_.end());
939  strings_.emplace_back();
940  }
941 
942  assert(stack_pos + 1 == strings_.size());
944 
945  return;
946  }
947 
948  strings_.erase(strings_.begin() + stack_pos, strings_.end());
949 }
950 
952 {
953  while(true) {
954  int c = in_.peek();
955 
956  if(in_.eof() || (c != ' ' && c != '\t')) {
957  return;
958  }
959 
960  in_.get();
961  }
962 }
963 
965 {
966  while(true) {
967  int c = in_.get();
968 
969  if(c == '\n') {
970  ++linenum_;
971  return;
972  }
973 
974  if(in_.eof()) {
975  return;
976  }
977  }
978 }
979 
981 {
982  std::string res;
983 
984  while(true) {
985  int c = in_.peek();
986 
987  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
988  // DBG_PREPROC << "(" << res << ")";
989  return res;
990  }
991 
992  in_.get();
993  res += static_cast<char>(c);
994  }
995 }
996 
998 {
999  std::string res;
1000 
1001  while(true) {
1002  int c = in_.get();
1003 
1004  if(c == '\n') {
1005  ++linenum_;
1006  return res;
1007  }
1008 
1009  if(in_.eof()) {
1010  return res;
1011  }
1012 
1013  if(c != '\r') {
1014  res += static_cast<char>(c);
1015  }
1016  }
1017 }
1018 
1020 {
1021  std::string res;
1022 
1023  while(in_.peek() != '\n' && !in_.eof()) {
1024  int c = in_.get();
1025 
1026  if(c != '\r') {
1027  res += static_cast<char>(c);
1028  }
1029  }
1030 
1031  return res;
1032 }
1033 
1035 {
1036  if(skipping_) {
1037  return;
1038  }
1039 
1040  if(slowpath_) {
1041  strings_.back() += c;
1042  return;
1043  }
1044 
1045  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1046 
1047  if(unsigned diff = cond_linenum - parent_.linenum_) {
1048  parent_.linenum_ = cond_linenum;
1049 
1050  if(diff <= parent_.location_.size() + 11) {
1051  parent_.buffer_ << std::string(diff, '\n');
1052  } else {
1053  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1054  }
1055  }
1056 
1057  if(c == '\n') {
1058  ++parent_.linenum_;
1059  }
1060 
1061  parent_.buffer_ << c;
1062 }
1063 
1064 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1065 {
1066  if(skipping_) {
1067  return;
1068  }
1069 
1070  if(slowpath_) {
1071  strings_.back() += s;
1072  return;
1073  }
1074 
1075  parent_.buffer_ << s;
1076  // parent_.linenum_ += line_change;
1077 }
1078 
1080 {
1081  if(skip) {
1082  ++skipping_;
1083  }
1084 
1086 }
1087 
1089 {
1090  char c = static_cast<char>(in_.get());
1091  token_desc& token = tokens_.back();
1092 
1093  if(in_.eof()) {
1094  // The end of file was reached.
1095  // Make sure we don't have any incomplete tokens.
1096  char const* s;
1097 
1098  switch(token.type) {
1100  return false; // everything is fine
1105  s = "#ifdef or #ifndef";
1106  break;
1108  s = "Quoted string";
1109  break;
1111  s = "Verbatim string";
1112  break;
1115  s = "Macro substitution";
1116  break;
1118  s = "Macro argument";
1119  break;
1120  default:
1121  s = "???";
1122  }
1123 
1124  parent_.error(std::string(s) + " not terminated", token.linenum);
1125  }
1126 
1127  if(c == '\n') {
1128  ++linenum_;
1129  }
1130 
1131  if(c == OUTPUT_SEPARATOR) {
1132  std::string buffer(1, c);
1133 
1134  while(true) {
1135  char d = static_cast<char>(in_.get());
1136 
1137  if(in_.eof() || d == '\n') {
1138  break;
1139  }
1140 
1141  buffer += d;
1142  }
1143 
1144  buffer += '\n';
1145  // line_change = 1-1 = 0
1146  put(buffer);
1148  put(c);
1149 
1150  if(c == '>' && in_.peek() == '>') {
1151  put(in_.get());
1152  pop_token();
1153  }
1154  } else if(c == '<' && in_.peek() == '<') {
1155  in_.get();
1157  put('<');
1158  put('<');
1159  } else if(c == '"') {
1161  parent_.quoted_ = false;
1162  put(c);
1163  pop_token();
1164  } else if(!parent_.quoted_) {
1165  parent_.quoted_ = true;
1167  put(c);
1168  } else {
1169  parent_.error("Nested quoted string", linenum_);
1170  }
1171  } else if(c == '{') {
1173  ++slowpath_;
1174  } else if(c == ')' && token.type == token_desc::token_type::macro_parens) {
1175  pop_token();
1176  } else if(c == '#' && !parent_.quoted_) {
1177  std::string command = read_word();
1178  bool comment = false;
1179 
1180  if(command == "define") {
1181  skip_spaces();
1182  int linenum = linenum_;
1183  std::vector<std::string> items = utils::split(read_line(), ' ');
1184  std::map<std::string, std::string> optargs;
1185 
1186  if(items.empty()) {
1187  parent_.error("No macro name found after #define directive", linenum);
1188  }
1189 
1190  std::string symbol = items.front();
1191  items.erase(items.begin());
1192  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1193  std::optional<DEP_LEVEL> deprecation_level;
1194  std::string buffer, deprecation_detail;
1195  version_info deprecation_version = game_config::wesnoth_version;
1196  while(true) {
1197  if(in_.eof())
1198  break;
1199  char d = static_cast<char>(in_.get());
1200  if(d == '\n')
1201  ++linenum_;
1202  buffer += d;
1203  if(d == '#') {
1204  if(in_.peek() == 'a') {
1205  found_arg = 1;
1206  } else if(in_.peek() == 'd') {
1207  found_deprecate = 1;
1208  } else {
1209  found_enddef = 1;
1210  }
1211  } else {
1212  if(found_arg > 0 && ++found_arg == 4) {
1213  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1214  buffer.erase(buffer.end() - 4, buffer.end());
1215 
1216  skip_spaces();
1217  std::string argname = read_word();
1218  skip_eol();
1219 
1220  std::string argbuffer;
1221 
1222  int found_endarg = 0;
1223  while(true) {
1224  if(in_.eof()) {
1225  break;
1226  }
1227 
1228  char e = static_cast<char>(in_.get());
1229  if(e == '\n') {
1230  ++linenum_;
1231  }
1232 
1233  argbuffer += e;
1234 
1235  if(e == '#') {
1236  found_endarg = 1;
1237  } else if(found_endarg > 0 && ++found_endarg == 7) {
1238  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1239  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1240  optargs[argname] = argbuffer;
1241  skip_eol();
1242  break;
1243  } else {
1244  parent_.error("Unterminated #arg definition", linenum_);
1245  }
1246  }
1247  }
1248  }
1249  }
1250 
1251  if(found_deprecate > 0 && ++found_deprecate == 11) {
1252  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1253  buffer.erase(buffer.end() - 11, buffer.end());
1254  skip_spaces();
1255  try {
1256  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1257  if(deprecation_level) {
1258  deprecation_level = std::max(*deprecation_level, level);
1259  } else {
1260  deprecation_level = level;
1261  }
1262  } catch(const std::invalid_argument&) {
1263  // Meh, fall back to default of PREEMPTIVE...
1264  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1265  }
1266  deprecation_version = game_config::wesnoth_version;
1267  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1268  skip_spaces();
1269  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1270  }
1271  skip_spaces();
1272  if(!deprecation_detail.empty()){
1273  deprecation_detail += '\n';
1274  }
1275  deprecation_detail += read_rest_of_line();
1276  skip_eol();
1277  }
1278  }
1279 
1280  if(found_enddef > 0 && ++found_enddef == 7) {
1281  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1282  break;
1283  } else {
1284  found_enddef = 0;
1285  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1286  // this? This would fill feature
1287  // request #21343
1288  parent_.error(
1289  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1290  linenum);
1291  }
1292  }
1293  }
1294  }
1295  }
1296 
1297  if(found_enddef != 7) {
1298  parent_.error("Unterminated preprocessor definition", linenum_);
1299  }
1300 
1301  if(!skipping_) {
1302  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1303  if(old_i != parent_.defines_->end()) {
1304  std::ostringstream new_pos, old_pos;
1305  const preproc_define& old_d = old_i->second;
1306 
1307  new_pos << linenum << ' ' << parent_.location_;
1308  old_pos << old_d.linenum << ' ' << old_d.location;
1309 
1310  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1311  << lineno_string(new_pos.str()) << '\n'
1312  << "previously defined at " << lineno_string(old_pos.str());
1313  }
1314 
1315  buffer.erase(buffer.end() - 7, buffer.end());
1316  (*parent_.defines_)[symbol]
1317  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1318  deprecation_detail, deprecation_level, deprecation_version);
1319 
1320  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1321  }
1322  } else if(command == "ifdef" || command == "ifndef") {
1323  const bool negate = command[2] == 'n';
1324  skip_spaces();
1325  const std::string& symbol = read_word();
1326  if(symbol.empty()) {
1327  parent_.error("No macro argument found after #ifdef/#ifndef directive", linenum_);
1328  }
1329  bool found = parent_.defines_->count(symbol) != 0;
1330  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined");
1331  conditional_skip(negate ? found : !found);
1332  } else if(command == "ifhave" || command == "ifnhave") {
1333  const bool negate = command[2] == 'n';
1334  skip_spaces();
1335  const std::string& symbol = read_word();
1336  if(symbol.empty()) {
1337  parent_.error("No path argument found after #ifhave/#ifnhave directive", linenum_);
1338  }
1339  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1340  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found");
1341  conditional_skip(negate ? found : !found);
1342  } else if(command == "ifver" || command == "ifnver") {
1343  const bool negate = command[2] == 'n';
1344 
1345  skip_spaces();
1346  const std::string& vsymstr = read_word();
1347  skip_spaces();
1348  const std::string& vopstr = read_word();
1349  skip_spaces();
1350  const std::string& vverstr = read_word();
1351 
1352  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1353 
1354  if(vop == OP_INVALID) {
1355  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1356  } else if(parent_.defines_->count(vsymstr) != 0) {
1357  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1358 
1359  if(!sym.arguments.empty()) {
1360  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1361  }
1362 
1363  version_info const version1(sym.value);
1364  version_info const version2(vverstr);
1365 
1366  const bool found = do_version_check(version1, vop, version2);
1367  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1368  << vopstr << "): " << (found ? "match" : "no match");
1369 
1370  conditional_skip(negate ? found : !found);
1371  } else {
1372  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1373  err += vsymstr;
1374  err += "'";
1376  }
1377  } else if(command == "else") {
1379  pop_token();
1380  --skipping_;
1383  pop_token();
1384  ++skipping_;
1386  } else {
1387  parent_.error("Unexpected #else", linenum_);
1388  }
1389  } else if(command == "endif") {
1390  switch(token.type) {
1393  --skipping_;
1396  break;
1397  default:
1398  parent_.error("Unexpected #endif", linenum_);
1399  }
1400  pop_token();
1401  } else if(command == "textdomain") {
1402  skip_spaces();
1403  const std::string& s = read_word();
1404  if(s != parent_.textdomain_) {
1405  put("#textdomain ");
1406  put(s);
1407  parent_.textdomain_ = s;
1408  }
1409  comment = true;
1410  } else if(command == "enddef") {
1411  parent_.error("Unexpected #enddef", linenum_);
1412  } else if(command == "undef") {
1413  skip_spaces();
1414  const std::string& symbol = read_word();
1415  if(!skipping_) {
1416  parent_.defines_->erase(symbol);
1417  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1418  }
1419  } else if(command == "error") {
1420  if(!skipping_) {
1421  skip_spaces();
1422  std::ostringstream error;
1423  error << "#error: \"" << read_rest_of_line() << '"';
1424  parent_.error(error.str(), linenum_);
1425  } else
1426  DBG_PREPROC << "Skipped an error";
1427  } else if(command == "warning") {
1428  if(!skipping_) {
1429  skip_spaces();
1430  std::ostringstream warning;
1431  warning << "#warning: \"" << read_rest_of_line() << '"';
1432  parent_.warning(warning.str(), linenum_);
1433  } else {
1434  DBG_PREPROC << "Skipped a warning";
1435  }
1436  } else if(command == "deprecated") {
1437  // The current file is deprecated, so print a message
1438  skip_spaces();
1440  try {
1441  level = DEP_LEVEL(std::stoi(read_word()));
1442  } catch(const std::invalid_argument&) {
1443  // Meh, just fall back to the default of PREEMPTIVE...
1444  }
1447  skip_spaces();
1448  version = version_info(read_word());
1449  }
1450  skip_spaces();
1451  std::string detail = read_rest_of_line();
1453  } else {
1455  }
1456 
1457  skip_eol();
1458  if(comment) {
1459  put('\n');
1460  }
1462  if(c == '(') {
1463  // If a macro argument was started, it is implicitly ended.
1466  } else if(utils::portable_isspace(c)) {
1467  // If a macro argument was started, it is implicitly ended.
1469  } else if(c == '}') {
1470  --slowpath_;
1471  if(skipping_) {
1472  pop_token();
1473  return true;
1474  }
1475 
1476  // FIXME: is this obsolete?
1477  // if (token.type == token_desc::MACRO_SPACE) {
1478  // if (!strings_.back().empty()) {
1479  // std::ostringstream error;
1480  // std::ostringstream location;
1481  // error << "Can't parse new macro parameter with a macro call scope open";
1482  // location<<linenum_<<' '<<parent_.location_;
1483  // parent_.error(error.str(), location.str());
1484  // }
1485  // strings_.pop_back();
1486  //}
1487 
1488  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1489  parent_.error("No macro or file substitution target specified", linenum_);
1490  }
1491 
1492  std::string symbol = strings_[token.stack_pos];
1493  std::string::size_type pos;
1494  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1495  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1496  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1497  }
1498 
1499  std::map<std::string, std::string>::const_iterator arg;
1500  preproc_map::const_iterator macro;
1501 
1502  // If this is a known pre-processing symbol, then we insert it,
1503  // otherwise we assume it's a file name to load.
1504  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1505  pop_token();
1507  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1508  pop_token();
1510  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1511  pop_token();
1512  put("{");
1513  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1514  pop_token();
1515  put("}");
1516  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1517  if(strings_.size() - token.stack_pos != 1) {
1518  std::ostringstream error;
1519  error << "Macro argument '" << symbol << "' does not expect any arguments";
1520  parent_.error(error.str(), linenum_);
1521  }
1522 
1523  std::ostringstream v;
1524  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1525  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1526 
1527  pop_token();
1528  put(v.str());
1529  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1530  const preproc_define& val = macro->second;
1531  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1532  std::size_t optional_arg_num = 0;
1533 
1534  auto defines = std::make_unique<std::map<std::string, std::string>>();
1535  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1536 
1537  if(val.is_deprecated()) {
1539  }
1540 
1541  for(std::size_t i = 0; i < nb_arg; ++i) {
1542  if(i < val.arguments.size()) {
1543  // Normal mandatory arguments
1544 
1545  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1546  } else {
1547  // These should be optional argument overrides
1548 
1549  std::string str = strings_[token.stack_pos + i + 1];
1550  std::size_t equals_pos = str.find_first_of("=");
1551 
1552  if(equals_pos != std::string::npos) {
1553  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1554 
1555  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1556 
1557  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1558  (*defines)[argname] = str.substr(equals_pos + 1);
1559 
1560  optional_arg_num++;
1561 
1562  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol;
1563  } else {
1564  std::ostringstream warning;
1565  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1566  << argname << "'";
1567  parent_.warning(warning.str(), linenum_);
1568 
1569  optional_arg_num++; // To prevent the argument number check from blowing up
1570  }
1571  }
1572  }
1573  }
1574 
1575  // If the macro definition has any optional arguments, insert their defaults
1576  if(val.optional_arguments.size() > 0) {
1577  for(const auto& argument : val.optional_arguments) {
1578  if(defines->find(argument.first) == defines->end()) {
1579  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1580 
1581  buf->textdomain_ = parent_.textdomain_;
1582  std::istream in(buf.get());
1583 
1584  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1585 
1586  auto temp_defines = std::make_unique<std::map<std::string, std::string>>();
1587  temp_defines->insert(defines->begin(), defines->end());
1588 
1589  buf->add_preprocessor<preprocessor_data>(
1590  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1591 
1592  std::ostringstream res;
1593  res << in.rdbuf();
1594 
1595  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1596  << symbol;
1597 
1598  (*defines)[argument.first] = res.str();
1599  }
1600  }
1601  }
1602 
1603  if(nb_arg - optional_arg_num != val.arguments.size()) {
1604  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1605  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1606  std::ostringstream error;
1607  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1608  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1609  << nb_arg - optional_arg_num << " arguments";
1610  parent_.error(error.str(), linenum_);
1611  }
1612 
1613  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1614 
1615  pop_token();
1616 
1617  if(!slowpath_) {
1618  DBG_PREPROC << "substituting macro " << symbol;
1619 
1621  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1622  } else {
1623  DBG_PREPROC << "substituting (slow) macro " << symbol;
1624 
1625  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1626 
1627  // Make the nested preprocessor_data responsible for
1628  // restoring our current textdomain if needed.
1629  buf->textdomain_ = parent_.textdomain_;
1630 
1631  std::ostringstream res;
1632  {
1633  std::istream in(buf.get());
1634  buf->add_preprocessor<preprocessor_data>(
1635  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1636 
1637  res << in.rdbuf();
1638  }
1639 
1640  put(res.str());
1641  }
1642  } else if(parent_.depth() < 40) {
1643  LOG_PREPROC << "Macro definition not found for " << symbol << ", attempting to open as file.";
1644  pop_token();
1645 
1646  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1647  if(!nfname.empty()) {
1648  if(!slowpath_)
1649  // nfname.size() - symbol.size() gives you an index into nfname
1650  // This does not necessarily match the symbol though, as it can start with ~ or ./
1651  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1652  else {
1653  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1654 
1655  std::ostringstream res;
1656  {
1657  std::istream in(buf.get());
1658  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1659 
1660  res << in.rdbuf();
1661  }
1662 
1663  put(res.str());
1664  }
1665  } else {
1666  std::ostringstream error;
1667  error << "Macro/file '" << symbol << "' is missing";
1668  parent_.error(error.str(), linenum_);
1669  }
1670  } else {
1671  parent_.error("Too many nested preprocessing inclusions", linenum_);
1672  }
1673  } else if(!skipping_) {
1675  std::ostringstream s;
1676  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1677  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1678 
1679  strings_.push_back(s.str());
1681  }
1682  put(c);
1683  }
1684  } else {
1685  put(c);
1686  }
1687 
1688  return true;
1689 }
1690 
1691 
1692 // ==================================================================================
1693 // PREPROCESSOR SCOPE HELPER
1694 // ==================================================================================
1695 
1696 struct preprocessor_scope_helper : std::basic_istream<char>
1697 {
1698  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1699  : std::basic_istream<char>(nullptr)
1700  , buf_(nullptr)
1701  , local_defines_(nullptr)
1702  {
1703  //
1704  // If no defines were provided, we create a new local preproc_map and assign
1705  // it to defines temporarily. In this case, the map will be deleted once this
1706  // object is destroyed and defines will still be subsequently null.
1707  //
1708  if(!defines) {
1709  local_defines_.reset(new preproc_map);
1710  defines = local_defines_.get();
1711  }
1712 
1713  buf_.reset(new preprocessor_streambuf(defines));
1714 
1715  // Begin processing.
1716  buf_->add_preprocessor<preprocessor_file>(fname);
1717 
1718  //
1719  // TODO: not sure if this call is needed. Previously, this call was passed a
1720  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1721  // called with its contents. However, at that point the preprocessing should
1722  // already have completed, meaning this call might be redundant. Not sure.
1723  //
1724  // - vultraz, 2017-08-31
1725  //
1726  init(buf_.get());
1727  }
1728 
1730  {
1731  clear(std::ios_base::goodbit);
1732  exceptions(std::ios_base::goodbit);
1733  rdbuf(nullptr);
1734  }
1735 
1736  std::unique_ptr<preprocessor_streambuf> buf_;
1737  std::unique_ptr<preproc_map> local_defines_;
1738 };
1739 
1740 
1741 // ==================================================================================
1742 // FREE-STANDING FUNCTIONS
1743 // ==================================================================================
1744 
1745 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1746 {
1747  log_scope("preprocessing file " + fname + " ...");
1748 
1749  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1750  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1751 }
1752 
1753 void preprocess_resource(const std::string& res_name,
1754  preproc_map* defines_map,
1755  bool write_cfg,
1756  bool write_plain_cfg,
1757  const std::string& parent_directory)
1758 {
1759  if(filesystem::is_directory(res_name)) {
1760  std::vector<std::string> dirs, files;
1761 
1764 
1765  // Subdirectories
1766  for(const std::string& dir : dirs) {
1767  LOG_PREPROC << "processing sub-dir: " << dir;
1768  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1769  }
1770 
1771  // Files in current directory
1772  for(const std::string& file : files) {
1773  if(filesystem::ends_with(file, ".cfg")) {
1774  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1775  }
1776  }
1777 
1778  return;
1779  }
1780 
1781  LOG_PREPROC << "processing resource: " << res_name;
1782 
1783  // disable filename encoding to get clear #line in cfg.plain
1784  encode_filename = false;
1785 
1786  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1787 
1788  std::stringstream ss;
1789 
1790  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1791  // they will be propagated in the main program, instead of just setting the
1792  // failbit on the stream. This was necessary in order for the MSVC and GCC
1793  // binaries to behave the same way.
1794  ss.exceptions(std::ios_base::failbit);
1795 
1796  ss << (*stream).rdbuf();
1797 
1798  LOG_PREPROC << "processing finished";
1799 
1800  if(write_cfg || write_plain_cfg) {
1801  config cfg;
1802  std::string streamContent = ss.str();
1803 
1804  read(cfg, streamContent);
1805 
1806  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1807 
1808  // Write the processed cfg file
1809  if(write_cfg) {
1810  LOG_PREPROC << "writing cfg file: " << preproc_res_name;
1811 
1813  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1814 
1815  write(*outStream, cfg);
1816  }
1817 
1818  // Write the plain cfg file
1819  if(write_plain_cfg) {
1820  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain");
1821 
1823  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1824  }
1825  }
1826 }
double t
Definition: astarsearch.cpp:63
Helper class for buffering a std::istream.
Helper class for buffering a std::istream.
bool eof() const
Is the end of input reached?
int get()
Gets and consumes a character from the buffer.
int peek()
Gets a character from the buffer.
Class for writing a config out to a file in pieces.
void close_child(const std::string &key)
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
void open_child(const std::string &key)
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:159
bool has_attribute(config_key_type key) const
Definition: config.cpp:155
child_itors child_range(config_key_type key)
Definition: config.cpp:273
optional_config_impl< config > optional_child(config_key_type key, int n=0)
Euivalent to mandatory_child, but returns an empty optional if the nth child was not found.
Definition: config.cpp:385
Specialized preprocessor for handling any kind of input stream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
int skipping_
Non-zero when the preprocessor has to skip some input text.
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
void conditional_skip(bool skip)
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
std::string read_rest_of_line()
friend bool operator!=(preprocessor_data::token_desc::token_type, char)
std::vector< std::string > strings_
Buffer for delayed input processing.
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
buffered_istream in_
Input stream.
friend bool operator==(preprocessor_data::token_desc::token_type, char)
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
std::string read_line()
std::string read_word()
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
std::string directory_
void push_token(token_desc::token_type)
Specialized preprocessor for handling a file or a set of files.
std::vector< std::string >::const_iterator pos_
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
const std::string & name_
std::vector< std::string >::const_iterator end_
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
Target for sending preprocessed output.
std::stringstream buffer_
Buffer filled by the current preprocessor.
preprocessor * current() const
bool quoted_
Set to true if one preprocessor for this target started to read a string.
preprocessor_streambuf(preproc_map *def)
preproc_map default_defines_
virtual int underflow() override
Inherited from basic_streambuf.
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string out_buffer_
Buffer read by the STL stream.
std::string get_current_file()
Decodes the filenames placed in a location.
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
void add_preprocessor(A &&... args)
void error(const std::string &, int)
preprocessor_streambuf(const preprocessor_streambuf &t)
void warning(const std::string &, int)
Base class for preprocessing an input.
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
std::string old_textdomain_
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
virtual ~preprocessor()
std::string old_location_
friend class preprocessor_streambuf
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
preprocessor_streambuf & parent_
Represents version numbers.
std::string str() const
Serializes the version number into string form.
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:21
std::size_t i
Definition: function.cpp:968
unsigned in
If equal to search_counter, the node is off the list.
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
VERSION_COMP_OP parse_version_op(const std::string &op_str)
Interfaces for manipulating version numbers of engine, add-ons, etc.
VERSION_COMP_OP
@ OP_INVALID
Standard logging facilities (interface).
#define log_scope(description)
Definition: log.hpp:274
void clear()
Clear the current render target.
Definition: draw.cpp:40
void line(int from_x, int from_y, int to_x, int to_y)
Draw a line.
Definition: draw.cpp:180
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:405
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn't pres...
bool ends_with(const std::string &str, const std::string &suffix)
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:377
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:50
void write_file(const std::string &fname, const std::string &data, std::ios_base::openmode mode)
Throws io_exception if an error occurs.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:51
const version_info wesnoth_version(VERSION)
logger & err()
Definition: log.cpp:302
const std::vector< std::string > items
struct utils::detail::formula_initer init
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
bool portable_isspace(const char c)
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
std::vector< std::string > split(const config_attribute_value &val)
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
#define DBG_PREPROC
static lg::log_domain log_preprocessor("preprocessor")
static const std::string left_curly_str
static const std::string current_dir_str
#define ERR_PREPROC
std::string lineno_string(const std::string &lineno)
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
#define WRN_PREPROC
static std::string get_file_code(const std::string &filename)
static bool encode_filename
static const std::string current_file_str
bool operator==(preprocessor_data::token_desc::token_type, char)
static std::string get_filename(const std::string &file_code)
static std::string get_location(const std::string &loc)
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
#define LOG_PREPROC
static std::map< std::string, int > file_number_map
static const std::string right_curly_str
bool operator!=(preprocessor_data::token_desc::token_type rhs, char lhs)
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static const char OUTPUT_SEPARATOR
static std::string preprocessor_error_detail_prefix
std::map< std::string, struct preproc_define > preproc_map
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:627
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:764
std::optional< DEP_LEVEL > deprecation_level
void write(config_writer &, const std::string &) const
std::string value
version_info deprecation_version
bool operator<(const preproc_define &) const
std::string textdomain
void write_argument(config_writer &, const std::string &) const
void read(const config &)
void read_argument(const config &)
std::string deprecation_message
std::vector< std::string > arguments
std::map< std::string, std::string > optional_arguments
bool operator==(const preproc_define &) const
static preproc_map::value_type read_pair(const config &)
std::string location
bool is_deprecated() const
Description of a preprocessing chunk.
token_desc(token_type type, const int stack_pos, const int linenum)
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
std::unique_ptr< preprocessor_streambuf > buf_
std::unique_ptr< preproc_map > local_defines_
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
token_type type
Definition: tokenizer.hpp:51
mock_char c
mock_party p
static map_location::DIRECTION n
static map_location::DIRECTION s
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
#define PACKAGE
Definition: wesconfig.h:23
#define d
#define e
#define b