The Battle for Wesnoth  1.17.10+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2005 - 2022
3  by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Copyright (C) 2003 by David White <dave@whitevine.net>
5  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
6 
7  This program is free software; you can redistribute it and/or modify
8  it under the terms of the GNU General Public License as published by
9  the Free Software Foundation; either version 2 of the License, or
10  (at your option) any later version.
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY.
13 
14  See the COPYING file for more details.
15 */
16 
17 /**
18  * @file
19  * WML preprocessor.
20  */
21 
23 
24 #include "buffered_istream.hpp"
25 #include "config.hpp"
26 #include "filesystem.hpp"
27 #include "log.hpp"
29 #include "serialization/parser.hpp"
31 #include "game_version.hpp"
32 #include "wesconfig.h"
33 #include "deprecation.hpp"
34 
35 #include <stdexcept>
36 #include <deque>
37 
38 static lg::log_domain log_preprocessor("preprocessor");
39 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
40 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
41 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
42 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
43 
44 static const std::string current_file_str = "CURRENT_FILE";
45 static const std::string current_dir_str = "CURRENT_DIRECTORY";
46 static const std::string left_curly_str = "LEFT_BRACE";
47 static const std::string right_curly_str = "RIGHT_BRACE";
48 
49 // map associating each filename encountered to a number
50 static std::map<std::string, int> file_number_map;
51 
52 static bool encode_filename = true;
53 
54 static std::string preprocessor_error_detail_prefix = "\n ";
55 
56 static const char OUTPUT_SEPARATOR = '\xFE';
57 
58 // get filename associated to this code
59 static std::string get_filename(const std::string& file_code)
60 {
61  if(!encode_filename) {
62  return file_code;
63  }
64 
65  std::stringstream s;
66  s << file_code;
67  int n = 0;
68  s >> std::hex >> n;
69 
70  for(const auto& p : file_number_map) {
71  if(p.second == n) {
72  return p.first;
73  }
74  }
75 
76  return "<unknown>";
77 }
78 
79 // Get code associated to this filename
80 static std::string get_file_code(const std::string& filename)
81 {
82  if(!encode_filename) {
83  return filename;
84  }
85 
86  // Current number of encountered filenames
87  static int current_file_number = 0;
88 
89  int& fnum = file_number_map[utils::escape(filename, " \\")];
90  if(fnum == 0) {
91  fnum = ++current_file_number;
92  }
93 
94  std::ostringstream shex;
95  shex << std::hex << fnum;
96 
97  return shex.str();
98 }
99 
100 // decode the filenames placed in a location
101 static std::string get_location(const std::string& loc)
102 {
103  std::string res;
104  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
105 
106  if(pos.empty()) {
107  return res;
108  }
109 
110  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
111  while(true) {
112  res += get_filename(*(i++));
113 
114  if(i == end) {
115  break;
116  }
117 
118  res += ' ';
119  res += *(i++);
120 
121  if(i == end) {
122  break;
123  }
124 
125  res += ' ';
126  }
127 
128  return res;
129 }
130 
131 
132 // ==================================================================================
133 // PREPROC_DEFINE IMPLEMENTATION
134 // ==================================================================================
135 
137 {
138  return value == v.value && arguments == v.arguments;
139 }
140 
142 {
143  if(location < v.location) {
144  return true;
145  }
146 
147  if(v.location < location) {
148  return false;
149  }
150 
151  return linenum < v.linenum;
152 }
153 
154 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
155 {
156  const std::string key = "argument";
157 
158  writer.open_child(key);
159 
160  writer.write_key_val("name", arg);
161  writer.close_child(key);
162 }
163 
164 void preproc_define::write_argument(config_writer& writer, const std::string& arg, const std::string& default_value) const
165 {
166  const std::string key = "argument";
167 
168  writer.open_child(key);
169 
170  writer.write_key_val("name", arg);
171  writer.write_key_val("default", default_value);
172  writer.close_child(key);
173 }
174 
175 void preproc_define::write(config_writer& writer, const std::string& name) const
176 {
177  const std::string key = "preproc_define";
178  writer.open_child(key);
179 
180  writer.write_key_val("name", name);
181  writer.write_key_val("value", value);
182  writer.write_key_val("textdomain", textdomain);
183  writer.write_key_val("linenum", std::to_string(linenum));
184  writer.write_key_val("location", get_location(location));
185 
186  if(is_deprecated()) {
187  writer.open_child("deprecated");
188  writer.write_key_val("level", int(*deprecation_level));
189  writer.write_key_val("version", deprecation_version.str());
190  writer.write_key_val("message", deprecation_message);
191  writer.close_child("deprecated");
192  }
193 
194  for(const std::string& arg : arguments) {
195  write_argument(writer, arg);
196  }
197 
198  for(const auto& [key, default_value] : optional_arguments) {
199  write_argument(writer, key, default_value);
200  }
201 
202  writer.close_child(key);
203 }
204 
206 {
207  if(cfg.has_attribute("default")) {
208  optional_arguments.emplace(cfg["name"], cfg["default"]);
209  } else {
210  arguments.push_back(cfg["name"]);
211  }
212 }
213 
214 void preproc_define::read(const config& cfg)
215 {
216  value = cfg["value"].str();
217  textdomain = cfg["textdomain"].str();
218  linenum = cfg["linenum"];
219  location = cfg["location"].str();
220 
221  if(auto deprecated = cfg.optional_child("deprecated")) {
222  deprecation_level = DEP_LEVEL(deprecated.value()["level"].to_int());
223  deprecation_version = deprecated.value()["version"].str();
224  deprecation_message = deprecated.value()["message"].str();
225  }
226 
227  for(const config& arg : cfg.child_range("argument")) {
228  read_argument(arg);
229  }
230 }
231 
232 preproc_map::value_type preproc_define::read_pair(const config& cfg)
233 {
234  preproc_define second;
235  second.read(cfg);
236 
237  return preproc_map::value_type(cfg["name"], second);
238 }
239 
240 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
241 {
242  return stream << "value: " << def.value << " arguments: " << def.location;
243 }
244 
245 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
246 {
247  return stream << def.second;
248 }
249 
250 // ==================================================================================
251 // PREPROCESSOR BASE
252 // ==================================================================================
253 
255 
256 /**
257  * Base class for preprocessing an input.
258  */
260 {
262 
263 protected:
264  /**
265  * Sets up a new preprocessor for stream buffer \a t.
266  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
267  *
268  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
269  */
271 
273 
274 public:
275  virtual ~preprocessor()
276  {
277  }
278 
279  /** Allows specifying any actions that need to be called after the constructor completes. */
280  virtual void init()
281  {
282  }
283 
284  /**
285  * Preprocesses and sends some text to the #parent_ buffer.
286  * @return false when the input has no data left.
287  */
288  virtual bool get_chunk() = 0;
289 
290  enum MODE { NO_PARSING, PARSES_FILE, PARSES_MACRO };
291 
292  /** Returns the appropriate parsing mode for this preprocessor. */
293  virtual MODE parse_mode()
294  {
295  return NO_PARSING;
296  }
297 
298 private:
299  std::string old_textdomain_;
300  std::string old_location_;
301 
303 };
304 
305 
306 // ==================================================================================
307 // PREPROCESSOR BUFFER
308 // ==================================================================================
309 
310 /**
311  * Target for sending preprocessed output.
312  * Objects of this class can be plugged into an STL stream.
313  */
314 class preprocessor_streambuf : public std::streambuf
315 {
316 public:
318  : std::streambuf()
319  , out_buffer_("")
320  , buffer_()
321  , preprocessor_queue_()
322  , defines_(def)
323  , default_defines_()
324  , textdomain_(PACKAGE)
325  , location_("")
326  , linenum_(0)
327  , quoted_(false)
328  {
329  }
330 
331  /** Decodes the filenames placed in a location. */
332  std::string get_current_file();
333 
334  void error(const std::string&, int);
335  void warning(const std::string&, int);
336 
337  template<typename T, typename... A>
338  void add_preprocessor(A&&... args)
339  {
340  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
341  preprocessor_queue_.back()->init();
342  }
343 
345  {
346  preprocessor_queue_.pop_back();
347  }
348 
349  int depth() const
350  {
351  return preprocessor_queue_.size();
352  }
353 
355  {
356  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
357  }
358 
359 private:
361  : std::streambuf()
362  , out_buffer_("")
363  , buffer_()
364  , preprocessor_queue_()
365  , defines_(t.defines_)
366  , default_defines_()
367  , textdomain_(PACKAGE)
368  , location_("")
369  , linenum_(0)
370  , quoted_(t.quoted_)
371  {
372  }
373 
374  /** Inherited from basic_streambuf. */
375  virtual int underflow() override;
376 
377  void restore_old_preprocessor();
378 
379  /** Buffer read by the STL stream. */
380  std::string out_buffer_;
381 
382  /** Buffer filled by the _current_ preprocessor. */
383  std::stringstream buffer_;
384 
385  /** Input preprocessor queue. */
386  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
387 
390 
391  std::string textdomain_;
392  std::string location_;
393 
394  int linenum_;
395 
396  /**
397  * Set to true if one preprocessor for this target started to read a string.
398  * Deeper-nested preprocessors are then forbidden to.
399  */
400  bool quoted_;
401 
402  friend class preprocessor;
403  friend class preprocessor_file;
404  friend class preprocessor_data;
406 };
407 
408 /** Preprocessor constructor. */
410  : parent_(t)
411  , old_textdomain_(t.textdomain_)
412  , old_location_(t.location_)
413  , old_linenum_(t.linenum_)
414 {
415 }
416 
417 /**
418  * Called by an STL stream whenever it has reached the end of #out_buffer_.
419  * Fills #buffer_ by calling the _current_ preprocessor, then copies its
420  * content into #out_buffer_.
421  * @return the first character of #out_buffer_ if any, EOF otherwise.
422  */
424 {
425  unsigned sz = 0;
426  if(char* gp = gptr()) {
427  if(gp < egptr()) {
428  // Sanity check: the internal buffer has not been totally consumed,
429  // should we force the caller to use what remains first?
430  return *gp;
431  }
432 
433  // The buffer has been completely read; fill it again.
434  // Keep part of the previous buffer, to ensure putback capabilities.
435  sz = out_buffer_.size();
436  buffer_.str(std::string());
437 
438  if(sz > 3) {
439  buffer_ << out_buffer_.substr(sz - 3);
440  sz = 3;
441  } else {
442  buffer_ << out_buffer_;
443  }
444  } else {
445  // The internal get-data pointer is null
446  }
447 
448  const int desired_fill_amount = 2000;
449 
450  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
451  // Process files and data chunks until the desired buffer size is reached
452  if(!current()->get_chunk()) {
453  // Drop the current preprocessor item from the queue.
454  restore_old_preprocessor();
455  }
456  }
457 
458  // Update the internal state and data pointers
459  out_buffer_ = buffer_.str();
460  if(out_buffer_.empty()) {
461  return EOF;
462  }
463 
464  char* begin = &*out_buffer_.begin();
465  unsigned bs = out_buffer_.size();
466 
467  setg(begin, begin + sz, begin + bs);
468 
469  if(sz >= bs) {
470  return EOF;
471  }
472 
473  return static_cast<unsigned char>(*(begin + sz));
474 }
475 
476 /**
477 * Restores the old preprocessing context.
478 * Appends location and domain directives to the buffer, so that the parser
479 * notices these changes.
480 */
482 {
483  preprocessor* current = this->current();
484 
485  if(!current->old_location_.empty()) {
486  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
487  }
488 
489  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
490  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
491  }
492 
493  location_ = current->old_location_;
494  linenum_ = current->old_linenum_;
495  textdomain_ = current->old_textdomain_;
496 
497  // Drop the preprocessor from the queue.
498  drop_preprocessor();
499 }
500 
502 {
503  unsigned nested_level = 0;
504 
505  preprocessor* pre = nullptr;
506 
507  // Iterate backwards over queue to get the last non-macro preprocessor.
508  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
509  pre = p->get();
510 
511  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
512  break;
513  }
514 
515  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
516  ++nested_level;
517  }
518  }
519 
520  std::string res;
521  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
522 
523  if(pos.size() <= 2 * nested_level) {
524  return res;
525  }
526 
527  return get_filename(pos[2 * nested_level]);
528 }
529 
530 std::string lineno_string(const std::string& lineno)
531 {
532  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
533  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
534  std::string included_from = preprocessor_error_detail_prefix + "included from ";
535  std::string res;
536 
537  while(i != end) {
538  const std::string& line = *(i++);
539 
540  if(!res.empty()) {
541  res += included_from;
542  }
543 
544  if(i != end) {
545  res += get_filename(*(i++));
546  } else {
547  res += "<unknown>";
548  }
549 
550  res += ':' + line;
551  }
552 
553  if(res.empty()) {
554  res = "???";
555  }
556 
557  return res;
558 }
559 
560 void preprocessor_streambuf::error(const std::string& error_type, int l)
561 {
562  std::string position, error;
563  std::ostringstream pos;
564 
565  pos << l << ' ' << location_;
566  position = lineno_string(pos.str());
567 
568  error = error_type + '\n';
569  error += "at " + position;
570 
571  ERR_PREPROC << error;
572 
573  throw preproc_config::error(error);
574 }
575 
576 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
577 {
578  std::string position, warning;
579  std::ostringstream pos;
580 
581  pos << l << ' ' << location_;
582  position = lineno_string(pos.str());
583 
584  warning = warning_type + '\n';
585  warning += "at " + position;
586 
587  WRN_PREPROC << warning;
588 }
589 
590 
591 // ==================================================================================
592 // PREPROCESSOR FILE
593 // ==================================================================================
594 
595 /**
596  * Specialized preprocessor for handling a file or a set of files.
597  * A preprocessor_file object is created when a preprocessor encounters an
598  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
599  */
601 {
602 public:
603  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
604  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
605 
606  virtual void init() override;
607 
608  /**
609  * Inserts and processes the next file in the list of included files.
610  * @return false if there is no next file.
611  */
612  virtual bool get_chunk() override
613  {
614  while(pos_ != end_) {
615  const std::string& name = *(pos_++);
616  unsigned sz = name.size();
617 
618  // Use reverse iterator to optimize testing
619  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
620  continue;
621  }
622 
624  return true;
625  }
626 
627  return false;
628  }
629 
630 private:
631  std::vector<std::string> files_;
632  std::vector<std::string>::const_iterator pos_, end_;
633 
634  const std::string& name_;
635 
637 };
638 
639 
640 // ==================================================================================
641 // PREPROCESSOR DATA
642 // ==================================================================================
643 
644 /**
645  * Specialized preprocessor for handling any kind of input stream.
646  * This is the core of the preprocessor.
647  */
649 {
650  /** Description of a preprocessing chunk. */
651  struct token_desc
652  {
653  enum class token_type {
654  start, // Toplevel
655  process_if, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
656  process_else, // Processing the "else" branch of a ifdef/ifndef
657  skip_if, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
658  skip_else, // Skipping the "else" branch of a ifdef/ifndef
659  string, // Processing a string
660  verbatim, // Processing a verbatim string
661  macro_space, // Processing between chunks of a macro call (skip spaces)
662  macro_chunk, // Processing inside a chunk of a macro call (stop on space or '(')
663  macro_parens // Processing a parenthesized macro argument
664  };
665 
666  token_desc(token_type type, const int stack_pos, const int linenum)
667  : type(type)
668  , stack_pos(stack_pos)
669  , linenum(linenum)
670  {
671  }
672 
674 
675  /** Starting position in #strings_ of the delayed text for this chunk. */
677  int linenum;
678  };
679 
680  /**
681  * Manages the lifetime of the @c std::istream pointer we own.
682  *
683  * Since @ref in_ uses the stream as well this object must be created
684  * before @ref in_ and destroyed after @ref in_ is destroyed.
685  */
687 
688  /** Input stream. */
690 
691  std::string directory_;
692 
693  /** Buffer for delayed input processing. */
694  std::vector<std::string> strings_;
695 
696  /** Mapping of macro arguments to their content. */
697  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
698 
699  /** Stack of nested preprocessing chunks. */
700  std::vector<token_desc> tokens_;
701 
702  /**
703  * Set to true whenever input tokens cannot be directly sent to the target
704  * buffer. For instance, this happens with macro arguments. In that case,
705  * the output is redirected toward #strings_ until it can be processed.
706  */
708 
709  /**
710  * Non-zero when the preprocessor has to skip some input text.
711  * Increased whenever entering a conditional branch that is not useful,
712  * e.g. a ifdef that evaluates to false.
713  */
715  int linenum_;
716 
717  /** True iff we are currently parsing a macros content, otherwise false. */
719 
720  std::string read_word();
721  std::string read_line();
722  std::string read_rest_of_line();
723 
724  void skip_spaces();
725  void skip_eol();
726  void push_token(token_desc::token_type);
727  void pop_token();
728  void put(char);
729  void put(const std::string& /*, int change_line = 0 */);
730  void conditional_skip(bool skip);
731 
732 public:
735  const std::string& history,
736  const std::string& name,
737  int line,
738  const std::string& dir,
739  const std::string& domain,
740  std::unique_ptr<std::map<std::string, std::string>> defines,
741  bool is_define = false);
742 
743  virtual bool get_chunk() override;
744 
745  virtual preprocessor::MODE parse_mode() override
746  {
747  return is_define_ ? PARSES_MACRO : PARSES_FILE;
748  }
749 
754 };
755 
757 {
758  throw std::logic_error("don't compare tokens with characters");
759 }
760 
762 {
763  return rhs == lhs;
764 }
765 
767 {
768  return !(lhs == rhs);
769 }
770 
772 {
773  return rhs != lhs;
774 }
775 
776 /** preprocessor_file constructor. */
777 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
778  : preprocessor(t)
779  , files_()
780  , pos_()
781  , end_()
782  , name_(name)
783  , is_directory_(filesystem::is_directory(name))
784 {
785  if(is_directory_) {
786  filesystem::get_files_in_dir(name, &files_, nullptr,
790  );
791 
792  for(const std::string& fname : files_) {
793  std::size_t cpos = fname.rfind(" ");
794 
795  if(cpos != std::string::npos && cpos >= symbol_index) {
796  std::stringstream ss;
797  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
798  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
799  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
800 
801  // TODO: find a real linenumber
802  parent_.error(ss.str(), -1);
803  }
804  }
805  } else {
806  // Handled in the init() function.
807  }
808 
809  pos_ = files_.begin();
810  end_ = files_.end();
811 }
812 
814 {
815  if(is_directory_) {
816  return;
817  }
818 
820 
821  if(!file_stream->good()) {
822  ERR_PREPROC << "Could not open file " << name_;
823  } else {
824  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
827  );
828  }
829 }
830 
833  const std::string& history,
834  const std::string& name,
835  int linenum,
836  const std::string& directory,
837  const std::string& domain,
838  std::unique_ptr<std::map<std::string, std::string>> defines,
839  bool is_define)
840  : preprocessor(t)
841  , in_scope_(std::move(i))
842  , in_(*in_scope_)
843  , directory_(directory)
844  , strings_()
845  , local_defines_(std::move(defines))
846  , tokens_()
847  , slowpath_(0)
848  , skipping_(0)
849  , linenum_(linenum)
850  , is_define_(is_define)
851 {
852  std::ostringstream s;
853  s << history;
854 
855  if(!name.empty()) {
856  if(!history.empty()) {
857  s << ' ';
858  }
859 
860  s << get_file_code(name);
861  }
862 
863  if(!t.location_.empty()) {
864  s << ' ' << t.linenum_ << ' ' << t.location_;
865  }
866 
867  t.location_ = s.str();
868  t.linenum_ = linenum;
869 
870  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
871 
872  if(t.textdomain_ != domain) {
873  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
874  t.textdomain_ = domain;
875  }
876 
878 }
879 
881 {
882  tokens_.emplace_back(t, strings_.size(), linenum_);
883 
885  // Macro expansions do not have any associated storage at start.
886  return;
888  /* Quoted strings are always inlined in the parent token. So
889  * they need neither storage nor metadata, unless the parent
890  * token is a macro expansion.
891  */
892  token_desc::token_type& outer_type = tokens_[tokens_.size() - 2].type;
893  if(outer_type != token_desc::token_type::macro_space) {
894  return;
895  }
896 
898  tokens_.back().stack_pos = strings_.size() + 1;
899  }
900 
901  std::ostringstream s;
902  if(!skipping_ && slowpath_) {
903  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
904  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
905  }
906 
907  strings_.push_back(s.str());
908 }
909 
911 {
912  token_desc::token_type inner_type = tokens_.back().type;
913  unsigned stack_pos = tokens_.back().stack_pos;
914 
915  tokens_.pop_back();
916 
917  token_desc::token_type& outer_type = tokens_.back().type;
918 
919  if(inner_type == token_desc::token_type::macro_parens) {
920  // Parenthesized macro arguments are left on the stack.
921  assert(outer_type == token_desc::token_type::macro_space);
922  return;
923  }
924 
925  if(inner_type == token_desc::token_type::string || inner_type == token_desc::token_type::verbatim) {
926  // Quoted strings are always inlined.
927  assert(stack_pos == strings_.size());
928  return;
929  }
930 
931  if(outer_type == token_desc::token_type::macro_space) {
932  /* A macro expansion does not have any associated storage.
933  * Instead, storage of the inner token is not discarded
934  * but kept as a new macro argument. But if the inner token
935  * was a macro expansion, it is about to be appended, so
936  * prepare for it.
937  */
939  strings_.erase(strings_.begin() + stack_pos, strings_.end());
940  strings_.emplace_back();
941  }
942 
943  assert(stack_pos + 1 == strings_.size());
945 
946  return;
947  }
948 
949  strings_.erase(strings_.begin() + stack_pos, strings_.end());
950 }
951 
953 {
954  while(true) {
955  int c = in_.peek();
956 
957  if(in_.eof() || (c != ' ' && c != '\t')) {
958  return;
959  }
960 
961  in_.get();
962  }
963 }
964 
966 {
967  while(true) {
968  int c = in_.get();
969 
970  if(c == '\n') {
971  ++linenum_;
972  return;
973  }
974 
975  if(in_.eof()) {
976  return;
977  }
978  }
979 }
980 
982 {
983  std::string res;
984 
985  while(true) {
986  int c = in_.peek();
987 
988  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
989  // DBG_PREPROC << "(" << res << ")";
990  return res;
991  }
992 
993  in_.get();
994  res += static_cast<char>(c);
995  }
996 }
997 
999 {
1000  std::string res;
1001 
1002  while(true) {
1003  int c = in_.get();
1004 
1005  if(c == '\n') {
1006  ++linenum_;
1007  return res;
1008  }
1009 
1010  if(in_.eof()) {
1011  return res;
1012  }
1013 
1014  if(c != '\r') {
1015  res += static_cast<char>(c);
1016  }
1017  }
1018 }
1019 
1021 {
1022  std::string res;
1023 
1024  while(in_.peek() != '\n' && !in_.eof()) {
1025  int c = in_.get();
1026 
1027  if(c != '\r') {
1028  res += static_cast<char>(c);
1029  }
1030  }
1031 
1032  return res;
1033 }
1034 
1036 {
1037  if(skipping_) {
1038  return;
1039  }
1040 
1041  if(slowpath_) {
1042  strings_.back() += c;
1043  return;
1044  }
1045 
1046  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1047 
1048  if(unsigned diff = cond_linenum - parent_.linenum_) {
1049  parent_.linenum_ = cond_linenum;
1050 
1051  if(diff <= parent_.location_.size() + 11) {
1052  parent_.buffer_ << std::string(diff, '\n');
1053  } else {
1054  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1055  }
1056  }
1057 
1058  if(c == '\n') {
1059  ++parent_.linenum_;
1060  }
1061 
1062  parent_.buffer_ << c;
1063 }
1064 
1065 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1066 {
1067  if(skipping_) {
1068  return;
1069  }
1070 
1071  if(slowpath_) {
1072  strings_.back() += s;
1073  return;
1074  }
1075 
1076  parent_.buffer_ << s;
1077  // parent_.linenum_ += line_change;
1078 }
1079 
1081 {
1082  if(skip) {
1083  ++skipping_;
1084  }
1085 
1087 }
1088 
1090 {
1091  char c = static_cast<char>(in_.get());
1092  token_desc& token = tokens_.back();
1093 
1094  if(in_.eof()) {
1095  // The end of file was reached.
1096  // Make sure we don't have any incomplete tokens.
1097  char const* s;
1098 
1099  switch(token.type) {
1101  return false; // everything is fine
1106  s = "#ifdef or #ifndef";
1107  break;
1109  s = "Quoted string";
1110  break;
1112  s = "Verbatim string";
1113  break;
1116  s = "Macro substitution";
1117  break;
1119  s = "Macro argument";
1120  break;
1121  default:
1122  s = "???";
1123  }
1124 
1125  parent_.error(std::string(s) + " not terminated", token.linenum);
1126  }
1127 
1128  if(c == '\n') {
1129  ++linenum_;
1130  }
1131 
1132  if(c == OUTPUT_SEPARATOR) {
1133  std::string buffer(1, c);
1134 
1135  while(true) {
1136  char d = static_cast<char>(in_.get());
1137 
1138  if(in_.eof() || d == '\n') {
1139  break;
1140  }
1141 
1142  buffer += d;
1143  }
1144 
1145  buffer += '\n';
1146  // line_change = 1-1 = 0
1147  put(buffer);
1148  } else if(token.type == token_desc::token_type::verbatim) {
1149  put(c);
1150 
1151  if(c == '>' && in_.peek() == '>') {
1152  put(in_.get());
1153  pop_token();
1154  }
1155  } else if(c == '<' && in_.peek() == '<') {
1156  in_.get();
1158  put('<');
1159  put('<');
1160  } else if(c == '"') {
1161  if(token.type == token_desc::token_type::string) {
1162  parent_.quoted_ = false;
1163  put(c);
1164  pop_token();
1165  } else if(!parent_.quoted_) {
1166  parent_.quoted_ = true;
1168  put(c);
1169  } else {
1170  parent_.error("Nested quoted string", linenum_);
1171  }
1172  } else if(c == '{') {
1174  ++slowpath_;
1175  } else if(c == ')' && token.type == token_desc::token_type::macro_parens) {
1176  pop_token();
1177  } else if(c == '#' && !parent_.quoted_) {
1178  std::string command = read_word();
1179  bool comment = false;
1180 
1181  if(command == "define") {
1182  skip_spaces();
1183  int linenum = linenum_;
1184  std::vector<std::string> items = utils::split(read_line(), ' ');
1185  std::map<std::string, std::string> optargs;
1186 
1187  if(items.empty()) {
1188  parent_.error("No macro name found after #define directive", linenum);
1189  }
1190 
1191  std::string symbol = items.front();
1192  items.erase(items.begin());
1193  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1194  std::optional<DEP_LEVEL> deprecation_level;
1195  std::string buffer, deprecation_detail;
1196  version_info deprecation_version = game_config::wesnoth_version;
1197  while(true) {
1198  if(in_.eof())
1199  break;
1200  char d = static_cast<char>(in_.get());
1201  if(d == '\n')
1202  ++linenum_;
1203  buffer += d;
1204  if(d == '#') {
1205  if(in_.peek() == 'a') {
1206  found_arg = 1;
1207  } else if(in_.peek() == 'd') {
1208  found_deprecate = 1;
1209  } else {
1210  found_enddef = 1;
1211  }
1212  } else {
1213  if(found_arg > 0 && ++found_arg == 4) {
1214  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1215  buffer.erase(buffer.end() - 4, buffer.end());
1216 
1217  skip_spaces();
1218  std::string argname = read_word();
1219  skip_eol();
1220 
1221  std::string argbuffer;
1222 
1223  int found_endarg = 0;
1224  while(true) {
1225  if(in_.eof()) {
1226  break;
1227  }
1228 
1229  char e = static_cast<char>(in_.get());
1230  if(e == '\n') {
1231  ++linenum_;
1232  }
1233 
1234  argbuffer += e;
1235 
1236  if(e == '#') {
1237  found_endarg = 1;
1238  } else if(found_endarg > 0 && ++found_endarg == 7) {
1239  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1240  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1241  optargs[argname] = argbuffer;
1242  skip_eol();
1243  break;
1244  } else {
1245  parent_.error("Unterminated #arg definition", linenum_);
1246  }
1247  }
1248  }
1249  }
1250  }
1251 
1252  if(found_deprecate > 0 && ++found_deprecate == 11) {
1253  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1254  buffer.erase(buffer.end() - 11, buffer.end());
1255  skip_spaces();
1256  try {
1257  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1258  if(deprecation_level) {
1259  deprecation_level = std::max(*deprecation_level, level);
1260  } else {
1261  deprecation_level = level;
1262  }
1263  } catch(const std::invalid_argument&) {
1264  // Meh, fall back to default of PREEMPTIVE...
1265  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1266  }
1267  deprecation_version = game_config::wesnoth_version;
1268  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1269  skip_spaces();
1270  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1271  }
1272  skip_spaces();
1273  if(!deprecation_detail.empty()){
1274  deprecation_detail += '\n';
1275  }
1276  deprecation_detail += read_rest_of_line();
1277  skip_eol();
1278  }
1279  }
1280 
1281  if(found_enddef > 0 && ++found_enddef == 7) {
1282  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1283  break;
1284  } else {
1285  found_enddef = 0;
1286  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1287  // this? This would fill feature
1288  // request #21343
1289  parent_.error(
1290  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1291  linenum);
1292  }
1293  }
1294  }
1295  }
1296  }
1297 
1298  if(found_enddef != 7) {
1299  parent_.error("Unterminated preprocessor definition", linenum_);
1300  }
1301 
1302  if(!skipping_) {
1303  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1304  if(old_i != parent_.defines_->end()) {
1305  std::ostringstream new_pos, old_pos;
1306  const preproc_define& old_d = old_i->second;
1307 
1308  new_pos << linenum << ' ' << parent_.location_;
1309  old_pos << old_d.linenum << ' ' << old_d.location;
1310 
1311  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1312  << lineno_string(new_pos.str()) << '\n'
1313  << "previously defined at " << lineno_string(old_pos.str());
1314  }
1315 
1316  buffer.erase(buffer.end() - 7, buffer.end());
1317  (*parent_.defines_)[symbol]
1318  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1319  deprecation_detail, deprecation_level, deprecation_version);
1320 
1321  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1322  }
1323  } else if(command == "ifdef" || command == "ifndef") {
1324  const bool negate = command[2] == 'n';
1325  skip_spaces();
1326  const std::string& symbol = read_word();
1327  bool found = parent_.defines_->count(symbol) != 0;
1328  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined");
1329  conditional_skip(negate ? found : !found);
1330  } else if(command == "ifhave" || command == "ifnhave") {
1331  const bool negate = command[2] == 'n';
1332  skip_spaces();
1333  const std::string& symbol = read_word();
1334  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1335  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found");
1336  conditional_skip(negate ? found : !found);
1337  } else if(command == "ifver" || command == "ifnver") {
1338  const bool negate = command[2] == 'n';
1339 
1340  skip_spaces();
1341  const std::string& vsymstr = read_word();
1342  skip_spaces();
1343  const std::string& vopstr = read_word();
1344  skip_spaces();
1345  const std::string& vverstr = read_word();
1346 
1347  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1348 
1349  if(vop == OP_INVALID) {
1350  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1351  } else if(parent_.defines_->count(vsymstr) != 0) {
1352  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1353 
1354  if(!sym.arguments.empty()) {
1355  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1356  }
1357 
1358  version_info const version1(sym.value);
1359  version_info const version2(vverstr);
1360 
1361  const bool found = do_version_check(version1, vop, version2);
1362  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1363  << vopstr << "): " << (found ? "match" : "no match");
1364 
1365  conditional_skip(negate ? found : !found);
1366  } else {
1367  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1368  err += vsymstr;
1369  err += "'";
1370  parent_.error(err, linenum_);
1371  }
1372  } else if(command == "else") {
1374  pop_token();
1375  --skipping_;
1377  } else if(token.type == token_desc::token_type::process_if) {
1378  pop_token();
1379  ++skipping_;
1381  } else {
1382  parent_.error("Unexpected #else", linenum_);
1383  }
1384  } else if(command == "endif") {
1385  switch(token.type) {
1388  --skipping_;
1391  break;
1392  default:
1393  parent_.error("Unexpected #endif", linenum_);
1394  }
1395  pop_token();
1396  } else if(command == "textdomain") {
1397  skip_spaces();
1398  const std::string& s = read_word();
1399  if(s != parent_.textdomain_) {
1400  put("#textdomain ");
1401  put(s);
1402  parent_.textdomain_ = s;
1403  }
1404  comment = true;
1405  } else if(command == "enddef") {
1406  parent_.error("Unexpected #enddef", linenum_);
1407  } else if(command == "undef") {
1408  skip_spaces();
1409  const std::string& symbol = read_word();
1410  if(!skipping_) {
1411  parent_.defines_->erase(symbol);
1412  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1413  }
1414  } else if(command == "error") {
1415  if(!skipping_) {
1416  skip_spaces();
1417  std::ostringstream error;
1418  error << "#error: \"" << read_rest_of_line() << '"';
1419  parent_.error(error.str(), linenum_);
1420  } else
1421  DBG_PREPROC << "Skipped an error";
1422  } else if(command == "warning") {
1423  if(!skipping_) {
1424  skip_spaces();
1425  std::ostringstream warning;
1426  warning << "#warning: \"" << read_rest_of_line() << '"';
1427  parent_.warning(warning.str(), linenum_);
1428  } else {
1429  DBG_PREPROC << "Skipped a warning";
1430  }
1431  } else if(command == "deprecated") {
1432  // The current file is deprecated, so print a message
1433  skip_spaces();
1435  try {
1436  level = DEP_LEVEL(std::stoi(read_word()));
1437  } catch(const std::invalid_argument&) {
1438  // Meh, just fall back to the default of PREEMPTIVE...
1439  }
1441  if(level == DEP_LEVEL::PREEMPTIVE || level == DEP_LEVEL::FOR_REMOVAL) {
1442  skip_spaces();
1443  version = version_info(read_word());
1444  }
1445  skip_spaces();
1446  std::string detail = read_rest_of_line();
1447  deprecated_message(get_filename(parent_.location_), level, version, detail);
1448  } else {
1449  comment = token.type != token_desc::token_type::macro_space;
1450  }
1451 
1452  skip_eol();
1453  if(comment) {
1454  put('\n');
1455  }
1457  if(c == '(') {
1458  // If a macro argument was started, it is implicitly ended.
1461  } else if(utils::portable_isspace(c)) {
1462  // If a macro argument was started, it is implicitly ended.
1464  } else if(c == '}') {
1465  --slowpath_;
1466  if(skipping_) {
1467  pop_token();
1468  return true;
1469  }
1470 
1471  // FIXME: is this obsolete?
1472  // if (token.type == token_desc::MACRO_SPACE) {
1473  // if (!strings_.back().empty()) {
1474  // std::ostringstream error;
1475  // std::ostringstream location;
1476  // error << "Can't parse new macro parameter with a macro call scope open";
1477  // location<<linenum_<<' '<<parent_.location_;
1478  // parent_.error(error.str(), location.str());
1479  // }
1480  // strings_.pop_back();
1481  //}
1482 
1483  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1484  parent_.error("No macro or file substitution target specified", linenum_);
1485  }
1486 
1487  std::string symbol = strings_[token.stack_pos];
1488  std::string::size_type pos;
1489  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1490  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1491  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1492  }
1493 
1494  std::map<std::string, std::string>::const_iterator arg;
1495  preproc_map::const_iterator macro;
1496 
1497  // If this is a known pre-processing symbol, then we insert it,
1498  // otherwise we assume it's a file name to load.
1499  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1500  pop_token();
1502  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1503  pop_token();
1505  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1506  pop_token();
1507  put("{");
1508  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1509  pop_token();
1510  put("}");
1511  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1512  if(strings_.size() - token.stack_pos != 1) {
1513  std::ostringstream error;
1514  error << "Macro argument '" << symbol << "' does not expect any arguments";
1515  parent_.error(error.str(), linenum_);
1516  }
1517 
1518  std::ostringstream v;
1519  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1520  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1521 
1522  pop_token();
1523  put(v.str());
1524  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1525  const preproc_define& val = macro->second;
1526  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1527  std::size_t optional_arg_num = 0;
1528 
1529  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1530  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1531 
1532  if(val.is_deprecated()) {
1534  }
1535 
1536  for(std::size_t i = 0; i < nb_arg; ++i) {
1537  if(i < val.arguments.size()) {
1538  // Normal mandatory arguments
1539 
1540  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1541  } else {
1542  // These should be optional argument overrides
1543 
1544  std::string str = strings_[token.stack_pos + i + 1];
1545  std::size_t equals_pos = str.find_first_of("=");
1546 
1547  if(equals_pos != std::string::npos) {
1548  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1549 
1550  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1551 
1552  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1553  (*defines)[argname] = str.substr(equals_pos + 1);
1554 
1555  optional_arg_num++;
1556 
1557  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol;
1558  } else {
1559  std::ostringstream warning;
1560  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1561  << argname << "'";
1562  parent_.warning(warning.str(), linenum_);
1563 
1564  optional_arg_num++; // To prevent the argument number check from blowing up
1565  }
1566  }
1567  }
1568  }
1569 
1570  // If the macro definition has any optional arguments, insert their defaults
1571  if(val.optional_arguments.size() > 0) {
1572  for(const auto& argument : val.optional_arguments) {
1573  if(defines->find(argument.first) == defines->end()) {
1574  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1575 
1577  std::istream in(buf.get());
1578 
1579  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1580 
1581  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1582  temp_defines->insert(defines->begin(), defines->end());
1583 
1585  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1586 
1587  std::ostringstream res;
1588  res << in.rdbuf();
1589 
1590  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1591  << symbol;
1592 
1593  (*defines)[argument.first] = res.str();
1594  }
1595  }
1596  }
1597 
1598  if(nb_arg - optional_arg_num != val.arguments.size()) {
1599  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1600  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1601  std::ostringstream error;
1602  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1603  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1604  << nb_arg - optional_arg_num << " arguments";
1605  parent_.error(error.str(), linenum_);
1606  }
1607 
1608  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1609 
1610  pop_token();
1611 
1612  if(!slowpath_) {
1613  DBG_PREPROC << "substituting macro " << symbol;
1614 
1616  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1617  } else {
1618  DBG_PREPROC << "substituting (slow) macro " << symbol;
1619 
1620  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1621 
1622  // Make the nested preprocessor_data responsible for
1623  // restoring our current textdomain if needed.
1625 
1626  std::ostringstream res;
1627  {
1628  std::istream in(buf.get());
1630  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1631 
1632  res << in.rdbuf();
1633  }
1634 
1635  put(res.str());
1636  }
1637  } else if(parent_.depth() < 40) {
1638  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.";
1639  pop_token();
1640 
1641  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1642  if(!nfname.empty()) {
1643  if(!slowpath_)
1644  // nfname.size() - symbol.size() gives you an index into nfname
1645  // This does not necessarily match the symbol though, as it can start with ~ or ./
1646  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1647  else {
1648  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1649 
1650  std::ostringstream res;
1651  {
1652  std::istream in(buf.get());
1653  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1654 
1655  res << in.rdbuf();
1656  }
1657 
1658  put(res.str());
1659  }
1660  } else {
1661  std::ostringstream error;
1662  error << "Macro/file '" << symbol << "' is missing";
1663  parent_.error(error.str(), linenum_);
1664  }
1665  } else {
1666  parent_.error("Too many nested preprocessing inclusions", linenum_);
1667  }
1668  } else if(!skipping_) {
1670  std::ostringstream s;
1671  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1672  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1673 
1674  strings_.push_back(s.str());
1676  }
1677  put(c);
1678  }
1679  } else {
1680  put(c);
1681  }
1682 
1683  return true;
1684 }
1685 
1686 
1687 // ==================================================================================
1688 // PREPROCESSOR SCOPE HELPER
1689 // ==================================================================================
1690 
1691 struct preprocessor_scope_helper : std::basic_istream<char>
1692 {
1693  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1694  : std::basic_istream<char>(nullptr)
1695  , buf_(nullptr)
1696  , local_defines_(nullptr)
1697  {
1698  //
1699  // If no defines were provided, we create a new local preproc_map and assign
1700  // it to defines temporarily. In this case, the map will be deleted once this
1701  // object is destroyed and defines will still be subsequently null.
1702  //
1703  if(!defines) {
1704  local_defines_.reset(new preproc_map);
1705  defines = local_defines_.get();
1706  }
1707 
1708  buf_.reset(new preprocessor_streambuf(defines));
1709 
1710  // Begin processing.
1711  buf_->add_preprocessor<preprocessor_file>(fname);
1712 
1713  //
1714  // TODO: not sure if this call is needed. Previously, this call was passed a
1715  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1716  // called with its contents. However, at that point the preprocessing should
1717  // already have completed, meaning this call might be redundant. Not sure.
1718  //
1719  // - vultraz, 2017-08-31
1720  //
1721  init(buf_.get());
1722  }
1723 
1725  {
1726  clear(std::ios_base::goodbit);
1727  exceptions(std::ios_base::goodbit);
1728  rdbuf(nullptr);
1729  }
1730 
1731  std::unique_ptr<preprocessor_streambuf> buf_;
1732  std::unique_ptr<preproc_map> local_defines_;
1733 };
1734 
1735 
1736 // ==================================================================================
1737 // FREE-STANDING FUNCTIONS
1738 // ==================================================================================
1739 
1740 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1741 {
1742  log_scope("preprocessing file " + fname + " ...");
1743 
1744  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1745  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1746 }
1747 
1748 void preprocess_resource(const std::string& res_name,
1749  preproc_map* defines_map,
1750  bool write_cfg,
1751  bool write_plain_cfg,
1752  const std::string& parent_directory)
1753 {
1754  if(filesystem::is_directory(res_name)) {
1755  std::vector<std::string> dirs, files;
1756 
1759 
1760  // Subdirectories
1761  for(const std::string& dir : dirs) {
1762  LOG_PREPROC << "processing sub-dir: " << dir;
1763  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1764  }
1765 
1766  // Files in current directory
1767  for(const std::string& file : files) {
1768  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1769  }
1770 
1771  return;
1772  }
1773 
1774  // process only config files.
1775  if(!filesystem::ends_with(res_name, ".cfg")) {
1776  return;
1777  }
1778 
1779  LOG_PREPROC << "processing resource: " << res_name;
1780 
1781  // disable filename encoding to get clear #line in cfg.plain
1782  encode_filename = false;
1783 
1784  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1785 
1786  std::stringstream ss;
1787 
1788  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1789  // they will be propagated in the main program, instead of just setting the
1790  // failbit on the stream. This was necessary in order for the MSVC and GCC
1791  // binaries to behave the same way.
1792  ss.exceptions(std::ios_base::failbit);
1793 
1794  ss << (*stream).rdbuf();
1795 
1796  LOG_PREPROC << "processing finished";
1797 
1798  if(write_cfg || write_plain_cfg) {
1799  config cfg;
1800  std::string streamContent = ss.str();
1801 
1802  read(cfg, streamContent);
1803 
1804  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1805 
1806  // Write the processed cfg file
1807  if(write_cfg) {
1808  LOG_PREPROC << "writing cfg file: " << preproc_res_name;
1809 
1811  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1812 
1813  write(*outStream, cfg);
1814  }
1815 
1816  // Write the plain cfg file
1817  if(write_plain_cfg) {
1818  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain");
1819 
1821  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1822  }
1823  }
1824 }
std::string lineno_string(const std::string &lineno)
VERSION_COMP_OP
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
std::optional< DEP_LEVEL > deprecation_level
preprocessor * current() const
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
void write(config_writer &, const std::string &) const
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
Interfaces for manipulating version numbers of engine, add-ons, etc.
static const std::string current_file_str
std::vector< std::string >::const_iterator end_
VERSION_COMP_OP parse_version_op(const std::string &op_str)
std::string read_word()
preproc_map default_defines_
bool has_attribute(config_key_type key) const
Definition: config.cpp:211
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:344
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void clear(const std::string &key)
Definition: general.cpp:190
Helper class for buffering a std::istream.
const std::vector< std::string > items
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:21
#define d
std::unique_ptr< preprocessor_streambuf > buf_
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string >::const_iterator pos_
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
void write_argument(config_writer &, const std::string &) const
unsigned in
If equal to search_counter, the node is off the list.
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:764
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:30
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
void add_preprocessor(A &&... args)
static lg::log_domain log_preprocessor("preprocessor")
EXIT_STATUS start(const std::string &filename, bool take_screenshot, const std::string &screenshot_filename)
Main interface for launching the editor from the title screen.
Definition: editor_main.cpp:30
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:627
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
int skipping_
Non-zero when the preprocessor has to skip some input text.
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:350
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_deprecated() const
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
utils::optional_reference< config > optional_child(config_key_type key, int n=0)
Euivalent to child, but returns an empty optional if the nth child was not found. ...
Definition: config.cpp:445
std::string value
#define LOG_PREPROC
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
void write_file(const std::string &fname, const std::string &data, std::ios_base::openmode mode)
Throws io_exception if an error occurs.
void push_token(token_desc::token_type)
bool operator==(const preproc_define &) const
std::size_t i
Definition: function.cpp:967
void line(int from_x, int from_y, int to_x, int to_y)
Draw a line.
Definition: draw.cpp:171
logger & err()
Definition: log.cpp:170
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn&#39;t pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:237
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
double t
Definition: astarsearch.cpp:65
std::vector< std::string > split(const config_attribute_value &val)
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
static const std::string current_dir_str
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
std::string str() const
Serializes the version number into string form.
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
void error(const std::string &, int)
Target for sending preprocessed output.
bool operator!=(preprocessor_data::token_desc::token_type rhs, char lhs)
std::map< std::string, struct preproc_define > preproc_map
#define e
static const std::string right_curly_str
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:60
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static preproc_map::value_type read_pair(const config &)
mock_char c
bool operator==(preprocessor_data::token_desc::token_type, char)
const std::string & name_
std::string old_textdomain_
static map_location::DIRECTION n
bool operator<(const preproc_define &) const
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
std::stringstream buffer_
Buffer filled by the current preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool eof() const
Is the end of input reached?
token_desc(token_type type, const int stack_pos, const int linenum)
static const std::string left_curly_str
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:322
static std::string preprocessor_error_detail_prefix
std::map< std::string, std::string > optional_arguments