The Battle for Wesnoth  1.15.12+dev
lutf8lib.cpp
Go to the documentation of this file.
1 /*
2 ** $Id: lutf8lib.c $
3 ** Standard library for UTF-8 manipulation
4 ** See Copyright Notice in lua.h
5 */
6 
7 #define lutf8lib_c
8 #define LUA_LIB
9 
10 #include "lprefix.h"
11 
12 
13 #include <assert.h>
14 #include <limits.h>
15 #include <stdlib.h>
16 #include <string.h>
17 
18 #include "lua.h"
19 
20 #include "lauxlib.h"
21 #include "lualib.h"
22 
23 
24 #define MAXUNICODE 0x10FFFFu
25 
26 #define MAXUTF 0x7FFFFFFFu
27 
28 /*
29 ** Integer type for decoded UTF-8 values; MAXUTF needs 31 bits.
30 */
31 #if (UINT_MAX >> 30) >= 1
32 typedef unsigned int utfint;
33 #else
34 typedef unsigned long utfint;
35 #endif
36 
37 
38 #define iscont(p) ((*(p) & 0xC0) == 0x80)
39 
40 
41 /* from strlib */
42 /* translate a relative string position: negative means back from end */
43 static lua_Integer u_posrelat (lua_Integer pos, size_t len) {
44  if (pos >= 0) return pos;
45  else if (0u - (size_t)pos > len) return 0;
46  else return (lua_Integer)len + pos + 1;
47 }
48 
49 
50 /*
51 ** Decode one UTF-8 sequence, returning NULL if byte sequence is
52 ** invalid. The array 'limits' stores the minimum value for each
53 ** sequence length, to check for overlong representations. Its first
54 ** entry forces an error for non-ascii bytes with no continuation
55 ** bytes (count == 0).
56 */
57 static const char *utf8_decode (const char *s, utfint *val, int strict) {
58  static const utfint limits[] =
59  {~(utfint)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u};
60  unsigned int c = (unsigned char)s[0];
61  utfint res = 0; /* final result */
62  if (c < 0x80) /* ascii? */
63  res = c;
64  else {
65  int count = 0; /* to count number of continuation bytes */
66  for (; c & 0x40; c <<= 1) { /* while it needs continuation bytes... */
67  unsigned int cc = (unsigned char)s[++count]; /* read next byte */
68  if ((cc & 0xC0) != 0x80) /* not a continuation byte? */
69  return NULL; /* invalid byte sequence */
70  res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */
71  }
72  res |= ((utfint)(c & 0x7F) << (count * 5)); /* add first byte */
73  if (count > 5 || res > MAXUTF || res < limits[count])
74  return NULL; /* invalid byte sequence */
75  s += count; /* skip continuation bytes read */
76  }
77  if (strict) {
78  /* check for invalid code points; too large or surrogates */
79  if (res > MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu))
80  return NULL;
81  }
82  if (val) *val = res;
83  return s + 1; /* +1 to include first byte */
84 }
85 
86 
87 /*
88 ** utf8len(s [, i [, j [, lax]]]) --> number of characters that
89 ** start in the range [i,j], or nil + current position if 's' is not
90 ** well formed in that interval
91 */
92 static int utflen (lua_State *L) {
93  lua_Integer n = 0; /* counter for the number of characters */
94  size_t len; /* string length in bytes */
95  const char *s = luaL_checklstring(L, 1, &len);
96  lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
97  lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len);
98  int lax = lua_toboolean(L, 4);
99  luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2,
100  "initial position out of bounds");
101  luaL_argcheck(L, --posj < (lua_Integer)len, 3,
102  "final position out of bounds");
103  while (posi <= posj) {
104  const char *s1 = utf8_decode(s + posi, NULL, !lax);
105  if (s1 == NULL) { /* conversion error? */
106  luaL_pushfail(L); /* return fail ... */
107  lua_pushinteger(L, posi + 1); /* ... and current position */
108  return 2;
109  }
110  posi = s1 - s;
111  n++;
112  }
113  lua_pushinteger(L, n);
114  return 1;
115 }
116 
117 
118 /*
119 ** codepoint(s, [i, [j [, lax]]]) -> returns codepoints for all
120 ** characters that start in the range [i,j]
121 */
122 static int codepoint (lua_State *L) {
123  size_t len;
124  const char *s = luaL_checklstring(L, 1, &len);
125  lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
126  lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len);
127  int lax = lua_toboolean(L, 4);
128  int n;
129  const char *se;
130  luaL_argcheck(L, posi >= 1, 2, "out of bounds");
131  luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of bounds");
132  if (posi > pose) return 0; /* empty interval; return no values */
133  if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */
134  return luaL_error(L, "string slice too long");
135  n = (int)(pose - posi) + 1; /* upper bound for number of returns */
136  luaL_checkstack(L, n, "string slice too long");
137  n = 0; /* count the number of returns */
138  se = s + pose; /* string end */
139  for (s += posi - 1; s < se;) {
140  utfint code;
141  s = utf8_decode(s, &code, !lax);
142  if (s == NULL)
143  return luaL_error(L, "invalid UTF-8 code");
144  lua_pushinteger(L, code);
145  n++;
146  }
147  return n;
148 }
149 
150 
151 static void pushutfchar (lua_State *L, int arg) {
153  luaL_argcheck(L, code <= MAXUTF, arg, "value out of range");
154  lua_pushfstring(L, "%U", (long)code);
155 }
156 
157 
158 /*
159 ** utfchar(n1, n2, ...) -> char(n1)..char(n2)...
160 */
161 static int utfchar (lua_State *L) {
162  int n = lua_gettop(L); /* number of arguments */
163  if (n == 1) /* optimize common case of single char */
164  pushutfchar(L, 1);
165  else {
166  int i;
167  luaL_Buffer b;
168  luaL_buffinit(L, &b);
169  for (i = 1; i <= n; i++) {
170  pushutfchar(L, i);
171  luaL_addvalue(&b);
172  }
173  luaL_pushresult(&b);
174  }
175  return 1;
176 }
177 
178 
179 /*
180 ** offset(s, n, [i]) -> index where n-th character counting from
181 ** position 'i' starts; 0 means character at 'i'.
182 */
183 static int byteoffset (lua_State *L) {
184  size_t len;
185  const char *s = luaL_checklstring(L, 1, &len);
187  lua_Integer posi = (n >= 0) ? 1 : len + 1;
188  posi = u_posrelat(luaL_optinteger(L, 3, posi), len);
189  luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3,
190  "position out of bounds");
191  if (n == 0) {
192  /* find beginning of current byte sequence */
193  while (posi > 0 && iscont(s + posi)) posi--;
194  }
195  else {
196  if (iscont(s + posi))
197  return luaL_error(L, "initial position is a continuation byte");
198  if (n < 0) {
199  while (n < 0 && posi > 0) { /* move back */
200  do { /* find beginning of previous character */
201  posi--;
202  } while (posi > 0 && iscont(s + posi));
203  n++;
204  }
205  }
206  else {
207  n--; /* do not move for 1st character */
208  while (n > 0 && posi < (lua_Integer)len) {
209  do { /* find beginning of next character */
210  posi++;
211  } while (iscont(s + posi)); /* (cannot pass final '\0') */
212  n--;
213  }
214  }
215  }
216  if (n == 0) /* did it find given character? */
217  lua_pushinteger(L, posi + 1);
218  else /* no such character */
219  luaL_pushfail(L);
220  return 1;
221 }
222 
223 
224 static int iter_aux (lua_State *L, int strict) {
225  size_t len;
226  const char *s = luaL_checklstring(L, 1, &len);
227  lua_Integer n = lua_tointeger(L, 2) - 1;
228  if (n < 0) /* first iteration? */
229  n = 0; /* start from here */
230  else if (n < (lua_Integer)len) {
231  n++; /* skip current byte */
232  while (iscont(s + n)) n++; /* and its continuations */
233  }
234  if (n >= (lua_Integer)len)
235  return 0; /* no more codepoints */
236  else {
237  utfint code;
238  const char *next = utf8_decode(s + n, &code, strict);
239  if (next == NULL)
240  return luaL_error(L, "invalid UTF-8 code");
241  lua_pushinteger(L, n + 1);
242  lua_pushinteger(L, code);
243  return 2;
244  }
245 }
246 
247 
248 static int iter_auxstrict (lua_State *L) {
249  return iter_aux(L, 1);
250 }
251 
252 static int iter_auxlax (lua_State *L) {
253  return iter_aux(L, 0);
254 }
255 
256 
257 static int iter_codes (lua_State *L) {
258  int lax = lua_toboolean(L, 2);
259  luaL_checkstring(L, 1);
261  lua_pushvalue(L, 1);
262  lua_pushinteger(L, 0);
263  return 3;
264 }
265 
266 
267 /* pattern to match a single UTF-8 character */
268 #define UTF8PATT "[\0-\x7F\xC2-\xFD][\x80-\xBF]*"
269 
270 
271 static const luaL_Reg funcs[] = {
272  {"offset", byteoffset},
273  {"codepoint", codepoint},
274  {"char", utfchar},
275  {"len", utflen},
276  {"codes", iter_codes},
277  /* placeholders */
278  {"charpattern", NULL},
279  {NULL, NULL}
280 };
281 
282 
284  luaL_newlib(L, funcs);
285  lua_pushlstring(L, UTF8PATT, sizeof(UTF8PATT)/sizeof(char) - 1);
286  lua_setfield(L, -2, "charpattern");
287  return 1;
288 }
289 
LUALIB_API void luaL_addvalue(luaL_Buffer *B)
Definition: lauxlib.cpp:609
#define lua_pushcfunction(L, f)
Definition: lua.h:370
#define luaL_pushfail(L)
Definition: lauxlib.h:157
static lua_Integer u_posrelat(lua_Integer pos, size_t len)
Definition: lutf8lib.cpp:43
#define iscont(p)
Definition: lutf8lib.cpp:38
LUALIB_API void luaL_pushresult(luaL_Buffer *B)
Definition: lauxlib.cpp:584
LUA_API int lua_gettop(lua_State *L)
Definition: lapi.cpp:168
static int iter_auxlax(lua_State *L)
Definition: lutf8lib.cpp:252
#define lua_tointeger(L, i)
Definition: lua.h:362
static const char * utf8_decode(const char *s, utfint *val, int strict)
Definition: lutf8lib.cpp:57
LUALIB_API lua_Integer luaL_optinteger(lua_State *L, int arg, lua_Integer def)
Definition: lauxlib.cpp:452
LUALIB_API void luaL_checkstack(lua_State *L, int space, const char *msg)
Definition: lauxlib.cpp:380
static int byteoffset(lua_State *L)
Definition: lutf8lib.cpp:183
#define b
static int utfchar(lua_State *L)
Definition: lutf8lib.cpp:161
#define MAXUNICODE
Definition: lutf8lib.cpp:24
LUA_INTEGER lua_Integer
Definition: lua.h:94
static int iter_auxstrict(lua_State *L)
Definition: lutf8lib.cpp:248
LUA_API int lua_toboolean(lua_State *L, int idx)
Definition: lapi.cpp:375
LUALIB_API void luaL_buffinit(lua_State *L, luaL_Buffer *B)
Definition: lauxlib.cpp:620
LUA_API const char * lua_pushlstring(lua_State *L, const char *s, size_t len)
Definition: lapi.cpp:502
static int utflen(lua_State *L)
Definition: lutf8lib.cpp:92
static int codepoint(lua_State *L)
Definition: lutf8lib.cpp:122
static map_location::DIRECTION se
#define UTF8PATT
Definition: lutf8lib.cpp:268
static const luaL_Reg funcs[]
Definition: lutf8lib.cpp:271
static void pushutfchar(lua_State *L, int arg)
Definition: lutf8lib.cpp:151
std::size_t i
Definition: function.cpp:940
LUALIB_API lua_Integer luaL_checkinteger(lua_State *L, int arg)
Definition: lauxlib.cpp:442
static map_location::DIRECTION s
#define LUAMOD_API
Definition: luaconf.h:286
LUAMOD_API int luaopen_utf8(lua_State *L)
Definition: lutf8lib.cpp:283
LUA_API void lua_pushvalue(lua_State *L, int idx)
Definition: lapi.cpp:246
#define luaL_newlib(L, l)
Definition: lauxlib.h:129
LUA_UNSIGNED lua_Unsigned
Definition: lua.h:97
LUALIB_API int luaL_error(lua_State *L, const char *fmt,...)
Definition: lauxlib.cpp:234
#define luaL_argcheck(L, cond, arg, extramsg)
Definition: lauxlib.h:132
#define next(ls)
Definition: llex.cpp:32
LUA_API const char * lua_pushfstring(lua_State *L, const char *fmt,...)
Definition: lapi.cpp:542
unsigned long utfint
Definition: lutf8lib.cpp:34
#define MAXUTF
Definition: lutf8lib.cpp:26
static int iter_codes(lua_State *L)
Definition: lutf8lib.cpp:257
mock_char c
static map_location::DIRECTION n
LUA_API void lua_pushinteger(lua_State *L, lua_Integer n)
Definition: lapi.cpp:489
LUALIB_API const char * luaL_checklstring(lua_State *L, int arg, size_t *len)
Definition: lauxlib.cpp:402
LUA_API void lua_setfield(lua_State *L, int idx, const char *k)
Definition: lapi.cpp:837
static int iter_aux(lua_State *L, int strict)
Definition: lutf8lib.cpp:224
#define luaL_checkstring(L, n)
Definition: lauxlib.h:138