utf8.h 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881
  1. // © 2016 and later: Unicode, Inc. and others.
  2. // License & terms of use: http://www.unicode.org/copyright.html
  3. /*
  4. *******************************************************************************
  5. *
  6. * Copyright (C) 1999-2015, International Business Machines
  7. * Corporation and others. All Rights Reserved.
  8. *
  9. *******************************************************************************
  10. * file name: utf8.h
  11. * encoding: UTF-8
  12. * tab size: 8 (not used)
  13. * indentation:4
  14. *
  15. * created on: 1999sep13
  16. * created by: Markus W. Scherer
  17. */
  18. /**
  19. * \file
  20. * \brief C API: 8-bit Unicode handling macros
  21. *
  22. * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings.
  23. *
  24. * For more information see utf.h and the ICU User Guide Strings chapter
  25. * (http://userguide.icu-project.org/strings).
  26. *
  27. * <em>Usage:</em>
  28. * ICU coding guidelines for if() statements should be followed when using these macros.
  29. * Compound statements (curly braces {}) must be used for if-else-while...
  30. * bodies and all macro statements should be terminated with semicolon.
  31. */
  32. #ifndef __UTF8_H__
  33. #define __UTF8_H__
  34. #include "unicode/umachine.h"
  35. #ifndef __UTF_H__
  36. # include "unicode/utf.h"
  37. #endif
  38. /* internal definitions ----------------------------------------------------- */
  39. /**
  40. * Counts the trail bytes for a UTF-8 lead byte.
  41. * Returns 0 for 0..0xc1 as well as for 0xf5..0xff.
  42. * leadByte might be evaluated multiple times.
  43. *
  44. * This is internal since it is not meant to be called directly by external clients;
  45. * however it is called by public macros in this file and thus must remain stable.
  46. *
  47. * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
  48. * @internal
  49. */
  50. #define U8_COUNT_TRAIL_BYTES(leadByte) \
  51. (U8_IS_LEAD(leadByte) ? \
  52. ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0)
  53. /**
  54. * Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence.
  55. * Returns 0 for 0..0xc1. Undefined for 0xf5..0xff.
  56. * leadByte might be evaluated multiple times.
  57. *
  58. * This is internal since it is not meant to be called directly by external clients;
  59. * however it is called by public macros in this file and thus must remain stable.
  60. *
  61. * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
  62. * @internal
  63. */
  64. #define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \
  65. (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0))
  66. /**
  67. * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
  68. *
  69. * This is internal since it is not meant to be called directly by external clients;
  70. * however it is called by public macros in this file and thus must remain stable.
  71. * @internal
  72. */
  73. #define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1)
  74. /**
  75. * Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1.
  76. * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence.
  77. * Lead byte E0..EF bits 3..0 are used as byte index,
  78. * first trail byte bits 7..5 are used as bit index into that byte.
  79. * @see U8_IS_VALID_LEAD3_AND_T1
  80. * @internal
  81. */
  82. #define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30"
  83. /**
  84. * Internal 3-byte UTF-8 validity check.
  85. * Non-zero if lead byte E0..EF and first trail byte 00..FF start a valid sequence.
  86. * @internal
  87. */
  88. #define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5)))
  89. /**
  90. * Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1.
  91. * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence.
  92. * First trail byte bits 7..4 are used as byte index,
  93. * lead byte F0..F4 bits 2..0 are used as bit index into that byte.
  94. * @see U8_IS_VALID_LEAD4_AND_T1
  95. * @internal
  96. */
  97. #define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00"
  98. /**
  99. * Internal 4-byte UTF-8 validity check.
  100. * Non-zero if lead byte F0..F4 and first trail byte 00..FF start a valid sequence.
  101. * @internal
  102. */
  103. #define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7)))
  104. /**
  105. * Function for handling "next code point" with error-checking.
  106. *
  107. * This is internal since it is not meant to be called directly by external clients;
  108. * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
  109. * file and thus must remain stable, and should not be hidden when other internal
  110. * functions are hidden (otherwise public macros would fail to compile).
  111. * @internal
  112. */
  113. U_STABLE UChar32 U_EXPORT2
  114. utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict);
  115. /**
  116. * Function for handling "append code point" with error-checking.
  117. *
  118. * This is internal since it is not meant to be called directly by external clients;
  119. * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
  120. * file and thus must remain stable, and should not be hidden when other internal
  121. * functions are hidden (otherwise public macros would fail to compile).
  122. * @internal
  123. */
  124. U_STABLE int32_t U_EXPORT2
  125. utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError);
  126. /**
  127. * Function for handling "previous code point" with error-checking.
  128. *
  129. * This is internal since it is not meant to be called directly by external clients;
  130. * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
  131. * file and thus must remain stable, and should not be hidden when other internal
  132. * functions are hidden (otherwise public macros would fail to compile).
  133. * @internal
  134. */
  135. U_STABLE UChar32 U_EXPORT2
  136. utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict);
  137. /**
  138. * Function for handling "skip backward one code point" with error-checking.
  139. *
  140. * This is internal since it is not meant to be called directly by external clients;
  141. * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
  142. * file and thus must remain stable, and should not be hidden when other internal
  143. * functions are hidden (otherwise public macros would fail to compile).
  144. * @internal
  145. */
  146. U_STABLE int32_t U_EXPORT2
  147. utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);
  148. /* single-code point definitions -------------------------------------------- */
  149. /**
  150. * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)?
  151. * @param c 8-bit code unit (byte)
  152. * @return TRUE or FALSE
  153. * @stable ICU 2.4
  154. */
  155. #define U8_IS_SINGLE(c) (((c)&0x80)==0)
  156. /**
  157. * Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4)
  158. * @param c 8-bit code unit (byte)
  159. * @return TRUE or FALSE
  160. * @stable ICU 2.4
  161. */
  162. #define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32)
  163. // 0x32=0xf4-0xc2
  164. /**
  165. * Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF)
  166. * @param c 8-bit code unit (byte)
  167. * @return TRUE or FALSE
  168. * @stable ICU 2.4
  169. */
  170. #define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40)
  171. /**
  172. * How many code units (bytes) are used for the UTF-8 encoding
  173. * of this Unicode code point?
  174. * @param c 32-bit code point
  175. * @return 1..4, or 0 if c is a surrogate or not a Unicode code point
  176. * @stable ICU 2.4
  177. */
  178. #define U8_LENGTH(c) \
  179. ((uint32_t)(c)<=0x7f ? 1 : \
  180. ((uint32_t)(c)<=0x7ff ? 2 : \
  181. ((uint32_t)(c)<=0xd7ff ? 3 : \
  182. ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \
  183. ((uint32_t)(c)<=0xffff ? 3 : 4)\
  184. ) \
  185. ) \
  186. ) \
  187. )
  188. /**
  189. * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff).
  190. * @return 4
  191. * @stable ICU 2.4
  192. */
  193. #define U8_MAX_LENGTH 4
  194. /**
  195. * Get a code point from a string at a random-access offset,
  196. * without changing the offset.
  197. * The offset may point to either the lead byte or one of the trail bytes
  198. * for a code point, in which case the macro will read all of the bytes
  199. * for the code point.
  200. * The result is undefined if the offset points to an illegal UTF-8
  201. * byte sequence.
  202. * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
  203. *
  204. * @param s const uint8_t * string
  205. * @param i string offset
  206. * @param c output UChar32 variable
  207. * @see U8_GET
  208. * @stable ICU 2.4
  209. */
  210. #define U8_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  211. int32_t _u8_get_unsafe_index=(int32_t)(i); \
  212. U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \
  213. U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \
  214. } UPRV_BLOCK_MACRO_END
  215. /**
  216. * Get a code point from a string at a random-access offset,
  217. * without changing the offset.
  218. * The offset may point to either the lead byte or one of the trail bytes
  219. * for a code point, in which case the macro will read all of the bytes
  220. * for the code point.
  221. *
  222. * The length can be negative for a NUL-terminated string.
  223. *
  224. * If the offset points to an illegal UTF-8 byte sequence, then
  225. * c is set to a negative value.
  226. * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
  227. *
  228. * @param s const uint8_t * string
  229. * @param start int32_t starting string offset
  230. * @param i int32_t string offset, must be start<=i<length
  231. * @param length int32_t string length
  232. * @param c output UChar32 variable, set to <0 in case of an error
  233. * @see U8_GET_UNSAFE
  234. * @stable ICU 2.4
  235. */
  236. #define U8_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
  237. int32_t _u8_get_index=(i); \
  238. U8_SET_CP_START(s, start, _u8_get_index); \
  239. U8_NEXT(s, _u8_get_index, length, c); \
  240. } UPRV_BLOCK_MACRO_END
  241. /**
  242. * Get a code point from a string at a random-access offset,
  243. * without changing the offset.
  244. * The offset may point to either the lead byte or one of the trail bytes
  245. * for a code point, in which case the macro will read all of the bytes
  246. * for the code point.
  247. *
  248. * The length can be negative for a NUL-terminated string.
  249. *
  250. * If the offset points to an illegal UTF-8 byte sequence, then
  251. * c is set to U+FFFD.
  252. * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT_OR_FFFD.
  253. *
  254. * This macro does not distinguish between a real U+FFFD in the text
  255. * and U+FFFD returned for an ill-formed sequence.
  256. * Use U8_GET() if that distinction is important.
  257. *
  258. * @param s const uint8_t * string
  259. * @param start int32_t starting string offset
  260. * @param i int32_t string offset, must be start<=i<length
  261. * @param length int32_t string length
  262. * @param c output UChar32 variable, set to U+FFFD in case of an error
  263. * @see U8_GET
  264. * @stable ICU 51
  265. */
  266. #define U8_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
  267. int32_t _u8_get_index=(i); \
  268. U8_SET_CP_START(s, start, _u8_get_index); \
  269. U8_NEXT_OR_FFFD(s, _u8_get_index, length, c); \
  270. } UPRV_BLOCK_MACRO_END
  271. /* definitions with forward iteration --------------------------------------- */
  272. /**
  273. * Get a code point from a string at a code point boundary offset,
  274. * and advance the offset to the next code point boundary.
  275. * (Post-incrementing forward iteration.)
  276. * "Unsafe" macro, assumes well-formed UTF-8.
  277. *
  278. * The offset may point to the lead byte of a multi-byte sequence,
  279. * in which case the macro will read the whole sequence.
  280. * The result is undefined if the offset points to a trail byte
  281. * or an illegal UTF-8 sequence.
  282. *
  283. * @param s const uint8_t * string
  284. * @param i string offset
  285. * @param c output UChar32 variable
  286. * @see U8_NEXT
  287. * @stable ICU 2.4
  288. */
  289. #define U8_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  290. (c)=(uint8_t)(s)[(i)++]; \
  291. if(!U8_IS_SINGLE(c)) { \
  292. if((c)<0xe0) { \
  293. (c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \
  294. } else if((c)<0xf0) { \
  295. /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \
  296. (c)=(UChar)(((c)<<12)|(((s)[i]&0x3f)<<6)|((s)[(i)+1]&0x3f)); \
  297. (i)+=2; \
  298. } else { \
  299. (c)=(((c)&7)<<18)|(((s)[i]&0x3f)<<12)|(((s)[(i)+1]&0x3f)<<6)|((s)[(i)+2]&0x3f); \
  300. (i)+=3; \
  301. } \
  302. } \
  303. } UPRV_BLOCK_MACRO_END
  304. /**
  305. * Get a code point from a string at a code point boundary offset,
  306. * and advance the offset to the next code point boundary.
  307. * (Post-incrementing forward iteration.)
  308. * "Safe" macro, checks for illegal sequences and for string boundaries.
  309. *
  310. * The length can be negative for a NUL-terminated string.
  311. *
  312. * The offset may point to the lead byte of a multi-byte sequence,
  313. * in which case the macro will read the whole sequence.
  314. * If the offset points to a trail byte or an illegal UTF-8 sequence, then
  315. * c is set to a negative value.
  316. *
  317. * @param s const uint8_t * string
  318. * @param i int32_t string offset, must be i<length
  319. * @param length int32_t string length
  320. * @param c output UChar32 variable, set to <0 in case of an error
  321. * @see U8_NEXT_UNSAFE
  322. * @stable ICU 2.4
  323. */
  324. #define U8_NEXT(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, U_SENTINEL)
  325. /**
  326. * Get a code point from a string at a code point boundary offset,
  327. * and advance the offset to the next code point boundary.
  328. * (Post-incrementing forward iteration.)
  329. * "Safe" macro, checks for illegal sequences and for string boundaries.
  330. *
  331. * The length can be negative for a NUL-terminated string.
  332. *
  333. * The offset may point to the lead byte of a multi-byte sequence,
  334. * in which case the macro will read the whole sequence.
  335. * If the offset points to a trail byte or an illegal UTF-8 sequence, then
  336. * c is set to U+FFFD.
  337. *
  338. * This macro does not distinguish between a real U+FFFD in the text
  339. * and U+FFFD returned for an ill-formed sequence.
  340. * Use U8_NEXT() if that distinction is important.
  341. *
  342. * @param s const uint8_t * string
  343. * @param i int32_t string offset, must be i<length
  344. * @param length int32_t string length
  345. * @param c output UChar32 variable, set to U+FFFD in case of an error
  346. * @see U8_NEXT
  347. * @stable ICU 51
  348. */
  349. #define U8_NEXT_OR_FFFD(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, 0xfffd)
  350. /** @internal */
  351. #define U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, sub) UPRV_BLOCK_MACRO_BEGIN { \
  352. (c)=(uint8_t)(s)[(i)++]; \
  353. if(!U8_IS_SINGLE(c)) { \
  354. uint8_t __t = 0; \
  355. if((i)!=(length) && \
  356. /* fetch/validate/assemble all but last trail byte */ \
  357. ((c)>=0xe0 ? \
  358. ((c)<0xf0 ? /* U+0800..U+FFFF except surrogates */ \
  359. U8_LEAD3_T1_BITS[(c)&=0xf]&(1<<((__t=(s)[i])>>5)) && \
  360. (__t&=0x3f, 1) \
  361. : /* U+10000..U+10FFFF */ \
  362. ((c)-=0xf0)<=4 && \
  363. U8_LEAD4_T1_BITS[(__t=(s)[i])>>4]&(1<<(c)) && \
  364. ((c)=((c)<<6)|(__t&0x3f), ++(i)!=(length)) && \
  365. (__t=(s)[i]-0x80)<=0x3f) && \
  366. /* valid second-to-last trail byte */ \
  367. ((c)=((c)<<6)|__t, ++(i)!=(length)) \
  368. : /* U+0080..U+07FF */ \
  369. (c)>=0xc2 && ((c)&=0x1f, 1)) && \
  370. /* last trail byte */ \
  371. (__t=(s)[i]-0x80)<=0x3f && \
  372. ((c)=((c)<<6)|__t, ++(i), 1)) { \
  373. } else { \
  374. (c)=(sub); /* ill-formed*/ \
  375. } \
  376. } \
  377. } UPRV_BLOCK_MACRO_END
  378. /**
  379. * Append a code point to a string, overwriting 1 to 4 bytes.
  380. * The offset points to the current end of the string contents
  381. * and is advanced (post-increment).
  382. * "Unsafe" macro, assumes a valid code point and sufficient space in the string.
  383. * Otherwise, the result is undefined.
  384. *
  385. * @param s const uint8_t * string buffer
  386. * @param i string offset
  387. * @param c code point to append
  388. * @see U8_APPEND
  389. * @stable ICU 2.4
  390. */
  391. #define U8_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  392. uint32_t __uc=(c); \
  393. if(__uc<=0x7f) { \
  394. (s)[(i)++]=(uint8_t)__uc; \
  395. } else { \
  396. if(__uc<=0x7ff) { \
  397. (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \
  398. } else { \
  399. if(__uc<=0xffff) { \
  400. (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \
  401. } else { \
  402. (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \
  403. (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \
  404. } \
  405. (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
  406. } \
  407. (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
  408. } \
  409. } UPRV_BLOCK_MACRO_END
  410. /**
  411. * Append a code point to a string, overwriting 1 to 4 bytes.
  412. * The offset points to the current end of the string contents
  413. * and is advanced (post-increment).
  414. * "Safe" macro, checks for a valid code point.
  415. * If a non-ASCII code point is written, checks for sufficient space in the string.
  416. * If the code point is not valid or trail bytes do not fit,
  417. * then isError is set to TRUE.
  418. *
  419. * @param s const uint8_t * string buffer
  420. * @param i int32_t string offset, must be i<capacity
  421. * @param capacity int32_t size of the string buffer
  422. * @param c UChar32 code point to append
  423. * @param isError output UBool set to TRUE if an error occurs, otherwise not modified
  424. * @see U8_APPEND_UNSAFE
  425. * @stable ICU 2.4
  426. */
  427. #define U8_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \
  428. uint32_t __uc=(c); \
  429. if(__uc<=0x7f) { \
  430. (s)[(i)++]=(uint8_t)__uc; \
  431. } else if(__uc<=0x7ff && (i)+1<(capacity)) { \
  432. (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \
  433. (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
  434. } else if((__uc<=0xd7ff || (0xe000<=__uc && __uc<=0xffff)) && (i)+2<(capacity)) { \
  435. (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \
  436. (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
  437. (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
  438. } else if(0xffff<__uc && __uc<=0x10ffff && (i)+3<(capacity)) { \
  439. (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \
  440. (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \
  441. (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
  442. (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
  443. } else { \
  444. (isError)=TRUE; \
  445. } \
  446. } UPRV_BLOCK_MACRO_END
  447. /**
  448. * Advance the string offset from one code point boundary to the next.
  449. * (Post-incrementing iteration.)
  450. * "Unsafe" macro, assumes well-formed UTF-8.
  451. *
  452. * @param s const uint8_t * string
  453. * @param i string offset
  454. * @see U8_FWD_1
  455. * @stable ICU 2.4
  456. */
  457. #define U8_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
  458. (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \
  459. } UPRV_BLOCK_MACRO_END
  460. /**
  461. * Advance the string offset from one code point boundary to the next.
  462. * (Post-incrementing iteration.)
  463. * "Safe" macro, checks for illegal sequences and for string boundaries.
  464. *
  465. * The length can be negative for a NUL-terminated string.
  466. *
  467. * @param s const uint8_t * string
  468. * @param i int32_t string offset, must be i<length
  469. * @param length int32_t string length
  470. * @see U8_FWD_1_UNSAFE
  471. * @stable ICU 2.4
  472. */
  473. #define U8_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \
  474. uint8_t __b=(s)[(i)++]; \
  475. if(U8_IS_LEAD(__b) && (i)!=(length)) { \
  476. uint8_t __t1=(s)[i]; \
  477. if((0xe0<=__b && __b<0xf0)) { \
  478. if(U8_IS_VALID_LEAD3_AND_T1(__b, __t1) && \
  479. ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \
  480. ++(i); \
  481. } \
  482. } else if(__b<0xe0) { \
  483. if(U8_IS_TRAIL(__t1)) { \
  484. ++(i); \
  485. } \
  486. } else /* c>=0xf0 */ { \
  487. if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \
  488. ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \
  489. ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \
  490. ++(i); \
  491. } \
  492. } \
  493. } \
  494. } UPRV_BLOCK_MACRO_END
  495. /**
  496. * Advance the string offset from one code point boundary to the n-th next one,
  497. * i.e., move forward by n code points.
  498. * (Post-incrementing iteration.)
  499. * "Unsafe" macro, assumes well-formed UTF-8.
  500. *
  501. * @param s const uint8_t * string
  502. * @param i string offset
  503. * @param n number of code points to skip
  504. * @see U8_FWD_N
  505. * @stable ICU 2.4
  506. */
  507. #define U8_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
  508. int32_t __N=(n); \
  509. while(__N>0) { \
  510. U8_FWD_1_UNSAFE(s, i); \
  511. --__N; \
  512. } \
  513. } UPRV_BLOCK_MACRO_END
  514. /**
  515. * Advance the string offset from one code point boundary to the n-th next one,
  516. * i.e., move forward by n code points.
  517. * (Post-incrementing iteration.)
  518. * "Safe" macro, checks for illegal sequences and for string boundaries.
  519. *
  520. * The length can be negative for a NUL-terminated string.
  521. *
  522. * @param s const uint8_t * string
  523. * @param i int32_t string offset, must be i<length
  524. * @param length int32_t string length
  525. * @param n number of code points to skip
  526. * @see U8_FWD_N_UNSAFE
  527. * @stable ICU 2.4
  528. */
  529. #define U8_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \
  530. int32_t __N=(n); \
  531. while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \
  532. U8_FWD_1(s, i, length); \
  533. --__N; \
  534. } \
  535. } UPRV_BLOCK_MACRO_END
  536. /**
  537. * Adjust a random-access offset to a code point boundary
  538. * at the start of a code point.
  539. * If the offset points to a UTF-8 trail byte,
  540. * then the offset is moved backward to the corresponding lead byte.
  541. * Otherwise, it is not modified.
  542. * "Unsafe" macro, assumes well-formed UTF-8.
  543. *
  544. * @param s const uint8_t * string
  545. * @param i string offset
  546. * @see U8_SET_CP_START
  547. * @stable ICU 2.4
  548. */
  549. #define U8_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
  550. while(U8_IS_TRAIL((s)[i])) { --(i); } \
  551. } UPRV_BLOCK_MACRO_END
  552. /**
  553. * Adjust a random-access offset to a code point boundary
  554. * at the start of a code point.
  555. * If the offset points to a UTF-8 trail byte,
  556. * then the offset is moved backward to the corresponding lead byte.
  557. * Otherwise, it is not modified.
  558. *
  559. * "Safe" macro, checks for illegal sequences and for string boundaries.
  560. * Unlike U8_TRUNCATE_IF_INCOMPLETE(), this macro always reads s[i].
  561. *
  562. * @param s const uint8_t * string
  563. * @param start int32_t starting string offset (usually 0)
  564. * @param i int32_t string offset, must be start<=i
  565. * @see U8_SET_CP_START_UNSAFE
  566. * @see U8_TRUNCATE_IF_INCOMPLETE
  567. * @stable ICU 2.4
  568. */
  569. #define U8_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
  570. if(U8_IS_TRAIL((s)[(i)])) { \
  571. (i)=utf8_back1SafeBody(s, start, (i)); \
  572. } \
  573. } UPRV_BLOCK_MACRO_END
  574. /**
  575. * If the string ends with a UTF-8 byte sequence that is valid so far
  576. * but incomplete, then reduce the length of the string to end before
  577. * the lead byte of that incomplete sequence.
  578. * For example, if the string ends with E1 80, the length is reduced by 2.
  579. *
  580. * In all other cases (the string ends with a complete sequence, or it is not
  581. * possible for any further trail byte to extend the trailing sequence)
  582. * the length remains unchanged.
  583. *
  584. * Useful for processing text split across multiple buffers
  585. * (save the incomplete sequence for later)
  586. * and for optimizing iteration
  587. * (check for string length only once per character).
  588. *
  589. * "Safe" macro, checks for illegal sequences and for string boundaries.
  590. * Unlike U8_SET_CP_START(), this macro never reads s[length].
  591. *
  592. * (In UTF-16, simply check for U16_IS_LEAD(last code unit).)
  593. *
  594. * @param s const uint8_t * string
  595. * @param start int32_t starting string offset (usually 0)
  596. * @param length int32_t string length (usually start<=length)
  597. * @see U8_SET_CP_START
  598. * @stable ICU 61
  599. */
  600. #define U8_TRUNCATE_IF_INCOMPLETE(s, start, length) UPRV_BLOCK_MACRO_BEGIN { \
  601. if((length)>(start)) { \
  602. uint8_t __b1=s[(length)-1]; \
  603. if(U8_IS_SINGLE(__b1)) { \
  604. /* common ASCII character */ \
  605. } else if(U8_IS_LEAD(__b1)) { \
  606. --(length); \
  607. } else if(U8_IS_TRAIL(__b1) && ((length)-2)>=(start)) { \
  608. uint8_t __b2=s[(length)-2]; \
  609. if(0xe0<=__b2 && __b2<=0xf4) { \
  610. if(__b2<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(__b2, __b1) : \
  611. U8_IS_VALID_LEAD4_AND_T1(__b2, __b1)) { \
  612. (length)-=2; \
  613. } \
  614. } else if(U8_IS_TRAIL(__b2) && ((length)-3)>=(start)) { \
  615. uint8_t __b3=s[(length)-3]; \
  616. if(0xf0<=__b3 && __b3<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(__b3, __b2)) { \
  617. (length)-=3; \
  618. } \
  619. } \
  620. } \
  621. } \
  622. } UPRV_BLOCK_MACRO_END
  623. /* definitions with backward iteration -------------------------------------- */
  624. /**
  625. * Move the string offset from one code point boundary to the previous one
  626. * and get the code point between them.
  627. * (Pre-decrementing backward iteration.)
  628. * "Unsafe" macro, assumes well-formed UTF-8.
  629. *
  630. * The input offset may be the same as the string length.
  631. * If the offset is behind a multi-byte sequence, then the macro will read
  632. * the whole sequence.
  633. * If the offset is behind a lead byte, then that itself
  634. * will be returned as the code point.
  635. * The result is undefined if the offset is behind an illegal UTF-8 sequence.
  636. *
  637. * @param s const uint8_t * string
  638. * @param i string offset
  639. * @param c output UChar32 variable
  640. * @see U8_PREV
  641. * @stable ICU 2.4
  642. */
  643. #define U8_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  644. (c)=(uint8_t)(s)[--(i)]; \
  645. if(U8_IS_TRAIL(c)) { \
  646. uint8_t __b, __count=1, __shift=6; \
  647. \
  648. /* c is a trail byte */ \
  649. (c)&=0x3f; \
  650. for(;;) { \
  651. __b=(s)[--(i)]; \
  652. if(__b>=0xc0) { \
  653. U8_MASK_LEAD_BYTE(__b, __count); \
  654. (c)|=(UChar32)__b<<__shift; \
  655. break; \
  656. } else { \
  657. (c)|=(UChar32)(__b&0x3f)<<__shift; \
  658. ++__count; \
  659. __shift+=6; \
  660. } \
  661. } \
  662. } \
  663. } UPRV_BLOCK_MACRO_END
  664. /**
  665. * Move the string offset from one code point boundary to the previous one
  666. * and get the code point between them.
  667. * (Pre-decrementing backward iteration.)
  668. * "Safe" macro, checks for illegal sequences and for string boundaries.
  669. *
  670. * The input offset may be the same as the string length.
  671. * If the offset is behind a multi-byte sequence, then the macro will read
  672. * the whole sequence.
  673. * If the offset is behind a lead byte, then that itself
  674. * will be returned as the code point.
  675. * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value.
  676. *
  677. * @param s const uint8_t * string
  678. * @param start int32_t starting string offset (usually 0)
  679. * @param i int32_t string offset, must be start<i
  680. * @param c output UChar32 variable, set to <0 in case of an error
  681. * @see U8_PREV_UNSAFE
  682. * @stable ICU 2.4
  683. */
  684. #define U8_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  685. (c)=(uint8_t)(s)[--(i)]; \
  686. if(!U8_IS_SINGLE(c)) { \
  687. (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \
  688. } \
  689. } UPRV_BLOCK_MACRO_END
  690. /**
  691. * Move the string offset from one code point boundary to the previous one
  692. * and get the code point between them.
  693. * (Pre-decrementing backward iteration.)
  694. * "Safe" macro, checks for illegal sequences and for string boundaries.
  695. *
  696. * The input offset may be the same as the string length.
  697. * If the offset is behind a multi-byte sequence, then the macro will read
  698. * the whole sequence.
  699. * If the offset is behind a lead byte, then that itself
  700. * will be returned as the code point.
  701. * If the offset is behind an illegal UTF-8 sequence, then c is set to U+FFFD.
  702. *
  703. * This macro does not distinguish between a real U+FFFD in the text
  704. * and U+FFFD returned for an ill-formed sequence.
  705. * Use U8_PREV() if that distinction is important.
  706. *
  707. * @param s const uint8_t * string
  708. * @param start int32_t starting string offset (usually 0)
  709. * @param i int32_t string offset, must be start<i
  710. * @param c output UChar32 variable, set to U+FFFD in case of an error
  711. * @see U8_PREV
  712. * @stable ICU 51
  713. */
  714. #define U8_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
  715. (c)=(uint8_t)(s)[--(i)]; \
  716. if(!U8_IS_SINGLE(c)) { \
  717. (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -3); \
  718. } \
  719. } UPRV_BLOCK_MACRO_END
  720. /**
  721. * Move the string offset from one code point boundary to the previous one.
  722. * (Pre-decrementing backward iteration.)
  723. * The input offset may be the same as the string length.
  724. * "Unsafe" macro, assumes well-formed UTF-8.
  725. *
  726. * @param s const uint8_t * string
  727. * @param i string offset
  728. * @see U8_BACK_1
  729. * @stable ICU 2.4
  730. */
  731. #define U8_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
  732. while(U8_IS_TRAIL((s)[--(i)])) {} \
  733. } UPRV_BLOCK_MACRO_END
  734. /**
  735. * Move the string offset from one code point boundary to the previous one.
  736. * (Pre-decrementing backward iteration.)
  737. * The input offset may be the same as the string length.
  738. * "Safe" macro, checks for illegal sequences and for string boundaries.
  739. *
  740. * @param s const uint8_t * string
  741. * @param start int32_t starting string offset (usually 0)
  742. * @param i int32_t string offset, must be start<i
  743. * @see U8_BACK_1_UNSAFE
  744. * @stable ICU 2.4
  745. */
  746. #define U8_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
  747. if(U8_IS_TRAIL((s)[--(i)])) { \
  748. (i)=utf8_back1SafeBody(s, start, (i)); \
  749. } \
  750. } UPRV_BLOCK_MACRO_END
  751. /**
  752. * Move the string offset from one code point boundary to the n-th one before it,
  753. * i.e., move backward by n code points.
  754. * (Pre-decrementing backward iteration.)
  755. * The input offset may be the same as the string length.
  756. * "Unsafe" macro, assumes well-formed UTF-8.
  757. *
  758. * @param s const uint8_t * string
  759. * @param i string offset
  760. * @param n number of code points to skip
  761. * @see U8_BACK_N
  762. * @stable ICU 2.4
  763. */
  764. #define U8_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
  765. int32_t __N=(n); \
  766. while(__N>0) { \
  767. U8_BACK_1_UNSAFE(s, i); \
  768. --__N; \
  769. } \
  770. } UPRV_BLOCK_MACRO_END
  771. /**
  772. * Move the string offset from one code point boundary to the n-th one before it,
  773. * i.e., move backward by n code points.
  774. * (Pre-decrementing backward iteration.)
  775. * The input offset may be the same as the string length.
  776. * "Safe" macro, checks for illegal sequences and for string boundaries.
  777. *
  778. * @param s const uint8_t * string
  779. * @param start int32_t index of the start of the string
  780. * @param i int32_t string offset, must be start<i
  781. * @param n number of code points to skip
  782. * @see U8_BACK_N_UNSAFE
  783. * @stable ICU 2.4
  784. */
  785. #define U8_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \
  786. int32_t __N=(n); \
  787. while(__N>0 && (i)>(start)) { \
  788. U8_BACK_1(s, start, i); \
  789. --__N; \
  790. } \
  791. } UPRV_BLOCK_MACRO_END
  792. /**
  793. * Adjust a random-access offset to a code point boundary after a code point.
  794. * If the offset is behind a partial multi-byte sequence,
  795. * then the offset is incremented to behind the whole sequence.
  796. * Otherwise, it is not modified.
  797. * The input offset may be the same as the string length.
  798. * "Unsafe" macro, assumes well-formed UTF-8.
  799. *
  800. * @param s const uint8_t * string
  801. * @param i string offset
  802. * @see U8_SET_CP_LIMIT
  803. * @stable ICU 2.4
  804. */
  805. #define U8_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
  806. U8_BACK_1_UNSAFE(s, i); \
  807. U8_FWD_1_UNSAFE(s, i); \
  808. } UPRV_BLOCK_MACRO_END
  809. /**
  810. * Adjust a random-access offset to a code point boundary after a code point.
  811. * If the offset is behind a partial multi-byte sequence,
  812. * then the offset is incremented to behind the whole sequence.
  813. * Otherwise, it is not modified.
  814. * The input offset may be the same as the string length.
  815. * "Safe" macro, checks for illegal sequences and for string boundaries.
  816. *
  817. * The length can be negative for a NUL-terminated string.
  818. *
  819. * @param s const uint8_t * string
  820. * @param start int32_t starting string offset (usually 0)
  821. * @param i int32_t string offset, must be start<=i<=length
  822. * @param length int32_t string length
  823. * @see U8_SET_CP_LIMIT_UNSAFE
  824. * @stable ICU 2.4
  825. */
  826. #define U8_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \
  827. if((start)<(i) && ((i)<(length) || (length)<0)) { \
  828. U8_BACK_1(s, start, i); \
  829. U8_FWD_1(s, i, length); \
  830. } \
  831. } UPRV_BLOCK_MACRO_END
  832. #endif