unicode_iterator.hpp 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790
  1. /*
  2. *
  3. * Copyright (c) 2004
  4. * John Maddock
  5. *
  6. * Use, modification and distribution are subject to the
  7. * Boost Software License, Version 1.0. (See accompanying file
  8. * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  9. *
  10. */
  11. /*
  12. * LOCATION: see http://www.boost.org for most recent version.
  13. * FILE unicode_iterator.hpp
  14. * VERSION see <boost/version.hpp>
  15. * DESCRIPTION: Iterator adapters for converting between different Unicode encodings.
  16. */
  17. /****************************************************************************
  18. Contents:
  19. ~~~~~~~~~
  20. 1) Read Only, Input Adapters:
  21. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  22. template <class BaseIterator, class U8Type = ::boost::uint8_t>
  23. class u32_to_u8_iterator;
  24. Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8.
  25. template <class BaseIterator, class U32Type = ::boost::uint32_t>
  26. class u8_to_u32_iterator;
  27. Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32.
  28. template <class BaseIterator, class U16Type = ::boost::uint16_t>
  29. class u32_to_u16_iterator;
  30. Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16.
  31. template <class BaseIterator, class U32Type = ::boost::uint32_t>
  32. class u16_to_u32_iterator;
  33. Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32.
  34. 2) Single pass output iterator adapters:
  35. template <class BaseIterator>
  36. class utf8_output_iterator;
  37. Accepts UTF-32 code points and forwards them on as UTF-8 code points.
  38. template <class BaseIterator>
  39. class utf16_output_iterator;
  40. Accepts UTF-32 code points and forwards them on as UTF-16 code points.
  41. ****************************************************************************/
  42. #ifndef BOOST_REGEX_V4_UNICODE_ITERATOR_HPP
  43. #define BOOST_REGEX_V4_UNICODE_ITERATOR_HPP
  44. #include <boost/cstdint.hpp>
  45. #include <boost/regex/config.hpp>
  46. #include <boost/iterator/iterator_facade.hpp>
  47. #include <boost/static_assert.hpp>
  48. #include <boost/throw_exception.hpp>
  49. #include <stdexcept>
  50. #ifndef BOOST_NO_STD_LOCALE
  51. #include <sstream>
  52. #include <ios>
  53. #endif
  54. #include <limits.h> // CHAR_BIT
  55. #ifdef BOOST_REGEX_CXX03
  56. #else
  57. #endif
  58. namespace boost{
  59. namespace detail{
  60. static const ::boost::uint16_t high_surrogate_base = 0xD7C0u;
  61. static const ::boost::uint16_t low_surrogate_base = 0xDC00u;
  62. static const ::boost::uint32_t ten_bit_mask = 0x3FFu;
  63. inline bool is_high_surrogate(::boost::uint16_t v)
  64. {
  65. return (v & 0xFFFFFC00u) == 0xd800u;
  66. }
  67. inline bool is_low_surrogate(::boost::uint16_t v)
  68. {
  69. return (v & 0xFFFFFC00u) == 0xdc00u;
  70. }
  71. template <class T>
  72. inline bool is_surrogate(T v)
  73. {
  74. return (v & 0xFFFFF800u) == 0xd800;
  75. }
  76. inline unsigned utf8_byte_count(boost::uint8_t c)
  77. {
  78. // if the most significant bit with a zero in it is in position
  79. // 8-N then there are N bytes in this UTF-8 sequence:
  80. boost::uint8_t mask = 0x80u;
  81. unsigned result = 0;
  82. while(c & mask)
  83. {
  84. ++result;
  85. mask >>= 1;
  86. }
  87. return (result == 0) ? 1 : ((result > 4) ? 4 : result);
  88. }
  89. inline unsigned utf8_trailing_byte_count(boost::uint8_t c)
  90. {
  91. return utf8_byte_count(c) - 1;
  92. }
  93. #ifdef BOOST_MSVC
  94. #pragma warning(push)
  95. #pragma warning(disable:4100)
  96. #endif
  97. #ifndef BOOST_NO_EXCEPTIONS
  98. BOOST_NORETURN
  99. #endif
  100. inline void invalid_utf32_code_point(::boost::uint32_t val)
  101. {
  102. #ifndef BOOST_NO_STD_LOCALE
  103. std::stringstream ss;
  104. ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence";
  105. std::out_of_range e(ss.str());
  106. #else
  107. std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence");
  108. #endif
  109. boost::throw_exception(e);
  110. }
  111. #ifdef BOOST_MSVC
  112. #pragma warning(pop)
  113. #endif
  114. } // namespace detail
  115. template <class BaseIterator, class U16Type = ::boost::uint16_t>
  116. class u32_to_u16_iterator
  117. : public boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type>
  118. {
  119. typedef boost::iterator_facade<u32_to_u16_iterator<BaseIterator, U16Type>, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type;
  120. #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
  121. typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
  122. BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
  123. BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16);
  124. #endif
  125. public:
  126. typename base_type::reference
  127. dereference()const
  128. {
  129. if(m_current == 2)
  130. extract_current();
  131. return m_values[m_current];
  132. }
  133. bool equal(const u32_to_u16_iterator& that)const
  134. {
  135. if(m_position == that.m_position)
  136. {
  137. // Both m_currents must be equal, or both even
  138. // this is the same as saying their sum must be even:
  139. return (m_current + that.m_current) & 1u ? false : true;
  140. }
  141. return false;
  142. }
  143. void increment()
  144. {
  145. // if we have a pending read then read now, so that we know whether
  146. // to skip a position, or move to a low-surrogate:
  147. if(m_current == 2)
  148. {
  149. // pending read:
  150. extract_current();
  151. }
  152. // move to the next surrogate position:
  153. ++m_current;
  154. // if we've reached the end skip a position:
  155. if(m_values[m_current] == 0)
  156. {
  157. m_current = 2;
  158. ++m_position;
  159. }
  160. }
  161. void decrement()
  162. {
  163. if(m_current != 1)
  164. {
  165. // decrementing an iterator always leads to a valid position:
  166. --m_position;
  167. extract_current();
  168. m_current = m_values[1] ? 1 : 0;
  169. }
  170. else
  171. {
  172. m_current = 0;
  173. }
  174. }
  175. BaseIterator base()const
  176. {
  177. return m_position;
  178. }
  179. // construct:
  180. u32_to_u16_iterator() : m_position(), m_current(0)
  181. {
  182. m_values[0] = 0;
  183. m_values[1] = 0;
  184. m_values[2] = 0;
  185. }
  186. u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2)
  187. {
  188. m_values[0] = 0;
  189. m_values[1] = 0;
  190. m_values[2] = 0;
  191. }
  192. private:
  193. void extract_current()const
  194. {
  195. // begin by checking for a code point out of range:
  196. ::boost::uint32_t v = *m_position;
  197. if(v >= 0x10000u)
  198. {
  199. if(v > 0x10FFFFu)
  200. detail::invalid_utf32_code_point(*m_position);
  201. // split into two surrogates:
  202. m_values[0] = static_cast<U16Type>(v >> 10) + detail::high_surrogate_base;
  203. m_values[1] = static_cast<U16Type>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
  204. m_current = 0;
  205. BOOST_REGEX_ASSERT(detail::is_high_surrogate(m_values[0]));
  206. BOOST_REGEX_ASSERT(detail::is_low_surrogate(m_values[1]));
  207. }
  208. else
  209. {
  210. // 16-bit code point:
  211. m_values[0] = static_cast<U16Type>(*m_position);
  212. m_values[1] = 0;
  213. m_current = 0;
  214. // value must not be a surrogate:
  215. if(detail::is_surrogate(m_values[0]))
  216. detail::invalid_utf32_code_point(*m_position);
  217. }
  218. }
  219. BaseIterator m_position;
  220. mutable U16Type m_values[3];
  221. mutable unsigned m_current;
  222. };
  223. template <class BaseIterator, class U32Type = ::boost::uint32_t>
  224. class u16_to_u32_iterator
  225. : public boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
  226. {
  227. typedef boost::iterator_facade<u16_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
  228. // special values for pending iterator reads:
  229. BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
  230. #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
  231. typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
  232. BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16);
  233. BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
  234. #endif
  235. public:
  236. typename base_type::reference
  237. dereference()const
  238. {
  239. if(m_value == pending_read)
  240. extract_current();
  241. return m_value;
  242. }
  243. bool equal(const u16_to_u32_iterator& that)const
  244. {
  245. return m_position == that.m_position;
  246. }
  247. void increment()
  248. {
  249. // skip high surrogate first if there is one:
  250. if(detail::is_high_surrogate(*m_position)) ++m_position;
  251. ++m_position;
  252. m_value = pending_read;
  253. }
  254. void decrement()
  255. {
  256. --m_position;
  257. // if we have a low surrogate then go back one more:
  258. if(detail::is_low_surrogate(*m_position))
  259. --m_position;
  260. m_value = pending_read;
  261. }
  262. BaseIterator base()const
  263. {
  264. return m_position;
  265. }
  266. // construct:
  267. u16_to_u32_iterator() : m_position()
  268. {
  269. m_value = pending_read;
  270. }
  271. u16_to_u32_iterator(BaseIterator b) : m_position(b)
  272. {
  273. m_value = pending_read;
  274. }
  275. //
  276. // Range checked version:
  277. //
  278. u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
  279. {
  280. m_value = pending_read;
  281. //
  282. // The range must not start with a low surrogate, or end in a high surrogate,
  283. // otherwise we run the risk of running outside the underlying input range.
  284. // Likewise b must not be located at a low surrogate.
  285. //
  286. boost::uint16_t val;
  287. if(start != end)
  288. {
  289. if((b != start) && (b != end))
  290. {
  291. val = *b;
  292. if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
  293. invalid_code_point(val);
  294. }
  295. val = *start;
  296. if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u))
  297. invalid_code_point(val);
  298. val = *--end;
  299. if(detail::is_high_surrogate(val))
  300. invalid_code_point(val);
  301. }
  302. }
  303. private:
  304. static void invalid_code_point(::boost::uint16_t val)
  305. {
  306. #ifndef BOOST_NO_STD_LOCALE
  307. std::stringstream ss;
  308. ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence";
  309. std::out_of_range e(ss.str());
  310. #else
  311. std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence");
  312. #endif
  313. boost::throw_exception(e);
  314. }
  315. void extract_current()const
  316. {
  317. m_value = static_cast<U32Type>(static_cast< ::boost::uint16_t>(*m_position));
  318. // if the last value is a high surrogate then adjust m_position and m_value as needed:
  319. if(detail::is_high_surrogate(*m_position))
  320. {
  321. // precondition; next value must have be a low-surrogate:
  322. BaseIterator next(m_position);
  323. ::boost::uint16_t t = *++next;
  324. if((t & 0xFC00u) != 0xDC00u)
  325. invalid_code_point(t);
  326. m_value = (m_value - detail::high_surrogate_base) << 10;
  327. m_value |= (static_cast<U32Type>(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask);
  328. }
  329. // postcondition; result must not be a surrogate:
  330. if(detail::is_surrogate(m_value))
  331. invalid_code_point(static_cast< ::boost::uint16_t>(m_value));
  332. }
  333. BaseIterator m_position;
  334. mutable U32Type m_value;
  335. };
  336. template <class BaseIterator, class U8Type = ::boost::uint8_t>
  337. class u32_to_u8_iterator
  338. : public boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type>
  339. {
  340. typedef boost::iterator_facade<u32_to_u8_iterator<BaseIterator, U8Type>, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type;
  341. #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
  342. typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
  343. BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32);
  344. BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8);
  345. #endif
  346. public:
  347. typename base_type::reference
  348. dereference()const
  349. {
  350. if(m_current == 4)
  351. extract_current();
  352. return m_values[m_current];
  353. }
  354. bool equal(const u32_to_u8_iterator& that)const
  355. {
  356. if(m_position == that.m_position)
  357. {
  358. // either the m_current's must be equal, or one must be 0 and
  359. // the other 4: which means neither must have bits 1 or 2 set:
  360. return (m_current == that.m_current)
  361. || (((m_current | that.m_current) & 3) == 0);
  362. }
  363. return false;
  364. }
  365. void increment()
  366. {
  367. // if we have a pending read then read now, so that we know whether
  368. // to skip a position, or move to a low-surrogate:
  369. if(m_current == 4)
  370. {
  371. // pending read:
  372. extract_current();
  373. }
  374. // move to the next surrogate position:
  375. ++m_current;
  376. // if we've reached the end skip a position:
  377. if(m_values[m_current] == 0)
  378. {
  379. m_current = 4;
  380. ++m_position;
  381. }
  382. }
  383. void decrement()
  384. {
  385. if((m_current & 3) == 0)
  386. {
  387. --m_position;
  388. extract_current();
  389. m_current = 3;
  390. while(m_current && (m_values[m_current] == 0))
  391. --m_current;
  392. }
  393. else
  394. --m_current;
  395. }
  396. BaseIterator base()const
  397. {
  398. return m_position;
  399. }
  400. // construct:
  401. u32_to_u8_iterator() : m_position(), m_current(0)
  402. {
  403. m_values[0] = 0;
  404. m_values[1] = 0;
  405. m_values[2] = 0;
  406. m_values[3] = 0;
  407. m_values[4] = 0;
  408. }
  409. u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4)
  410. {
  411. m_values[0] = 0;
  412. m_values[1] = 0;
  413. m_values[2] = 0;
  414. m_values[3] = 0;
  415. m_values[4] = 0;
  416. }
  417. private:
  418. void extract_current()const
  419. {
  420. boost::uint32_t c = *m_position;
  421. if(c > 0x10FFFFu)
  422. detail::invalid_utf32_code_point(c);
  423. if(c < 0x80u)
  424. {
  425. m_values[0] = static_cast<unsigned char>(c);
  426. m_values[1] = static_cast<unsigned char>(0u);
  427. m_values[2] = static_cast<unsigned char>(0u);
  428. m_values[3] = static_cast<unsigned char>(0u);
  429. }
  430. else if(c < 0x800u)
  431. {
  432. m_values[0] = static_cast<unsigned char>(0xC0u + (c >> 6));
  433. m_values[1] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  434. m_values[2] = static_cast<unsigned char>(0u);
  435. m_values[3] = static_cast<unsigned char>(0u);
  436. }
  437. else if(c < 0x10000u)
  438. {
  439. m_values[0] = static_cast<unsigned char>(0xE0u + (c >> 12));
  440. m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
  441. m_values[2] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  442. m_values[3] = static_cast<unsigned char>(0u);
  443. }
  444. else
  445. {
  446. m_values[0] = static_cast<unsigned char>(0xF0u + (c >> 18));
  447. m_values[1] = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
  448. m_values[2] = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
  449. m_values[3] = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  450. }
  451. m_current= 0;
  452. }
  453. BaseIterator m_position;
  454. mutable U8Type m_values[5];
  455. mutable unsigned m_current;
  456. };
  457. template <class BaseIterator, class U32Type = ::boost::uint32_t>
  458. class u8_to_u32_iterator
  459. : public boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type>
  460. {
  461. typedef boost::iterator_facade<u8_to_u32_iterator<BaseIterator, U32Type>, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type;
  462. // special values for pending iterator reads:
  463. BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu);
  464. #if !defined(BOOST_NO_STD_ITERATOR_TRAITS)
  465. typedef typename std::iterator_traits<BaseIterator>::value_type base_value_type;
  466. BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8);
  467. BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32);
  468. #endif
  469. public:
  470. typename base_type::reference
  471. dereference()const
  472. {
  473. if(m_value == pending_read)
  474. extract_current();
  475. return m_value;
  476. }
  477. bool equal(const u8_to_u32_iterator& that)const
  478. {
  479. return m_position == that.m_position;
  480. }
  481. void increment()
  482. {
  483. // We must not start with a continuation character:
  484. if((static_cast<boost::uint8_t>(*m_position) & 0xC0) == 0x80)
  485. invalid_sequence();
  486. // skip high surrogate first if there is one:
  487. unsigned c = detail::utf8_byte_count(*m_position);
  488. if(m_value == pending_read)
  489. {
  490. // Since we haven't read in a value, we need to validate the code points:
  491. for(unsigned i = 0; i < c; ++i)
  492. {
  493. ++m_position;
  494. // We must have a continuation byte:
  495. if((i != c - 1) && ((static_cast<boost::uint8_t>(*m_position) & 0xC0) != 0x80))
  496. invalid_sequence();
  497. }
  498. }
  499. else
  500. {
  501. std::advance(m_position, c);
  502. }
  503. m_value = pending_read;
  504. }
  505. void decrement()
  506. {
  507. // Keep backtracking until we don't have a trailing character:
  508. unsigned count = 0;
  509. while((*--m_position & 0xC0u) == 0x80u) ++count;
  510. // now check that the sequence was valid:
  511. if(count != detail::utf8_trailing_byte_count(*m_position))
  512. invalid_sequence();
  513. m_value = pending_read;
  514. }
  515. BaseIterator base()const
  516. {
  517. return m_position;
  518. }
  519. // construct:
  520. u8_to_u32_iterator() : m_position()
  521. {
  522. m_value = pending_read;
  523. }
  524. u8_to_u32_iterator(BaseIterator b) : m_position(b)
  525. {
  526. m_value = pending_read;
  527. }
  528. //
  529. // Checked constructor:
  530. //
  531. u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b)
  532. {
  533. m_value = pending_read;
  534. //
  535. // We must not start with a continuation character, or end with a
  536. // truncated UTF-8 sequence otherwise we run the risk of going past
  537. // the start/end of the underlying sequence:
  538. //
  539. if(start != end)
  540. {
  541. unsigned char v = *start;
  542. if((v & 0xC0u) == 0x80u)
  543. invalid_sequence();
  544. if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u))
  545. invalid_sequence();
  546. BaseIterator pos = end;
  547. do
  548. {
  549. v = *--pos;
  550. }
  551. while((start != pos) && ((v & 0xC0u) == 0x80u));
  552. std::ptrdiff_t extra = detail::utf8_byte_count(v);
  553. if(std::distance(pos, end) < extra)
  554. invalid_sequence();
  555. }
  556. }
  557. private:
  558. static void invalid_sequence()
  559. {
  560. std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character");
  561. boost::throw_exception(e);
  562. }
  563. void extract_current()const
  564. {
  565. m_value = static_cast<U32Type>(static_cast< ::boost::uint8_t>(*m_position));
  566. // we must not have a continuation character:
  567. if((m_value & 0xC0u) == 0x80u)
  568. invalid_sequence();
  569. // see how many extra bytes we have:
  570. unsigned extra = detail::utf8_trailing_byte_count(*m_position);
  571. // extract the extra bits, 6 from each extra byte:
  572. BaseIterator next(m_position);
  573. for(unsigned c = 0; c < extra; ++c)
  574. {
  575. ++next;
  576. m_value <<= 6;
  577. // We must have a continuation byte:
  578. if((static_cast<boost::uint8_t>(*next) & 0xC0) != 0x80)
  579. invalid_sequence();
  580. m_value += static_cast<boost::uint8_t>(*next) & 0x3Fu;
  581. }
  582. // we now need to remove a few of the leftmost bits, but how many depends
  583. // upon how many extra bytes we've extracted:
  584. static const boost::uint32_t masks[4] =
  585. {
  586. 0x7Fu,
  587. 0x7FFu,
  588. 0xFFFFu,
  589. 0x1FFFFFu,
  590. };
  591. m_value &= masks[extra];
  592. // check the result is in range:
  593. if(m_value > static_cast<U32Type>(0x10FFFFu))
  594. invalid_sequence();
  595. // The result must not be a surrogate:
  596. if((m_value >= static_cast<U32Type>(0xD800)) && (m_value <= static_cast<U32Type>(0xDFFF)))
  597. invalid_sequence();
  598. // We should not have had an invalidly encoded UTF8 sequence:
  599. if((extra > 0) && (m_value <= static_cast<U32Type>(masks[extra - 1])))
  600. invalid_sequence();
  601. }
  602. BaseIterator m_position;
  603. mutable U32Type m_value;
  604. };
  605. template <class BaseIterator>
  606. class utf16_output_iterator
  607. {
  608. public:
  609. typedef void difference_type;
  610. typedef void value_type;
  611. typedef boost::uint32_t* pointer;
  612. typedef boost::uint32_t& reference;
  613. typedef std::output_iterator_tag iterator_category;
  614. utf16_output_iterator(const BaseIterator& b)
  615. : m_position(b){}
  616. utf16_output_iterator(const utf16_output_iterator& that)
  617. : m_position(that.m_position){}
  618. utf16_output_iterator& operator=(const utf16_output_iterator& that)
  619. {
  620. m_position = that.m_position;
  621. return *this;
  622. }
  623. const utf16_output_iterator& operator*()const
  624. {
  625. return *this;
  626. }
  627. void operator=(boost::uint32_t val)const
  628. {
  629. push(val);
  630. }
  631. utf16_output_iterator& operator++()
  632. {
  633. return *this;
  634. }
  635. utf16_output_iterator& operator++(int)
  636. {
  637. return *this;
  638. }
  639. BaseIterator base()const
  640. {
  641. return m_position;
  642. }
  643. private:
  644. void push(boost::uint32_t v)const
  645. {
  646. if(v >= 0x10000u)
  647. {
  648. // begin by checking for a code point out of range:
  649. if(v > 0x10FFFFu)
  650. detail::invalid_utf32_code_point(v);
  651. // split into two surrogates:
  652. *m_position++ = static_cast<boost::uint16_t>(v >> 10) + detail::high_surrogate_base;
  653. *m_position++ = static_cast<boost::uint16_t>(v & detail::ten_bit_mask) + detail::low_surrogate_base;
  654. }
  655. else
  656. {
  657. // 16-bit code point:
  658. // value must not be a surrogate:
  659. if(detail::is_surrogate(v))
  660. detail::invalid_utf32_code_point(v);
  661. *m_position++ = static_cast<boost::uint16_t>(v);
  662. }
  663. }
  664. mutable BaseIterator m_position;
  665. };
  666. template <class BaseIterator>
  667. class utf8_output_iterator
  668. {
  669. public:
  670. typedef void difference_type;
  671. typedef void value_type;
  672. typedef boost::uint32_t* pointer;
  673. typedef boost::uint32_t& reference;
  674. typedef std::output_iterator_tag iterator_category;
  675. utf8_output_iterator(const BaseIterator& b)
  676. : m_position(b){}
  677. utf8_output_iterator(const utf8_output_iterator& that)
  678. : m_position(that.m_position){}
  679. utf8_output_iterator& operator=(const utf8_output_iterator& that)
  680. {
  681. m_position = that.m_position;
  682. return *this;
  683. }
  684. const utf8_output_iterator& operator*()const
  685. {
  686. return *this;
  687. }
  688. void operator=(boost::uint32_t val)const
  689. {
  690. push(val);
  691. }
  692. utf8_output_iterator& operator++()
  693. {
  694. return *this;
  695. }
  696. utf8_output_iterator& operator++(int)
  697. {
  698. return *this;
  699. }
  700. BaseIterator base()const
  701. {
  702. return m_position;
  703. }
  704. private:
  705. void push(boost::uint32_t c)const
  706. {
  707. if(c > 0x10FFFFu)
  708. detail::invalid_utf32_code_point(c);
  709. if(c < 0x80u)
  710. {
  711. *m_position++ = static_cast<unsigned char>(c);
  712. }
  713. else if(c < 0x800u)
  714. {
  715. *m_position++ = static_cast<unsigned char>(0xC0u + (c >> 6));
  716. *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  717. }
  718. else if(c < 0x10000u)
  719. {
  720. *m_position++ = static_cast<unsigned char>(0xE0u + (c >> 12));
  721. *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
  722. *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  723. }
  724. else
  725. {
  726. *m_position++ = static_cast<unsigned char>(0xF0u + (c >> 18));
  727. *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 12) & 0x3Fu));
  728. *m_position++ = static_cast<unsigned char>(0x80u + ((c >> 6) & 0x3Fu));
  729. *m_position++ = static_cast<unsigned char>(0x80u + (c & 0x3Fu));
  730. }
  731. }
  732. mutable BaseIterator m_position;
  733. };
  734. } // namespace boost
  735. #endif // BOOST_REGEX_UNICODE_ITERATOR_HPP