insert_whitespace_detection.hpp 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. /*=============================================================================
  2. Boost.Wave: A Standard compliant C++ preprocessor library
  3. Detect the need to insert a whitespace token into the output stream
  4. http://www.boost.org/
  5. Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
  6. Software License, Version 1.0. (See accompanying file
  7. LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  8. =============================================================================*/
  9. #if !defined(BOOST_INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)
  10. #define BOOST_INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED
  11. #include <boost/wave/wave_config.hpp>
  12. #include <boost/wave/token_ids.hpp>
  13. // this must occur after all of the includes and before any code appears
  14. #ifdef BOOST_HAS_ABI_HEADERS
  15. #include BOOST_ABI_PREFIX
  16. #endif
  17. ///////////////////////////////////////////////////////////////////////////////
  18. namespace boost {
  19. namespace wave {
  20. namespace util {
  21. namespace impl {
  22. // T_IDENTIFIER
  23. template <typename StringT>
  24. inline bool
  25. would_form_universal_char (StringT const &value)
  26. {
  27. if ('u' != value[0] && 'U' != value[0])
  28. return false;
  29. if ('u' == value[0] && value.size() < 5)
  30. return false;
  31. if ('U' == value[0] && value.size() < 9)
  32. return false;
  33. typename StringT::size_type pos =
  34. value.find_first_not_of("0123456789abcdefABCDEF", 1);
  35. if (StringT::npos == pos ||
  36. ('u' == value[0] && pos > 5) ||
  37. ('U' == value[0] && pos > 9))
  38. {
  39. return true; // would form an universal char
  40. }
  41. return false;
  42. }
  43. template <typename StringT>
  44. inline bool
  45. handle_identifier(boost::wave::token_id prev,
  46. boost::wave::token_id before, StringT const &value)
  47. {
  48. using namespace boost::wave;
  49. switch (prev) {
  50. case T_IDENTIFIER:
  51. case T_NONREPLACABLE_IDENTIFIER:
  52. case T_COMPL_ALT:
  53. case T_OR_ALT:
  54. case T_AND_ALT:
  55. case T_NOT_ALT:
  56. case T_XOR_ALT:
  57. case T_ANDASSIGN_ALT:
  58. case T_ORASSIGN_ALT:
  59. case T_XORASSIGN_ALT:
  60. case T_NOTEQUAL_ALT:
  61. case T_FIXEDPOINTLIT:
  62. return true;
  63. case T_FLOATLIT:
  64. case T_INTLIT:
  65. case T_PP_NUMBER:
  66. return (value.size() > 1 || (value[0] != 'e' && value[0] != 'E'));
  67. // avoid constructing universal characters (\u1234)
  68. case T_UNKNOWN_UNIVERSALCHAR:
  69. return would_form_universal_char(value);
  70. default:
  71. break;
  72. }
  73. return false;
  74. }
  75. // T_INTLIT
  76. inline bool
  77. handle_intlit(boost::wave::token_id prev, boost::wave::token_id /*before*/)
  78. {
  79. using namespace boost::wave;
  80. switch (prev) {
  81. case T_IDENTIFIER:
  82. case T_NONREPLACABLE_IDENTIFIER:
  83. case T_INTLIT:
  84. case T_FLOATLIT:
  85. case T_FIXEDPOINTLIT:
  86. case T_PP_NUMBER:
  87. return true;
  88. default:
  89. break;
  90. }
  91. return false;
  92. }
  93. // T_FLOATLIT
  94. inline bool
  95. handle_floatlit(boost::wave::token_id prev,
  96. boost::wave::token_id /*before*/)
  97. {
  98. using namespace boost::wave;
  99. switch (prev) {
  100. case T_IDENTIFIER:
  101. case T_NONREPLACABLE_IDENTIFIER:
  102. case T_INTLIT:
  103. case T_FLOATLIT:
  104. case T_FIXEDPOINTLIT:
  105. case T_PP_NUMBER:
  106. return true;
  107. default:
  108. break;
  109. }
  110. return false;
  111. }
  112. // <% T_LEFTBRACE
  113. inline bool
  114. handle_alt_leftbrace(boost::wave::token_id prev,
  115. boost::wave::token_id /*before*/)
  116. {
  117. using namespace boost::wave;
  118. switch (prev) {
  119. case T_LESS: // <<%
  120. case T_SHIFTLEFT: // <<<%
  121. return true;
  122. default:
  123. break;
  124. }
  125. return false;
  126. }
  127. // <: T_LEFTBRACKET
  128. inline bool
  129. handle_alt_leftbracket(boost::wave::token_id prev,
  130. boost::wave::token_id /*before*/)
  131. {
  132. using namespace boost::wave;
  133. switch (prev) {
  134. case T_LESS: // <<:
  135. case T_SHIFTLEFT: // <<<:
  136. return true;
  137. default:
  138. break;
  139. }
  140. return false;
  141. }
  142. // T_FIXEDPOINTLIT
  143. inline bool
  144. handle_fixedpointlit(boost::wave::token_id prev,
  145. boost::wave::token_id /*before*/)
  146. {
  147. using namespace boost::wave;
  148. switch (prev) {
  149. case T_IDENTIFIER:
  150. case T_NONREPLACABLE_IDENTIFIER:
  151. case T_INTLIT:
  152. case T_FLOATLIT:
  153. case T_FIXEDPOINTLIT:
  154. case T_PP_NUMBER:
  155. return true;
  156. default:
  157. break;
  158. }
  159. return false;
  160. }
  161. // T_DOT
  162. inline bool
  163. handle_dot(boost::wave::token_id prev, boost::wave::token_id before)
  164. {
  165. using namespace boost::wave;
  166. switch (prev) {
  167. case T_DOT:
  168. if (T_DOT == before)
  169. return true; // ...
  170. break;
  171. default:
  172. break;
  173. }
  174. return false;
  175. }
  176. // T_QUESTION_MARK
  177. inline bool
  178. handle_questionmark(boost::wave::token_id prev,
  179. boost::wave::token_id /*before*/)
  180. {
  181. using namespace boost::wave;
  182. switch(prev) {
  183. case T_UNKNOWN_UNIVERSALCHAR: // \?
  184. case T_QUESTION_MARK: // ??
  185. return true;
  186. default:
  187. break;
  188. }
  189. return false;
  190. }
  191. // T_NEWLINE
  192. inline bool
  193. handle_newline(boost::wave::token_id prev,
  194. boost::wave::token_id before)
  195. {
  196. using namespace boost::wave;
  197. switch(prev) {
  198. case TOKEN_FROM_ID('\\', UnknownTokenType): // \ \n
  199. case T_DIVIDE:
  200. if (T_QUESTION_MARK == before)
  201. return true; // ?/\n // may be \\n
  202. break;
  203. default:
  204. break;
  205. }
  206. return false;
  207. }
  208. inline bool
  209. handle_parens(boost::wave::token_id prev)
  210. {
  211. switch (prev) {
  212. case T_LEFTPAREN:
  213. case T_RIGHTPAREN:
  214. case T_LEFTBRACKET:
  215. case T_RIGHTBRACKET:
  216. case T_LEFTBRACE:
  217. case T_RIGHTBRACE:
  218. case T_SEMICOLON:
  219. case T_COMMA:
  220. case T_COLON:
  221. // no insertion between parens/brackets/braces and operators
  222. return false;
  223. default:
  224. break;
  225. }
  226. return true;
  227. }
  228. } // namespace impl
  229. class insert_whitespace_detection
  230. {
  231. public:
  232. insert_whitespace_detection(bool insert_whitespace_ = true)
  233. : insert_whitespace(insert_whitespace_),
  234. prev(boost::wave::T_EOF), beforeprev(boost::wave::T_EOF)
  235. {}
  236. template <typename StringT>
  237. bool must_insert(boost::wave::token_id current, StringT const &value)
  238. {
  239. if (!insert_whitespace)
  240. return false; // skip whitespace insertion alltogether
  241. using namespace boost::wave;
  242. switch (current) {
  243. case T_NONREPLACABLE_IDENTIFIER:
  244. case T_IDENTIFIER:
  245. return impl::handle_identifier(prev, beforeprev, value);
  246. case T_PP_NUMBER:
  247. case T_INTLIT:
  248. return impl::handle_intlit(prev, beforeprev);
  249. case T_FLOATLIT:
  250. return impl::handle_floatlit(prev, beforeprev);
  251. case T_STRINGLIT:
  252. if (TOKEN_FROM_ID('L', IdentifierTokenType) == prev) // 'L'
  253. return true;
  254. break;
  255. case T_LEFTBRACE_ALT:
  256. return impl::handle_alt_leftbrace(prev, beforeprev);
  257. case T_LEFTBRACKET_ALT:
  258. return impl::handle_alt_leftbracket(prev, beforeprev);
  259. case T_FIXEDPOINTLIT:
  260. return impl::handle_fixedpointlit(prev, beforeprev);
  261. case T_DOT:
  262. return impl::handle_dot(prev, beforeprev);
  263. case T_QUESTION_MARK:
  264. return impl::handle_questionmark(prev, beforeprev);
  265. case T_NEWLINE:
  266. return impl::handle_newline(prev, beforeprev);
  267. case T_LEFTPAREN:
  268. case T_RIGHTPAREN:
  269. case T_LEFTBRACKET:
  270. case T_RIGHTBRACKET:
  271. case T_SEMICOLON:
  272. case T_COMMA:
  273. case T_COLON:
  274. switch (prev) {
  275. case T_LEFTPAREN:
  276. case T_RIGHTPAREN:
  277. case T_LEFTBRACKET:
  278. case T_RIGHTBRACKET:
  279. case T_LEFTBRACE:
  280. case T_RIGHTBRACE:
  281. return false; // no insertion between parens/brackets/braces
  282. default:
  283. if (IS_CATEGORY(prev, OperatorTokenType))
  284. return false;
  285. break;
  286. }
  287. break;
  288. case T_LEFTBRACE:
  289. case T_RIGHTBRACE:
  290. switch (prev) {
  291. case T_LEFTPAREN:
  292. case T_RIGHTPAREN:
  293. case T_LEFTBRACKET:
  294. case T_RIGHTBRACKET:
  295. case T_LEFTBRACE:
  296. case T_RIGHTBRACE:
  297. case T_SEMICOLON:
  298. case T_COMMA:
  299. case T_COLON:
  300. return false; // no insertion between parens/brackets/braces
  301. case T_QUESTION_MARK:
  302. if (T_QUESTION_MARK == beforeprev)
  303. return true;
  304. if (IS_CATEGORY(prev, OperatorTokenType))
  305. return false;
  306. break;
  307. default:
  308. break;
  309. }
  310. break;
  311. case T_MINUS:
  312. case T_MINUSMINUS:
  313. case T_MINUSASSIGN:
  314. if (T_MINUS == prev || T_MINUSMINUS == prev)
  315. return true;
  316. if (!impl::handle_parens(prev))
  317. return false;
  318. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  319. return true;
  320. break;
  321. case T_PLUS:
  322. case T_PLUSPLUS:
  323. case T_PLUSASSIGN:
  324. if (T_PLUS == prev || T_PLUSPLUS == prev)
  325. return true;
  326. if (!impl::handle_parens(prev))
  327. return false;
  328. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  329. return true;
  330. break;
  331. case T_DIVIDE:
  332. case T_DIVIDEASSIGN:
  333. if (T_DIVIDE == prev)
  334. return true;
  335. if (!impl::handle_parens(prev))
  336. return false;
  337. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  338. return true;
  339. break;
  340. case T_EQUAL:
  341. case T_ASSIGN:
  342. switch (prev) {
  343. case T_PLUSASSIGN:
  344. case T_MINUSASSIGN:
  345. case T_DIVIDEASSIGN:
  346. case T_STARASSIGN:
  347. case T_SHIFTRIGHTASSIGN:
  348. case T_SHIFTLEFTASSIGN:
  349. case T_EQUAL:
  350. case T_NOTEQUAL:
  351. case T_LESSEQUAL:
  352. case T_GREATEREQUAL:
  353. case T_LESS:
  354. case T_GREATER:
  355. case T_PLUS:
  356. case T_MINUS:
  357. case T_STAR:
  358. case T_DIVIDE:
  359. case T_ORASSIGN:
  360. case T_ANDASSIGN:
  361. case T_XORASSIGN:
  362. case T_OR:
  363. case T_AND:
  364. case T_XOR:
  365. case T_OROR:
  366. case T_ANDAND:
  367. return true;
  368. case T_QUESTION_MARK:
  369. if (T_QUESTION_MARK == beforeprev)
  370. return true;
  371. break;
  372. default:
  373. if (!impl::handle_parens(prev))
  374. return false;
  375. break;
  376. }
  377. break;
  378. case T_GREATER:
  379. if (T_MINUS == prev || T_GREATER == prev)
  380. return true; // prevent -> or >>
  381. if (!impl::handle_parens(prev))
  382. return false;
  383. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  384. return true;
  385. break;
  386. case T_LESS:
  387. if (T_LESS == prev)
  388. return true; // prevent <<
  389. // fall through
  390. case T_CHARLIT:
  391. case T_NOT:
  392. case T_NOTEQUAL:
  393. if (!impl::handle_parens(prev))
  394. return false;
  395. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  396. return true;
  397. break;
  398. case T_AND:
  399. case T_ANDAND:
  400. if (!impl::handle_parens(prev))
  401. return false;
  402. if (T_AND == prev || T_ANDAND == prev)
  403. return true;
  404. break;
  405. case T_OR:
  406. if (!impl::handle_parens(prev))
  407. return false;
  408. if (T_OR == prev)
  409. return true;
  410. break;
  411. case T_XOR:
  412. if (!impl::handle_parens(prev))
  413. return false;
  414. if (T_XOR == prev)
  415. return true;
  416. break;
  417. case T_COMPL_ALT:
  418. case T_OR_ALT:
  419. case T_AND_ALT:
  420. case T_NOT_ALT:
  421. case T_XOR_ALT:
  422. case T_ANDASSIGN_ALT:
  423. case T_ORASSIGN_ALT:
  424. case T_XORASSIGN_ALT:
  425. case T_NOTEQUAL_ALT:
  426. switch (prev) {
  427. case T_LEFTPAREN:
  428. case T_RIGHTPAREN:
  429. case T_LEFTBRACKET:
  430. case T_RIGHTBRACKET:
  431. case T_LEFTBRACE:
  432. case T_RIGHTBRACE:
  433. case T_SEMICOLON:
  434. case T_COMMA:
  435. case T_COLON:
  436. // no insertion between parens/brackets/braces and operators
  437. return false;
  438. case T_IDENTIFIER:
  439. if (T_NONREPLACABLE_IDENTIFIER == prev ||
  440. IS_CATEGORY(prev, KeywordTokenType))
  441. {
  442. return true;
  443. }
  444. break;
  445. default:
  446. break;
  447. }
  448. break;
  449. case T_STAR:
  450. if (T_STAR == prev)
  451. return false; // '*****' do not need to be separated
  452. if (T_GREATER== prev &&
  453. (T_MINUS == beforeprev || T_MINUSMINUS == beforeprev)
  454. )
  455. {
  456. return true; // prevent ->*
  457. }
  458. break;
  459. case T_POUND:
  460. if (T_POUND == prev)
  461. return true;
  462. break;
  463. default:
  464. break;
  465. }
  466. // FIXME: else, handle operators separately (will catch to many cases)
  467. // if (IS_CATEGORY(current, OperatorTokenType) &&
  468. // IS_CATEGORY(prev, OperatorTokenType))
  469. // {
  470. // return true; // operators must be delimited always
  471. // }
  472. return false;
  473. }
  474. void shift_tokens (boost::wave::token_id next_id)
  475. {
  476. if (insert_whitespace) {
  477. beforeprev = prev;
  478. prev = next_id;
  479. }
  480. }
  481. private:
  482. bool insert_whitespace; // enable this component
  483. boost::wave::token_id prev; // the previous analyzed token
  484. boost::wave::token_id beforeprev; // the token before the previous
  485. };
  486. ///////////////////////////////////////////////////////////////////////////////
  487. } // namespace util
  488. } // namespace wave
  489. } // namespace boost
  490. // the suffix header occurs after all of the code
  491. #ifdef BOOST_HAS_ABI_HEADERS
  492. #include BOOST_ABI_SUFFIX
  493. #endif
  494. #endif // !defined(BOOST_INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)