regeximp.h 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. // © 2016 and later: Unicode, Inc. and others.
  2. // License & terms of use: http://www.unicode.org/copyright.html
  3. //
  4. // Copyright (C) 2002-2015 International Business Machines Corporation
  5. // and others. All rights reserved.
  6. //
  7. // file: regeximp.h
  8. //
  9. // ICU Regular Expressions,
  10. // Definitions of constant values used in the compiled form of
  11. // a regular expression pattern.
  12. //
  13. #ifndef _REGEXIMP_H
  14. #define _REGEXIMP_H
  15. #include "unicode/utypes.h"
  16. #include "unicode/uobject.h"
  17. #include "unicode/uniset.h"
  18. #include "unicode/utext.h"
  19. #include "cmemory.h"
  20. #include "ucase.h"
  21. U_NAMESPACE_BEGIN
  22. // For debugging, define REGEX_DEBUG
  23. // To define with configure,
  24. // CPPFLAGS="-DREGEX_DEBUG" ./runConfigureICU --enable-debug --disable-release Linux
  25. #ifdef REGEX_DEBUG
  26. //
  27. // debugging options. Enable one or more of the three #defines immediately following
  28. //
  29. //#define REGEX_SCAN_DEBUG
  30. #define REGEX_DUMP_DEBUG
  31. #define REGEX_RUN_DEBUG
  32. // End of #defines inteded to be directly set.
  33. #include <stdio.h>
  34. #endif
  35. #ifdef REGEX_SCAN_DEBUG
  36. #define REGEX_SCAN_DEBUG_PRINTF(a) printf a
  37. #else
  38. #define REGEX_SCAN_DEBUG_PRINTF(a)
  39. #endif
  40. //
  41. // Opcode types In the compiled form of the regexp, these are the type, or opcodes,
  42. // of the entries.
  43. //
  44. enum {
  45. URX_RESERVED_OP = 0, // For multi-operand ops, most non-first words.
  46. URX_RESERVED_OP_N = 255, // For multi-operand ops, negative operand values.
  47. URX_BACKTRACK = 1, // Force a backtrack, as if a match test had failed.
  48. URX_END = 2,
  49. URX_ONECHAR = 3, // Value field is the 21 bit unicode char to match
  50. URX_STRING = 4, // Value field is index of string start
  51. URX_STRING_LEN = 5, // Value field is string length (code units)
  52. URX_STATE_SAVE = 6, // Value field is pattern position to push
  53. URX_NOP = 7,
  54. URX_START_CAPTURE = 8, // Value field is capture group number.
  55. URX_END_CAPTURE = 9, // Value field is capture group number
  56. URX_STATIC_SETREF = 10, // Value field is index of set in array of sets.
  57. URX_SETREF = 11, // Value field is index of set in array of sets.
  58. URX_DOTANY = 12,
  59. URX_JMP = 13, // Value field is destination position in
  60. // the pattern.
  61. URX_FAIL = 14, // Stop match operation, No match.
  62. URX_JMP_SAV = 15, // Operand: JMP destination location
  63. URX_BACKSLASH_B = 16, // Value field: 0: \b 1: \B
  64. URX_BACKSLASH_G = 17,
  65. URX_JMP_SAV_X = 18, // Conditional JMP_SAV,
  66. // Used in (x)+, breaks loop on zero length match.
  67. // Operand: Jmp destination.
  68. URX_BACKSLASH_X = 19,
  69. URX_BACKSLASH_Z = 20, // \z Unconditional end of line.
  70. URX_DOTANY_ALL = 21, // ., in the . matches any mode.
  71. URX_BACKSLASH_D = 22, // Value field: 0: \d 1: \D
  72. URX_CARET = 23, // Value field: 1: multi-line mode.
  73. URX_DOLLAR = 24, // Also for \Z
  74. URX_CTR_INIT = 25, // Counter Inits for {Interval} loops.
  75. URX_CTR_INIT_NG = 26, // 2 kinds, normal and non-greedy.
  76. // These are 4 word opcodes. See description.
  77. // First Operand: Data loc of counter variable
  78. // 2nd Operand: Pat loc of the URX_CTR_LOOPx
  79. // at the end of the loop.
  80. // 3rd Operand: Minimum count.
  81. // 4th Operand: Max count, -1 for unbounded.
  82. URX_DOTANY_UNIX = 27, // '.' operator in UNIX_LINES mode, only \n marks end of line.
  83. URX_CTR_LOOP = 28, // Loop Ops for {interval} loops.
  84. URX_CTR_LOOP_NG = 29, // Also in three flavors.
  85. // Operand is loc of corresponding CTR_INIT.
  86. URX_CARET_M_UNIX = 30, // '^' operator, test for start of line in multi-line
  87. // plus UNIX_LINES mode.
  88. URX_RELOC_OPRND = 31, // Operand value in multi-operand ops that refers
  89. // back into compiled pattern code, and thus must
  90. // be relocated when inserting/deleting ops in code.
  91. URX_STO_SP = 32, // Store the stack ptr. Operand is location within
  92. // matcher data (not stack data) to store it.
  93. URX_LD_SP = 33, // Load the stack pointer. Operand is location
  94. // to load from.
  95. URX_BACKREF = 34, // Back Reference. Parameter is the index of the
  96. // capture group variables in the state stack frame.
  97. URX_STO_INP_LOC = 35, // Store the input location. Operand is location
  98. // within the matcher stack frame.
  99. URX_JMPX = 36, // Conditional JMP.
  100. // First Operand: JMP target location.
  101. // Second Operand: Data location containing an
  102. // input position. If current input position ==
  103. // saved input position, FAIL rather than taking
  104. // the JMP
  105. URX_LA_START = 37, // Starting a LookAround expression.
  106. // Save InputPos, SP and active region in static data.
  107. // Operand: Static data offset for the save
  108. URX_LA_END = 38, // Ending a Lookaround expression.
  109. // Restore InputPos and Stack to saved values.
  110. // Operand: Static data offset for saved data.
  111. URX_ONECHAR_I = 39, // Test for case-insensitive match of a literal character.
  112. // Operand: the literal char.
  113. URX_STRING_I = 40, // Case insensitive string compare.
  114. // First Operand: Index of start of string in string literals
  115. // Second Operand (next word in compiled code):
  116. // the length of the string.
  117. URX_BACKREF_I = 41, // Case insensitive back reference.
  118. // Parameter is the index of the
  119. // capture group variables in the state stack frame.
  120. URX_DOLLAR_M = 42, // $ in multi-line mode.
  121. URX_CARET_M = 43, // ^ in multi-line mode.
  122. URX_LB_START = 44, // LookBehind Start.
  123. // Paramater is data location
  124. URX_LB_CONT = 45, // LookBehind Continue.
  125. // Param 0: the data location
  126. // Param 1: The minimum length of the look-behind match
  127. // Param 2: The max length of the look-behind match
  128. URX_LB_END = 46, // LookBehind End.
  129. // Parameter is the data location.
  130. // Check that match ended at the right spot,
  131. // Restore original input string len.
  132. URX_LBN_CONT = 47, // Negative LookBehind Continue
  133. // Param 0: the data location
  134. // Param 1: The minimum length of the look-behind match
  135. // Param 2: The max length of the look-behind match
  136. // Param 3: The pattern loc following the look-behind block.
  137. URX_LBN_END = 48, // Negative LookBehind end
  138. // Parameter is the data location.
  139. // Check that the match ended at the right spot.
  140. URX_STAT_SETREF_N = 49, // Reference to a prebuilt set (e.g. \w), negated
  141. // Operand is index of set in array of sets.
  142. URX_LOOP_SR_I = 50, // Init a [set]* loop.
  143. // Operand is the sets index in array of user sets.
  144. URX_LOOP_C = 51, // Continue a [set]* or OneChar* loop.
  145. // Operand is a matcher static data location.
  146. // Must always immediately follow LOOP_x_I instruction.
  147. URX_LOOP_DOT_I = 52, // .*, initialization of the optimized loop.
  148. // Operand value:
  149. // bit 0:
  150. // 0: Normal (. doesn't match new-line) mode.
  151. // 1: . matches new-line mode.
  152. // bit 1: controls what new-lines are recognized by this operation.
  153. // 0: All Unicode New-lines
  154. // 1: UNIX_LINES, \u000a only.
  155. URX_BACKSLASH_BU = 53, // \b or \B in UREGEX_UWORD mode, using Unicode style
  156. // word boundaries.
  157. URX_DOLLAR_D = 54, // $ end of input test, in UNIX_LINES mode.
  158. URX_DOLLAR_MD = 55, // $ end of input test, in MULTI_LINE and UNIX_LINES mode.
  159. URX_BACKSLASH_H = 56, // Value field: 0: \h 1: \H
  160. URX_BACKSLASH_R = 57, // Any line break sequence.
  161. URX_BACKSLASH_V = 58 // Value field: 0: \v 1: \V
  162. };
  163. // Keep this list of opcode names in sync with the above enum
  164. // Used for debug printing only.
  165. #define URX_OPCODE_NAMES \
  166. " ", \
  167. "BACKTRACK", \
  168. "END", \
  169. "ONECHAR", \
  170. "STRING", \
  171. "STRING_LEN", \
  172. "STATE_SAVE", \
  173. "NOP", \
  174. "START_CAPTURE", \
  175. "END_CAPTURE", \
  176. "URX_STATIC_SETREF", \
  177. "SETREF", \
  178. "DOTANY", \
  179. "JMP", \
  180. "FAIL", \
  181. "JMP_SAV", \
  182. "BACKSLASH_B", \
  183. "BACKSLASH_G", \
  184. "JMP_SAV_X", \
  185. "BACKSLASH_X", \
  186. "BACKSLASH_Z", \
  187. "DOTANY_ALL", \
  188. "BACKSLASH_D", \
  189. "CARET", \
  190. "DOLLAR", \
  191. "CTR_INIT", \
  192. "CTR_INIT_NG", \
  193. "DOTANY_UNIX", \
  194. "CTR_LOOP", \
  195. "CTR_LOOP_NG", \
  196. "URX_CARET_M_UNIX", \
  197. "RELOC_OPRND", \
  198. "STO_SP", \
  199. "LD_SP", \
  200. "BACKREF", \
  201. "STO_INP_LOC", \
  202. "JMPX", \
  203. "LA_START", \
  204. "LA_END", \
  205. "ONECHAR_I", \
  206. "STRING_I", \
  207. "BACKREF_I", \
  208. "DOLLAR_M", \
  209. "CARET_M", \
  210. "LB_START", \
  211. "LB_CONT", \
  212. "LB_END", \
  213. "LBN_CONT", \
  214. "LBN_END", \
  215. "STAT_SETREF_N", \
  216. "LOOP_SR_I", \
  217. "LOOP_C", \
  218. "LOOP_DOT_I", \
  219. "BACKSLASH_BU", \
  220. "DOLLAR_D", \
  221. "DOLLAR_MD", \
  222. "URX_BACKSLASH_H", \
  223. "URX_BACKSLASH_R", \
  224. "URX_BACKSLASH_V"
  225. //
  226. // Convenience macros for assembling and disassembling a compiled operation.
  227. //
  228. #define URX_TYPE(x) ((uint32_t)(x) >> 24)
  229. #define URX_VAL(x) ((x) & 0xffffff)
  230. //
  231. // Access to Unicode Sets composite character properties
  232. // The sets are accessed by the match engine for things like \w (word boundary)
  233. //
  234. enum {
  235. URX_ISWORD_SET = 1,
  236. URX_ISALNUM_SET = 2,
  237. URX_ISALPHA_SET = 3,
  238. URX_ISSPACE_SET = 4,
  239. URX_GC_NORMAL, // Sets for finding grapheme cluster boundaries.
  240. URX_GC_EXTEND,
  241. URX_GC_CONTROL,
  242. URX_GC_L,
  243. URX_GC_LV,
  244. URX_GC_LVT,
  245. URX_GC_V,
  246. URX_GC_T,
  247. URX_LAST_SET,
  248. URX_NEG_SET = 0x800000 // Flag bit to reverse sense of set
  249. // membership test.
  250. };
  251. //
  252. // Match Engine State Stack Frame Layout.
  253. //
  254. struct REStackFrame {
  255. // Header
  256. int64_t fInputIdx; // Position of next character in the input string
  257. int64_t fPatIdx; // Position of next Op in the compiled pattern
  258. // (int64_t for UVector64, values fit in an int32_t)
  259. // Remainder
  260. int64_t fExtra[1]; // Extra state, for capture group start/ends
  261. // atomic parentheses, repeat counts, etc.
  262. // Locations assigned at pattern compile time.
  263. // Variable-length array.
  264. };
  265. // number of UVector elements in the header
  266. #define RESTACKFRAME_HDRCOUNT 2
  267. //
  268. // Start-Of-Match type. Used by find() to quickly scan to positions where a
  269. // match might start before firing up the full match engine.
  270. //
  271. enum StartOfMatch {
  272. START_NO_INFO, // No hint available.
  273. START_CHAR, // Match starts with a literal code point.
  274. START_SET, // Match starts with something matching a set.
  275. START_START, // Match starts at start of buffer only (^ or \A)
  276. START_LINE, // Match starts with ^ in multi-line mode.
  277. START_STRING // Match starts with a literal string.
  278. };
  279. #define START_OF_MATCH_STR(v) ((v)==START_NO_INFO? "START_NO_INFO" : \
  280. (v)==START_CHAR? "START_CHAR" : \
  281. (v)==START_SET? "START_SET" : \
  282. (v)==START_START? "START_START" : \
  283. (v)==START_LINE? "START_LINE" : \
  284. (v)==START_STRING? "START_STRING" : \
  285. "ILLEGAL")
  286. //
  287. // 8 bit set, to fast-path latin-1 set membership tests.
  288. //
  289. struct Regex8BitSet : public UMemory {
  290. inline Regex8BitSet();
  291. inline void operator = (const Regex8BitSet &s);
  292. inline void init(const UnicodeSet *src);
  293. inline UBool contains(UChar32 c);
  294. inline void add(UChar32 c);
  295. int8_t d[32];
  296. };
  297. inline Regex8BitSet::Regex8BitSet() {
  298. uprv_memset(d, 0, sizeof(d));
  299. }
  300. inline UBool Regex8BitSet::contains(UChar32 c) {
  301. // No bounds checking! This is deliberate.
  302. return ((d[c>>3] & 1 <<(c&7)) != 0);
  303. }
  304. inline void Regex8BitSet::add(UChar32 c) {
  305. d[c>>3] |= 1 << (c&7);
  306. }
  307. inline void Regex8BitSet::init(const UnicodeSet *s) {
  308. if (s != NULL) {
  309. for (int32_t i=0; i<=255; i++) {
  310. if (s->contains(i)) {
  311. this->add(i);
  312. }
  313. }
  314. }
  315. }
  316. inline void Regex8BitSet::operator = (const Regex8BitSet &s) {
  317. uprv_memcpy(d, s.d, sizeof(d));
  318. }
  319. // Case folded UText Iterator helper class.
  320. // Wraps a UText, provides a case-folded enumeration over its contents.
  321. // Used in implementing case insensitive matching constructs.
  322. // Implementation in rematch.cpp
  323. class CaseFoldingUTextIterator: public UMemory {
  324. public:
  325. CaseFoldingUTextIterator(UText &text);
  326. ~CaseFoldingUTextIterator();
  327. UChar32 next(); // Next case folded character
  328. UBool inExpansion(); // True if last char returned from next() and the
  329. // next to be returned both originated from a string
  330. // folding of the same code point from the orignal UText.
  331. private:
  332. UText &fUText;
  333. const UChar *fFoldChars;
  334. int32_t fFoldLength;
  335. int32_t fFoldIndex;
  336. };
  337. // Case folded UChar * string iterator.
  338. // Wraps a UChar *, provides a case-folded enumeration over its contents.
  339. // Used in implementing case insensitive matching constructs.
  340. // Implementation in rematch.cpp
  341. class CaseFoldingUCharIterator: public UMemory {
  342. public:
  343. CaseFoldingUCharIterator(const UChar *chars, int64_t start, int64_t limit);
  344. ~CaseFoldingUCharIterator();
  345. UChar32 next(); // Next case folded character
  346. UBool inExpansion(); // True if last char returned from next() and the
  347. // next to be returned both originated from a string
  348. // folding of the same code point from the orignal UText.
  349. int64_t getIndex(); // Return the current input buffer index.
  350. private:
  351. const UChar *fChars;
  352. int64_t fIndex;
  353. int64_t fLimit;
  354. const UChar *fFoldChars;
  355. int32_t fFoldLength;
  356. int32_t fFoldIndex;
  357. };
  358. U_NAMESPACE_END
  359. #endif