utrie2_impl.h 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. // © 2016 and later: Unicode, Inc. and others.
  2. // License & terms of use: http://www.unicode.org/copyright.html
  3. /*
  4. ******************************************************************************
  5. *
  6. * Copyright (C) 2001-2008, International Business Machines
  7. * Corporation and others. All Rights Reserved.
  8. *
  9. ******************************************************************************
  10. * file name: utrie2_impl.h
  11. * encoding: UTF-8
  12. * tab size: 8 (not used)
  13. * indentation:4
  14. *
  15. * created on: 2008sep26 (split off from utrie2.c)
  16. * created by: Markus W. Scherer
  17. *
  18. * Definitions needed for both runtime and builder code for UTrie2,
  19. * used by utrie2.c and utrie2_builder.c.
  20. */
  21. #ifndef __UTRIE2_IMPL_H__
  22. #define __UTRIE2_IMPL_H__
  23. #ifdef UCPTRIE_DEBUG
  24. #include "unicode/umutablecptrie.h"
  25. #endif
  26. #include "utrie2.h"
  27. /* Public UTrie2 API implementation ----------------------------------------- */
  28. /*
  29. * These definitions are mostly needed by utrie2.cpp,
  30. * but also by utrie2_serialize() and utrie2_swap().
  31. */
  32. // UTrie2 signature values, in platform endianness and opposite endianness.
  33. // The UTrie2 signature ASCII byte values spell "Tri2".
  34. #define UTRIE2_SIG 0x54726932
  35. #define UTRIE2_OE_SIG 0x32697254
  36. /**
  37. * Trie data structure in serialized form:
  38. *
  39. * UTrie2Header header;
  40. * uint16_t index[header.index2Length];
  41. * uint16_t data[header.shiftedDataLength<<2]; -- or uint32_t data[...]
  42. * @internal
  43. */
  44. typedef struct UTrie2Header {
  45. /** "Tri2" in big-endian US-ASCII (0x54726932) */
  46. uint32_t signature;
  47. /**
  48. * options bit field:
  49. * 15.. 4 reserved (0)
  50. * 3.. 0 UTrie2ValueBits valueBits
  51. */
  52. uint16_t options;
  53. /** UTRIE2_INDEX_1_OFFSET..UTRIE2_MAX_INDEX_LENGTH */
  54. uint16_t indexLength;
  55. /** (UTRIE2_DATA_START_OFFSET..UTRIE2_MAX_DATA_LENGTH)>>UTRIE2_INDEX_SHIFT */
  56. uint16_t shiftedDataLength;
  57. /** Null index and data blocks, not shifted. */
  58. uint16_t index2NullOffset, dataNullOffset;
  59. /**
  60. * First code point of the single-value range ending with U+10ffff,
  61. * rounded up and then shifted right by UTRIE2_SHIFT_1.
  62. */
  63. uint16_t shiftedHighStart;
  64. } UTrie2Header;
  65. /**
  66. * Constants for use with UTrie2Header.options.
  67. * @internal
  68. */
  69. enum {
  70. /** Mask to get the UTrie2ValueBits valueBits from options. */
  71. UTRIE2_OPTIONS_VALUE_BITS_MASK=0xf
  72. };
  73. /* Building a trie ---------------------------------------------------------- */
  74. /*
  75. * These definitions are mostly needed by utrie2_builder.c, but also by
  76. * utrie2_get32() and utrie2_enum().
  77. */
  78. enum {
  79. /**
  80. * At build time, leave a gap in the index-2 table,
  81. * at least as long as the maximum lengths of the 2-byte UTF-8 index-2 table
  82. * and the supplementary index-1 table.
  83. * Round up to UTRIE2_INDEX_2_BLOCK_LENGTH for proper compacting.
  84. */
  85. UNEWTRIE2_INDEX_GAP_OFFSET=UTRIE2_INDEX_2_BMP_LENGTH,
  86. UNEWTRIE2_INDEX_GAP_LENGTH=
  87. ((UTRIE2_UTF8_2B_INDEX_2_LENGTH+UTRIE2_MAX_INDEX_1_LENGTH)+UTRIE2_INDEX_2_MASK)&
  88. ~UTRIE2_INDEX_2_MASK,
  89. /**
  90. * Maximum length of the build-time index-2 array.
  91. * Maximum number of Unicode code points (0x110000) shifted right by UTRIE2_SHIFT_2,
  92. * plus the part of the index-2 table for lead surrogate code points,
  93. * plus the build-time index gap,
  94. * plus the null index-2 block.
  95. */
  96. UNEWTRIE2_MAX_INDEX_2_LENGTH=
  97. (0x110000>>UTRIE2_SHIFT_2)+
  98. UTRIE2_LSCP_INDEX_2_LENGTH+
  99. UNEWTRIE2_INDEX_GAP_LENGTH+
  100. UTRIE2_INDEX_2_BLOCK_LENGTH,
  101. UNEWTRIE2_INDEX_1_LENGTH=0x110000>>UTRIE2_SHIFT_1
  102. };
  103. /**
  104. * Maximum length of the build-time data array.
  105. * One entry per 0x110000 code points, plus the illegal-UTF-8 block and the null block,
  106. * plus values for the 0x400 surrogate code units.
  107. */
  108. #define UNEWTRIE2_MAX_DATA_LENGTH (0x110000+0x40+0x40+0x400)
  109. /*
  110. * Build-time trie structure.
  111. *
  112. * Just using a boolean flag for "repeat use" could lead to data array overflow
  113. * because we would not be able to detect when a data block becomes unused.
  114. * It also leads to orphan data blocks that are kept through serialization.
  115. *
  116. * Need to use reference counting for data blocks,
  117. * and allocDataBlock() needs to look for a free block before increasing dataLength.
  118. *
  119. * This scheme seems like overkill for index-2 blocks since the whole index array is
  120. * preallocated anyway (unlike the growable data array).
  121. * Just allocating multiple index-2 blocks as needed.
  122. */
  123. struct UNewTrie2 {
  124. int32_t index1[UNEWTRIE2_INDEX_1_LENGTH];
  125. int32_t index2[UNEWTRIE2_MAX_INDEX_2_LENGTH];
  126. uint32_t *data;
  127. #ifdef UCPTRIE_DEBUG
  128. UMutableCPTrie *t3;
  129. #endif
  130. uint32_t initialValue, errorValue;
  131. int32_t index2Length, dataCapacity, dataLength;
  132. int32_t firstFreeBlock;
  133. int32_t index2NullOffset, dataNullOffset;
  134. UChar32 highStart;
  135. UBool isCompacted;
  136. /**
  137. * Multi-purpose per-data-block table.
  138. *
  139. * Before compacting:
  140. *
  141. * Per-data-block reference counters/free-block list.
  142. * 0: unused
  143. * >0: reference counter (number of index-2 entries pointing here)
  144. * <0: next free data block in free-block list
  145. *
  146. * While compacting:
  147. *
  148. * Map of adjusted indexes, used in compactData() and compactIndex2().
  149. * Maps from original indexes to new ones.
  150. */
  151. int32_t map[UNEWTRIE2_MAX_DATA_LENGTH>>UTRIE2_SHIFT_2];
  152. };
  153. #endif