pagemap.h 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
  2. // Copyright (c) 2005, Google Inc.
  3. // All rights reserved.
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // ---
  31. // Author: Sanjay Ghemawat <opensource@google.com>
  32. //
  33. // A data structure used by the caching malloc. It maps from page# to
  34. // a pointer that contains info about that page. We use two
  35. // representations: one for 32-bit addresses, and another for 64 bit
  36. // addresses. Both representations provide the same interface. The
  37. // first representation is implemented as a flat array, the seconds as
  38. // a three-level radix tree that strips away approximately 1/3rd of
  39. // the bits every time.
  40. //
  41. // The BITS parameter should be the number of bits required to hold
  42. // a page number. E.g., with 32 bit pointers and 4K pages (i.e.,
  43. // page offset fits in lower 12 bits), BITS == 20.
  44. #ifndef TCMALLOC_PAGEMAP_H_
  45. #define TCMALLOC_PAGEMAP_H_
  46. #include "config.h"
  47. #include <stddef.h> // for NULL, size_t
  48. #include <string.h> // for memset
  49. #if defined HAVE_STDINT_H
  50. #include <stdint.h>
  51. #elif defined HAVE_INTTYPES_H
  52. #include <inttypes.h>
  53. #else
  54. #include <sys/types.h>
  55. #endif
  56. #include "internal_logging.h" // for ASSERT
  57. // Single-level array
  58. template <int BITS>
  59. class TCMalloc_PageMap1 {
  60. private:
  61. static const int LENGTH = 1 << BITS;
  62. void** array_;
  63. public:
  64. typedef uintptr_t Number;
  65. explicit TCMalloc_PageMap1(void* (*allocator)(size_t)) {
  66. array_ = reinterpret_cast<void**>((*allocator)(sizeof(void*) << BITS));
  67. memset(array_, 0, sizeof(void*) << BITS);
  68. }
  69. // Ensure that the map contains initialized entries "x .. x+n-1".
  70. // Returns true if successful, false if we could not allocate memory.
  71. bool Ensure(Number x, size_t n) {
  72. // Nothing to do since flat array was allocated at start. All
  73. // that's left is to check for overflow (that is, we don't want to
  74. // ensure a number y where array_[y] would be an out-of-bounds
  75. // access).
  76. return n <= LENGTH - x; // an overflow-free way to do "x + n <= LENGTH"
  77. }
  78. void PreallocateMoreMemory() {}
  79. // Return the current value for KEY. Returns NULL if not yet set,
  80. // or if k is out of range.
  81. ATTRIBUTE_ALWAYS_INLINE
  82. void* get(Number k) const {
  83. if ((k >> BITS) > 0) {
  84. return NULL;
  85. }
  86. return array_[k];
  87. }
  88. // REQUIRES "k" is in range "[0,2^BITS-1]".
  89. // REQUIRES "k" has been ensured before.
  90. //
  91. // Sets the value 'v' for key 'k'.
  92. void set(Number k, void* v) {
  93. array_[k] = v;
  94. }
  95. // Return the first non-NULL pointer found in this map for
  96. // a page number >= k. Returns NULL if no such number is found.
  97. void* Next(Number k) const {
  98. while (k < (1 << BITS)) {
  99. if (array_[k] != NULL) return array_[k];
  100. k++;
  101. }
  102. return NULL;
  103. }
  104. };
  105. // Two-level radix tree
  106. template <int BITS>
  107. class TCMalloc_PageMap2 {
  108. private:
  109. static const int LEAF_BITS = (BITS + 1) / 2;
  110. static const int LEAF_LENGTH = 1 << LEAF_BITS;
  111. static const int ROOT_BITS = BITS - LEAF_BITS;
  112. static const int ROOT_LENGTH = 1 << ROOT_BITS;
  113. // Leaf node
  114. struct Leaf {
  115. void* values[LEAF_LENGTH];
  116. };
  117. Leaf* root_[ROOT_LENGTH]; // Pointers to child nodes
  118. void* (*allocator_)(size_t); // Memory allocator
  119. public:
  120. typedef uintptr_t Number;
  121. explicit TCMalloc_PageMap2(void* (*allocator)(size_t)) {
  122. allocator_ = allocator;
  123. memset(root_, 0, sizeof(root_));
  124. }
  125. ATTRIBUTE_ALWAYS_INLINE
  126. void* get(Number k) const {
  127. const Number i1 = k >> LEAF_BITS;
  128. const Number i2 = k & (LEAF_LENGTH-1);
  129. if ((k >> BITS) > 0 || root_[i1] == NULL) {
  130. return NULL;
  131. }
  132. return root_[i1]->values[i2];
  133. }
  134. void set(Number k, void* v) {
  135. const Number i1 = k >> LEAF_BITS;
  136. const Number i2 = k & (LEAF_LENGTH-1);
  137. ASSERT(i1 < ROOT_LENGTH);
  138. root_[i1]->values[i2] = v;
  139. }
  140. bool Ensure(Number start, size_t n) {
  141. for (Number key = start; key <= start + n - 1; ) {
  142. const Number i1 = key >> LEAF_BITS;
  143. // Check for overflow
  144. if (i1 >= ROOT_LENGTH)
  145. return false;
  146. // Make 2nd level node if necessary
  147. if (root_[i1] == NULL) {
  148. Leaf* leaf = reinterpret_cast<Leaf*>((*allocator_)(sizeof(Leaf)));
  149. if (leaf == NULL) return false;
  150. memset(leaf, 0, sizeof(*leaf));
  151. root_[i1] = leaf;
  152. }
  153. // Advance key past whatever is covered by this leaf node
  154. key = ((key >> LEAF_BITS) + 1) << LEAF_BITS;
  155. }
  156. return true;
  157. }
  158. void PreallocateMoreMemory() {
  159. // Allocate enough to keep track of all possible pages
  160. if (BITS < 20) {
  161. Ensure(0, Number(1) << BITS);
  162. }
  163. }
  164. void* Next(Number k) const {
  165. while (k < (Number(1) << BITS)) {
  166. const Number i1 = k >> LEAF_BITS;
  167. Leaf* leaf = root_[i1];
  168. if (leaf != NULL) {
  169. // Scan forward in leaf
  170. for (Number i2 = k & (LEAF_LENGTH - 1); i2 < LEAF_LENGTH; i2++) {
  171. if (leaf->values[i2] != NULL) {
  172. return leaf->values[i2];
  173. }
  174. }
  175. }
  176. // Skip to next top-level entry
  177. k = (i1 + 1) << LEAF_BITS;
  178. }
  179. return NULL;
  180. }
  181. };
  182. // Three-level radix tree
  183. template <int BITS>
  184. class TCMalloc_PageMap3 {
  185. private:
  186. // How many bits should we consume at each interior level
  187. static const int INTERIOR_BITS = (BITS + 2) / 3; // Round-up
  188. static const int INTERIOR_LENGTH = 1 << INTERIOR_BITS;
  189. // How many bits should we consume at leaf level
  190. static const int LEAF_BITS = BITS - 2*INTERIOR_BITS;
  191. static const int LEAF_LENGTH = 1 << LEAF_BITS;
  192. // Interior node
  193. struct Node {
  194. Node* ptrs[INTERIOR_LENGTH];
  195. };
  196. // Leaf node
  197. struct Leaf {
  198. void* values[LEAF_LENGTH];
  199. };
  200. Node root_; // Root of radix tree
  201. void* (*allocator_)(size_t); // Memory allocator
  202. Node* NewNode() {
  203. Node* result = reinterpret_cast<Node*>((*allocator_)(sizeof(Node)));
  204. if (result != NULL) {
  205. memset(result, 0, sizeof(*result));
  206. }
  207. return result;
  208. }
  209. public:
  210. typedef uintptr_t Number;
  211. explicit TCMalloc_PageMap3(void* (*allocator)(size_t)) {
  212. allocator_ = allocator;
  213. memset(&root_, 0, sizeof(root_));
  214. }
  215. ATTRIBUTE_ALWAYS_INLINE
  216. void* get(Number k) const {
  217. const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
  218. const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
  219. const Number i3 = k & (LEAF_LENGTH-1);
  220. if ((k >> BITS) > 0 ||
  221. root_.ptrs[i1] == NULL || root_.ptrs[i1]->ptrs[i2] == NULL) {
  222. return NULL;
  223. }
  224. return reinterpret_cast<Leaf*>(root_.ptrs[i1]->ptrs[i2])->values[i3];
  225. }
  226. void set(Number k, void* v) {
  227. ASSERT(k >> BITS == 0);
  228. const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
  229. const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
  230. const Number i3 = k & (LEAF_LENGTH-1);
  231. reinterpret_cast<Leaf*>(root_.ptrs[i1]->ptrs[i2])->values[i3] = v;
  232. }
  233. bool Ensure(Number start, size_t n) {
  234. for (Number key = start; key <= start + n - 1; ) {
  235. const Number i1 = key >> (LEAF_BITS + INTERIOR_BITS);
  236. const Number i2 = (key >> LEAF_BITS) & (INTERIOR_LENGTH-1);
  237. // Check for overflow
  238. if (i1 >= INTERIOR_LENGTH || i2 >= INTERIOR_LENGTH)
  239. return false;
  240. // Make 2nd level node if necessary
  241. if (root_.ptrs[i1] == NULL) {
  242. Node* n = NewNode();
  243. if (n == NULL) return false;
  244. root_.ptrs[i1] = n;
  245. }
  246. // Make leaf node if necessary
  247. if (root_.ptrs[i1]->ptrs[i2] == NULL) {
  248. Leaf* leaf = reinterpret_cast<Leaf*>((*allocator_)(sizeof(Leaf)));
  249. if (leaf == NULL) return false;
  250. memset(leaf, 0, sizeof(*leaf));
  251. root_.ptrs[i1]->ptrs[i2] = reinterpret_cast<Node*>(leaf);
  252. }
  253. // Advance key past whatever is covered by this leaf node
  254. key = ((key >> LEAF_BITS) + 1) << LEAF_BITS;
  255. }
  256. return true;
  257. }
  258. void PreallocateMoreMemory() {
  259. }
  260. void* Next(Number k) const {
  261. while (k < (Number(1) << BITS)) {
  262. const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
  263. const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
  264. if (root_.ptrs[i1] == NULL) {
  265. // Advance to next top-level entry
  266. k = (i1 + 1) << (LEAF_BITS + INTERIOR_BITS);
  267. } else {
  268. Leaf* leaf = reinterpret_cast<Leaf*>(root_.ptrs[i1]->ptrs[i2]);
  269. if (leaf != NULL) {
  270. for (Number i3 = (k & (LEAF_LENGTH-1)); i3 < LEAF_LENGTH; i3++) {
  271. if (leaf->values[i3] != NULL) {
  272. return leaf->values[i3];
  273. }
  274. }
  275. }
  276. // Advance to next interior entry
  277. k = ((k >> LEAF_BITS) + 1) << LEAF_BITS;
  278. }
  279. }
  280. return NULL;
  281. }
  282. };
  283. #endif // TCMALLOC_PAGEMAP_H_