bits.h 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. // Copyright 2018 The Abseil Authors.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // https://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. #ifndef ABSL_BASE_INTERNAL_BITS_H_
  15. #define ABSL_BASE_INTERNAL_BITS_H_
  16. // This file contains bitwise ops which are implementation details of various
  17. // absl libraries.
  18. #include <cstdint>
  19. #include "absl/base/config.h"
  20. // Clang on Windows has __builtin_clzll; otherwise we need to use the
  21. // windows intrinsic functions.
  22. #if defined(_MSC_VER) && !defined(__clang__)
  23. #include <intrin.h>
  24. #if defined(_M_X64)
  25. #pragma intrinsic(_BitScanReverse64)
  26. #pragma intrinsic(_BitScanForward64)
  27. #endif
  28. #pragma intrinsic(_BitScanReverse)
  29. #pragma intrinsic(_BitScanForward)
  30. #endif
  31. #include "absl/base/attributes.h"
  32. #if defined(_MSC_VER) && !defined(__clang__)
  33. // We can achieve something similar to attribute((always_inline)) with MSVC by
  34. // using the __forceinline keyword, however this is not perfect. MSVC is
  35. // much less aggressive about inlining, and even with the __forceinline keyword.
  36. #define ABSL_BASE_INTERNAL_FORCEINLINE __forceinline
  37. #else
  38. // Use default attribute inline.
  39. #define ABSL_BASE_INTERNAL_FORCEINLINE inline ABSL_ATTRIBUTE_ALWAYS_INLINE
  40. #endif
  41. namespace absl {
  42. ABSL_NAMESPACE_BEGIN
  43. namespace base_internal {
  44. ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) {
  45. int zeroes = 60;
  46. if (n >> 32) {
  47. zeroes -= 32;
  48. n >>= 32;
  49. }
  50. if (n >> 16) {
  51. zeroes -= 16;
  52. n >>= 16;
  53. }
  54. if (n >> 8) {
  55. zeroes -= 8;
  56. n >>= 8;
  57. }
  58. if (n >> 4) {
  59. zeroes -= 4;
  60. n >>= 4;
  61. }
  62. return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
  63. }
  64. ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) {
  65. #if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64)
  66. // MSVC does not have __buitin_clzll. Use _BitScanReverse64.
  67. unsigned long result = 0; // NOLINT(runtime/int)
  68. if (_BitScanReverse64(&result, n)) {
  69. return 63 - result;
  70. }
  71. return 64;
  72. #elif defined(_MSC_VER) && !defined(__clang__)
  73. // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse
  74. unsigned long result = 0; // NOLINT(runtime/int)
  75. if ((n >> 32) &&
  76. _BitScanReverse(&result, static_cast<unsigned long>(n >> 32))) {
  77. return 31 - result;
  78. }
  79. if (_BitScanReverse(&result, static_cast<unsigned long>(n))) {
  80. return 63 - result;
  81. }
  82. return 64;
  83. #elif defined(__GNUC__) || defined(__clang__)
  84. // Use __builtin_clzll, which uses the following instructions:
  85. // x86: bsr
  86. // ARM64: clz
  87. // PPC: cntlzd
  88. static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
  89. "__builtin_clzll does not take 64-bit arg");
  90. // Handle 0 as a special case because __builtin_clzll(0) is undefined.
  91. if (n == 0) {
  92. return 64;
  93. }
  94. return __builtin_clzll(n);
  95. #else
  96. return CountLeadingZeros64Slow(n);
  97. #endif
  98. }
  99. ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) {
  100. int zeroes = 28;
  101. if (n >> 16) {
  102. zeroes -= 16;
  103. n >>= 16;
  104. }
  105. if (n >> 8) {
  106. zeroes -= 8;
  107. n >>= 8;
  108. }
  109. if (n >> 4) {
  110. zeroes -= 4;
  111. n >>= 4;
  112. }
  113. return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
  114. }
  115. ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) {
  116. #if defined(_MSC_VER) && !defined(__clang__)
  117. unsigned long result = 0; // NOLINT(runtime/int)
  118. if (_BitScanReverse(&result, n)) {
  119. return 31 - result;
  120. }
  121. return 32;
  122. #elif defined(__GNUC__) || defined(__clang__)
  123. // Use __builtin_clz, which uses the following instructions:
  124. // x86: bsr
  125. // ARM64: clz
  126. // PPC: cntlzd
  127. static_assert(sizeof(int) == sizeof(n),
  128. "__builtin_clz does not take 32-bit arg");
  129. // Handle 0 as a special case because __builtin_clz(0) is undefined.
  130. if (n == 0) {
  131. return 32;
  132. }
  133. return __builtin_clz(n);
  134. #else
  135. return CountLeadingZeros32Slow(n);
  136. #endif
  137. }
  138. ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) {
  139. int c = 63;
  140. n &= ~n + 1;
  141. if (n & 0x00000000FFFFFFFF) c -= 32;
  142. if (n & 0x0000FFFF0000FFFF) c -= 16;
  143. if (n & 0x00FF00FF00FF00FF) c -= 8;
  144. if (n & 0x0F0F0F0F0F0F0F0F) c -= 4;
  145. if (n & 0x3333333333333333) c -= 2;
  146. if (n & 0x5555555555555555) c -= 1;
  147. return c;
  148. }
  149. ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) {
  150. #if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64)
  151. unsigned long result = 0; // NOLINT(runtime/int)
  152. _BitScanForward64(&result, n);
  153. return result;
  154. #elif defined(_MSC_VER) && !defined(__clang__)
  155. unsigned long result = 0; // NOLINT(runtime/int)
  156. if (static_cast<uint32_t>(n) == 0) {
  157. _BitScanForward(&result, static_cast<unsigned long>(n >> 32));
  158. return result + 32;
  159. }
  160. _BitScanForward(&result, static_cast<unsigned long>(n));
  161. return result;
  162. #elif defined(__GNUC__) || defined(__clang__)
  163. static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
  164. "__builtin_ctzll does not take 64-bit arg");
  165. return __builtin_ctzll(n);
  166. #else
  167. return CountTrailingZerosNonZero64Slow(n);
  168. #endif
  169. }
  170. ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) {
  171. int c = 31;
  172. n &= ~n + 1;
  173. if (n & 0x0000FFFF) c -= 16;
  174. if (n & 0x00FF00FF) c -= 8;
  175. if (n & 0x0F0F0F0F) c -= 4;
  176. if (n & 0x33333333) c -= 2;
  177. if (n & 0x55555555) c -= 1;
  178. return c;
  179. }
  180. ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) {
  181. #if defined(_MSC_VER) && !defined(__clang__)
  182. unsigned long result = 0; // NOLINT(runtime/int)
  183. _BitScanForward(&result, n);
  184. return result;
  185. #elif defined(__GNUC__) || defined(__clang__)
  186. static_assert(sizeof(int) == sizeof(n),
  187. "__builtin_ctz does not take 32-bit arg");
  188. return __builtin_ctz(n);
  189. #else
  190. return CountTrailingZerosNonZero32Slow(n);
  191. #endif
  192. }
  193. #undef ABSL_BASE_INTERNAL_FORCEINLINE
  194. } // namespace base_internal
  195. ABSL_NAMESPACE_END
  196. } // namespace absl
  197. #endif // ABSL_BASE_INTERNAL_BITS_H_