extra_ops_generic.hpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. /*
  2. * Distributed under the Boost Software License, Version 1.0.
  3. * (See accompanying file LICENSE_1_0.txt or copy at
  4. * http://www.boost.org/LICENSE_1_0.txt)
  5. *
  6. * Copyright (c) 2015 Andrey Semashev
  7. */
  8. /*!
  9. * \file atomic/detail/extra_ops_generic.hpp
  10. *
  11. * This header contains generic implementation of the extra atomic operations.
  12. */
  13. #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_
  14. #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_
  15. #include <cstddef>
  16. #include <boost/memory_order.hpp>
  17. #include <boost/atomic/detail/config.hpp>
  18. #include <boost/atomic/detail/storage_traits.hpp>
  19. #include <boost/atomic/detail/integral_conversions.hpp>
  20. #include <boost/atomic/detail/extra_operations_fwd.hpp>
  21. #include <boost/atomic/detail/header.hpp>
  22. #ifdef BOOST_HAS_PRAGMA_ONCE
  23. #pragma once
  24. #endif
  25. namespace boost {
  26. namespace atomics {
  27. namespace detail {
  28. //! Generic implementation of extra operations
  29. template< typename Base, std::size_t Size, bool Signed, bool = Base::full_cas_based >
  30. struct extra_operations_generic :
  31. public Base
  32. {
  33. typedef Base base_type;
  34. typedef typename base_type::storage_type storage_type;
  35. typedef typename storage_traits< Size >::type emulated_storage_type;
  36. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  37. {
  38. storage_type old_val;
  39. atomics::detail::non_atomic_load(storage, old_val);
  40. while (!base_type::compare_exchange_weak(storage, old_val, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val)), order, memory_order_relaxed)) {}
  41. return old_val;
  42. }
  43. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  44. {
  45. storage_type old_val, new_val;
  46. atomics::detail::non_atomic_load(storage, old_val);
  47. do
  48. {
  49. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val));
  50. }
  51. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  52. return new_val;
  53. }
  54. static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  55. {
  56. return base_type::fetch_add(storage, v, order) + v;
  57. }
  58. static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  59. {
  60. return base_type::fetch_sub(storage, v, order) - v;
  61. }
  62. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  63. {
  64. return base_type::fetch_and(storage, v, order) & v;
  65. }
  66. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  67. {
  68. return base_type::fetch_or(storage, v, order) | v;
  69. }
  70. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  71. {
  72. return base_type::fetch_xor(storage, v, order) ^ v;
  73. }
  74. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  75. {
  76. return base_type::fetch_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  77. }
  78. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  79. {
  80. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u)));
  81. return base_type::fetch_xor(storage, mask, order) ^ mask;
  82. }
  83. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  84. {
  85. base_type::fetch_add(storage, v, order);
  86. }
  87. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  88. {
  89. base_type::fetch_sub(storage, v, order);
  90. }
  91. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  92. {
  93. fetch_negate(storage, order);
  94. }
  95. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  96. {
  97. base_type::fetch_and(storage, v, order);
  98. }
  99. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  100. {
  101. base_type::fetch_or(storage, v, order);
  102. }
  103. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  104. {
  105. base_type::fetch_xor(storage, v, order);
  106. }
  107. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  108. {
  109. fetch_complement(storage, order);
  110. }
  111. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  112. {
  113. return !!static_cast< emulated_storage_type >(add(storage, v, order));
  114. }
  115. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  116. {
  117. return !!static_cast< emulated_storage_type >(sub(storage, v, order));
  118. }
  119. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  120. {
  121. return !!negate(storage, order);
  122. }
  123. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  124. {
  125. return !!bitwise_and(storage, v, order);
  126. }
  127. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  128. {
  129. return !!bitwise_or(storage, v, order);
  130. }
  131. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  132. {
  133. return !!bitwise_xor(storage, v, order);
  134. }
  135. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  136. {
  137. return !!static_cast< emulated_storage_type >(bitwise_complement(storage, order));
  138. }
  139. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  140. {
  141. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  142. storage_type old_val = base_type::fetch_or(storage, mask, order);
  143. return !!(old_val & mask);
  144. }
  145. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  146. {
  147. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  148. storage_type old_val = base_type::fetch_and(storage, ~mask, order);
  149. return !!(old_val & mask);
  150. }
  151. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  152. {
  153. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  154. storage_type old_val = base_type::fetch_xor(storage, mask, order);
  155. return !!(old_val & mask);
  156. }
  157. };
  158. //! Specialization for cases when the platform only natively supports CAS
  159. template< typename Base, std::size_t Size, bool Signed >
  160. struct extra_operations_generic< Base, Size, Signed, true > :
  161. public Base
  162. {
  163. typedef Base base_type;
  164. typedef typename base_type::storage_type storage_type;
  165. typedef typename storage_traits< Size >::type emulated_storage_type;
  166. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  167. {
  168. storage_type old_val;
  169. atomics::detail::non_atomic_load(storage, old_val);
  170. while (!base_type::compare_exchange_weak(storage, old_val, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val)), order, memory_order_relaxed)) {}
  171. return old_val;
  172. }
  173. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  174. {
  175. storage_type old_val, new_val;
  176. atomics::detail::non_atomic_load(storage, old_val);
  177. do
  178. {
  179. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val));
  180. }
  181. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  182. return new_val;
  183. }
  184. static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  185. {
  186. storage_type old_val, new_val;
  187. atomics::detail::non_atomic_load(storage, old_val);
  188. do
  189. {
  190. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val + v));
  191. }
  192. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  193. return new_val;
  194. }
  195. static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  196. {
  197. storage_type old_val, new_val;
  198. atomics::detail::non_atomic_load(storage, old_val);
  199. do
  200. {
  201. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val - v));
  202. }
  203. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  204. return new_val;
  205. }
  206. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  207. {
  208. storage_type old_val, new_val;
  209. atomics::detail::non_atomic_load(storage, old_val);
  210. do
  211. {
  212. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val & v));
  213. }
  214. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  215. return new_val;
  216. }
  217. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  218. {
  219. storage_type old_val, new_val;
  220. atomics::detail::non_atomic_load(storage, old_val);
  221. do
  222. {
  223. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val | v));
  224. }
  225. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  226. return new_val;
  227. }
  228. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  229. {
  230. storage_type old_val, new_val;
  231. atomics::detail::non_atomic_load(storage, old_val);
  232. do
  233. {
  234. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val ^ v));
  235. }
  236. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  237. return new_val;
  238. }
  239. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  240. {
  241. return base_type::fetch_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  242. }
  243. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  244. {
  245. return bitwise_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  246. }
  247. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  248. {
  249. base_type::fetch_add(storage, v, order);
  250. }
  251. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  252. {
  253. base_type::fetch_sub(storage, v, order);
  254. }
  255. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  256. {
  257. fetch_negate(storage, order);
  258. }
  259. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  260. {
  261. base_type::fetch_and(storage, v, order);
  262. }
  263. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  264. {
  265. base_type::fetch_or(storage, v, order);
  266. }
  267. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  268. {
  269. base_type::fetch_xor(storage, v, order);
  270. }
  271. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  272. {
  273. fetch_complement(storage, order);
  274. }
  275. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  276. {
  277. return !!static_cast< emulated_storage_type >(add(storage, v, order));
  278. }
  279. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  280. {
  281. return !!static_cast< emulated_storage_type >(sub(storage, v, order));
  282. }
  283. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  284. {
  285. return !!negate(storage, order);
  286. }
  287. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  288. {
  289. return !!bitwise_and(storage, v, order);
  290. }
  291. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  292. {
  293. return !!bitwise_or(storage, v, order);
  294. }
  295. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  296. {
  297. return !!bitwise_xor(storage, v, order);
  298. }
  299. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  300. {
  301. return !!static_cast< emulated_storage_type >(bitwise_complement(storage, order));
  302. }
  303. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  304. {
  305. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  306. storage_type old_val = base_type::fetch_or(storage, mask, order);
  307. return !!(old_val & mask);
  308. }
  309. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  310. {
  311. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  312. storage_type old_val = base_type::fetch_and(storage, ~mask, order);
  313. return !!(old_val & mask);
  314. }
  315. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  316. {
  317. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  318. storage_type old_val = base_type::fetch_xor(storage, mask, order);
  319. return !!(old_val & mask);
  320. }
  321. };
  322. // Default extra_operations template definition will be used unless specialized for a specific platform
  323. template< typename Base, std::size_t Size, bool Signed >
  324. struct extra_operations< Base, Size, Signed, true > :
  325. public extra_operations_generic< Base, Size, Signed >
  326. {
  327. };
  328. } // namespace detail
  329. } // namespace atomics
  330. } // namespace boost
  331. #include <boost/atomic/detail/footer.hpp>
  332. #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_