signal_statistics.hpp 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. // (C) Copyright Nick Thompson 2018.
  2. // Use, modification and distribution are subject to the
  3. // Boost Software License, Version 1.0. (See accompanying file
  4. // LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5. #ifndef BOOST_MATH_TOOLS_SIGNAL_STATISTICS_HPP
  6. #define BOOST_MATH_TOOLS_SIGNAL_STATISTICS_HPP
  7. #include <algorithm>
  8. #include <iterator>
  9. #include <boost/assert.hpp>
  10. #include <boost/math/tools/complex.hpp>
  11. #include <boost/math/tools/roots.hpp>
  12. #include <boost/math/statistics/univariate_statistics.hpp>
  13. #include <boost/config/header_deprecated.hpp>
  14. BOOST_HEADER_DEPRECATED("<boost/math/statistics/signal_statistics.hpp>");
  15. namespace boost::math::tools {
  16. template<class ForwardIterator>
  17. auto absolute_gini_coefficient(ForwardIterator first, ForwardIterator last)
  18. {
  19. using std::abs;
  20. using RealOrComplex = typename std::iterator_traits<ForwardIterator>::value_type;
  21. BOOST_ASSERT_MSG(first != last && std::next(first) != last, "Computation of the Gini coefficient requires at least two samples.");
  22. std::sort(first, last, [](RealOrComplex a, RealOrComplex b) { return abs(b) > abs(a); });
  23. decltype(abs(*first)) i = 1;
  24. decltype(abs(*first)) num = 0;
  25. decltype(abs(*first)) denom = 0;
  26. for (auto it = first; it != last; ++it)
  27. {
  28. decltype(abs(*first)) tmp = abs(*it);
  29. num += tmp*i;
  30. denom += tmp;
  31. ++i;
  32. }
  33. // If the l1 norm is zero, all elements are zero, so every element is the same.
  34. if (denom == 0)
  35. {
  36. decltype(abs(*first)) zero = 0;
  37. return zero;
  38. }
  39. return ((2*num)/denom - i)/(i-1);
  40. }
  41. template<class RandomAccessContainer>
  42. inline auto absolute_gini_coefficient(RandomAccessContainer & v)
  43. {
  44. return boost::math::tools::absolute_gini_coefficient(v.begin(), v.end());
  45. }
  46. template<class ForwardIterator>
  47. auto sample_absolute_gini_coefficient(ForwardIterator first, ForwardIterator last)
  48. {
  49. size_t n = std::distance(first, last);
  50. return n*boost::math::tools::absolute_gini_coefficient(first, last)/(n-1);
  51. }
  52. template<class RandomAccessContainer>
  53. inline auto sample_absolute_gini_coefficient(RandomAccessContainer & v)
  54. {
  55. return boost::math::tools::sample_absolute_gini_coefficient(v.begin(), v.end());
  56. }
  57. // The Hoyer sparsity measure is defined in:
  58. // https://arxiv.org/pdf/0811.4706.pdf
  59. template<class ForwardIterator>
  60. auto hoyer_sparsity(const ForwardIterator first, const ForwardIterator last)
  61. {
  62. using T = typename std::iterator_traits<ForwardIterator>::value_type;
  63. using std::abs;
  64. using std::sqrt;
  65. BOOST_ASSERT_MSG(first != last && std::next(first) != last, "Computation of the Hoyer sparsity requires at least two samples.");
  66. if constexpr (std::is_unsigned<T>::value)
  67. {
  68. T l1 = 0;
  69. T l2 = 0;
  70. size_t n = 0;
  71. for (auto it = first; it != last; ++it)
  72. {
  73. l1 += *it;
  74. l2 += (*it)*(*it);
  75. n += 1;
  76. }
  77. double rootn = sqrt(n);
  78. return (rootn - l1/sqrt(l2) )/ (rootn - 1);
  79. }
  80. else {
  81. decltype(abs(*first)) l1 = 0;
  82. decltype(abs(*first)) l2 = 0;
  83. // We wouldn't need to count the elements if it was a random access iterator,
  84. // but our only constraint is that it's a forward iterator.
  85. size_t n = 0;
  86. for (auto it = first; it != last; ++it)
  87. {
  88. decltype(abs(*first)) tmp = abs(*it);
  89. l1 += tmp;
  90. l2 += tmp*tmp;
  91. n += 1;
  92. }
  93. if constexpr (std::is_integral<T>::value)
  94. {
  95. double rootn = sqrt(n);
  96. return (rootn - l1/sqrt(l2) )/ (rootn - 1);
  97. }
  98. else
  99. {
  100. decltype(abs(*first)) rootn = sqrt(static_cast<decltype(abs(*first))>(n));
  101. return (rootn - l1/sqrt(l2) )/ (rootn - 1);
  102. }
  103. }
  104. }
  105. template<class Container>
  106. inline auto hoyer_sparsity(Container const & v)
  107. {
  108. return boost::math::tools::hoyer_sparsity(v.cbegin(), v.cend());
  109. }
  110. template<class Container>
  111. auto oracle_snr(Container const & signal, Container const & noisy_signal)
  112. {
  113. using Real = typename Container::value_type;
  114. BOOST_ASSERT_MSG(signal.size() == noisy_signal.size(),
  115. "Signal and noisy_signal must be have the same number of elements.");
  116. if constexpr (std::is_integral<Real>::value)
  117. {
  118. double numerator = 0;
  119. double denominator = 0;
  120. for (size_t i = 0; i < signal.size(); ++i)
  121. {
  122. numerator += signal[i]*signal[i];
  123. denominator += (noisy_signal[i] - signal[i])*(noisy_signal[i] - signal[i]);
  124. }
  125. if (numerator == 0 && denominator == 0)
  126. {
  127. return std::numeric_limits<double>::quiet_NaN();
  128. }
  129. if (denominator == 0)
  130. {
  131. return std::numeric_limits<double>::infinity();
  132. }
  133. return numerator/denominator;
  134. }
  135. else if constexpr (boost::math::tools::is_complex_type<Real>::value)
  136. {
  137. using std::norm;
  138. typename Real::value_type numerator = 0;
  139. typename Real::value_type denominator = 0;
  140. for (size_t i = 0; i < signal.size(); ++i)
  141. {
  142. numerator += norm(signal[i]);
  143. denominator += norm(noisy_signal[i] - signal[i]);
  144. }
  145. if (numerator == 0 && denominator == 0)
  146. {
  147. return std::numeric_limits<typename Real::value_type>::quiet_NaN();
  148. }
  149. if (denominator == 0)
  150. {
  151. return std::numeric_limits<typename Real::value_type>::infinity();
  152. }
  153. return numerator/denominator;
  154. }
  155. else
  156. {
  157. Real numerator = 0;
  158. Real denominator = 0;
  159. for (size_t i = 0; i < signal.size(); ++i)
  160. {
  161. numerator += signal[i]*signal[i];
  162. denominator += (signal[i] - noisy_signal[i])*(signal[i] - noisy_signal[i]);
  163. }
  164. if (numerator == 0 && denominator == 0)
  165. {
  166. return std::numeric_limits<Real>::quiet_NaN();
  167. }
  168. if (denominator == 0)
  169. {
  170. return std::numeric_limits<Real>::infinity();
  171. }
  172. return numerator/denominator;
  173. }
  174. }
  175. template<class Container>
  176. auto mean_invariant_oracle_snr(Container const & signal, Container const & noisy_signal)
  177. {
  178. using Real = typename Container::value_type;
  179. BOOST_ASSERT_MSG(signal.size() == noisy_signal.size(), "Signal and noisy signal must be have the same number of elements.");
  180. Real mu = boost::math::tools::mean(signal);
  181. Real numerator = 0;
  182. Real denominator = 0;
  183. for (size_t i = 0; i < signal.size(); ++i)
  184. {
  185. Real tmp = signal[i] - mu;
  186. numerator += tmp*tmp;
  187. denominator += (signal[i] - noisy_signal[i])*(signal[i] - noisy_signal[i]);
  188. }
  189. if (numerator == 0 && denominator == 0)
  190. {
  191. return std::numeric_limits<Real>::quiet_NaN();
  192. }
  193. if (denominator == 0)
  194. {
  195. return std::numeric_limits<Real>::infinity();
  196. }
  197. return numerator/denominator;
  198. }
  199. template<class Container>
  200. auto mean_invariant_oracle_snr_db(Container const & signal, Container const & noisy_signal)
  201. {
  202. using std::log10;
  203. return 10*log10(boost::math::tools::mean_invariant_oracle_snr(signal, noisy_signal));
  204. }
  205. // Follows the definition of SNR given in Mallat, A Wavelet Tour of Signal Processing, equation 11.16.
  206. template<class Container>
  207. auto oracle_snr_db(Container const & signal, Container const & noisy_signal)
  208. {
  209. using std::log10;
  210. return 10*log10(boost::math::tools::oracle_snr(signal, noisy_signal));
  211. }
  212. // A good reference on the M2M4 estimator:
  213. // D. R. Pauluzzi and N. C. Beaulieu, "A comparison of SNR estimation techniques for the AWGN channel," IEEE Trans. Communications, Vol. 48, No. 10, pp. 1681-1691, 2000.
  214. // A nice python implementation:
  215. // https://github.com/gnuradio/gnuradio/blob/master/gr-digital/examples/snr_estimators.py
  216. template<class ForwardIterator>
  217. auto m2m4_snr_estimator(ForwardIterator first, ForwardIterator last, decltype(*first) estimated_signal_kurtosis=1, decltype(*first) estimated_noise_kurtosis=3)
  218. {
  219. BOOST_ASSERT_MSG(estimated_signal_kurtosis > 0, "The estimated signal kurtosis must be positive");
  220. BOOST_ASSERT_MSG(estimated_noise_kurtosis > 0, "The estimated noise kurtosis must be positive.");
  221. using Real = typename std::iterator_traits<ForwardIterator>::value_type;
  222. using std::sqrt;
  223. if constexpr (std::is_floating_point<Real>::value || std::numeric_limits<Real>::max_exponent)
  224. {
  225. // If we first eliminate N, we obtain the quadratic equation:
  226. // (ka+kw-6)S^2 + 2M2(3-kw)S + kw*M2^2 - M4 = 0 =: a*S^2 + bs*N + cs = 0
  227. // If we first eliminate S, we obtain the quadratic equation:
  228. // (ka+kw-6)N^2 + 2M2(3-ka)N + ka*M2^2 - M4 = 0 =: a*N^2 + bn*N + cn = 0
  229. // I believe these equations are totally independent quadratics;
  230. // if one has a complex solution it is not necessarily the case that the other must also.
  231. // However, I can't prove that, so there is a chance that this does unnecessary work.
  232. // Future improvements: There are algorithms which can solve quadratics much more effectively than the naive implementation found here.
  233. // See: https://stackoverflow.com/questions/48979861/numerically-stable-method-for-solving-quadratic-equations/50065711#50065711
  234. auto [M1, M2, M3, M4] = boost::math::tools::first_four_moments(first, last);
  235. if (M4 == 0)
  236. {
  237. // The signal is constant. There is no noise:
  238. return std::numeric_limits<Real>::infinity();
  239. }
  240. // Change to notation in Pauluzzi, equation 41:
  241. auto kw = estimated_noise_kurtosis;
  242. auto ka = estimated_signal_kurtosis;
  243. // A common case, since it's the default:
  244. Real a = (ka+kw-6);
  245. Real bs = 2*M2*(3-kw);
  246. Real cs = kw*M2*M2 - M4;
  247. Real bn = 2*M2*(3-ka);
  248. Real cn = ka*M2*M2 - M4;
  249. auto [S0, S1] = boost::math::tools::quadratic_roots(a, bs, cs);
  250. if (S1 > 0)
  251. {
  252. auto N = M2 - S1;
  253. if (N > 0)
  254. {
  255. return S1/N;
  256. }
  257. if (S0 > 0)
  258. {
  259. N = M2 - S0;
  260. if (N > 0)
  261. {
  262. return S0/N;
  263. }
  264. }
  265. }
  266. auto [N0, N1] = boost::math::tools::quadratic_roots(a, bn, cn);
  267. if (N1 > 0)
  268. {
  269. auto S = M2 - N1;
  270. if (S > 0)
  271. {
  272. return S/N1;
  273. }
  274. if (N0 > 0)
  275. {
  276. S = M2 - N0;
  277. if (S > 0)
  278. {
  279. return S/N0;
  280. }
  281. }
  282. }
  283. // This happens distressingly often. It's a limitation of the method.
  284. return std::numeric_limits<Real>::quiet_NaN();
  285. }
  286. else
  287. {
  288. BOOST_ASSERT_MSG(false, "The M2M4 estimator has not been implemented for this type.");
  289. return std::numeric_limits<Real>::quiet_NaN();
  290. }
  291. }
  292. template<class Container>
  293. inline auto m2m4_snr_estimator(Container const & noisy_signal, typename Container::value_type estimated_signal_kurtosis=1, typename Container::value_type estimated_noise_kurtosis=3)
  294. {
  295. return m2m4_snr_estimator(noisy_signal.cbegin(), noisy_signal.cend(), estimated_signal_kurtosis, estimated_noise_kurtosis);
  296. }
  297. template<class ForwardIterator>
  298. inline auto m2m4_snr_estimator_db(ForwardIterator first, ForwardIterator last, decltype(*first) estimated_signal_kurtosis=1, decltype(*first) estimated_noise_kurtosis=3)
  299. {
  300. using std::log10;
  301. return 10*log10(m2m4_snr_estimator(first, last, estimated_signal_kurtosis, estimated_noise_kurtosis));
  302. }
  303. template<class Container>
  304. inline auto m2m4_snr_estimator_db(Container const & noisy_signal, typename Container::value_type estimated_signal_kurtosis=1, typename Container::value_type estimated_noise_kurtosis=3)
  305. {
  306. using std::log10;
  307. return 10*log10(m2m4_snr_estimator(noisy_signal, estimated_signal_kurtosis, estimated_noise_kurtosis));
  308. }
  309. }
  310. #endif