21 #ifndef ROCRAND_NORMAL_H_
22 #define ROCRAND_NORMAL_H_
29 #include "rocrand/rocrand_lfsr113.h"
30 #include "rocrand/rocrand_mrg31k3p.h"
31 #include "rocrand/rocrand_mrg32k3a.h"
32 #include "rocrand/rocrand_mtgp32.h"
33 #include "rocrand/rocrand_philox4x32_10.h"
34 #include "rocrand/rocrand_scrambled_sobol32.h"
35 #include "rocrand/rocrand_scrambled_sobol64.h"
36 #include "rocrand/rocrand_sobol32.h"
37 #include "rocrand/rocrand_sobol64.h"
38 #include "rocrand/rocrand_threefry2x32_20.h"
39 #include "rocrand/rocrand_threefry2x64_20.h"
40 #include "rocrand/rocrand_threefry4x32_20.h"
41 #include "rocrand/rocrand_threefry4x64_20.h"
42 #include "rocrand/rocrand_xorwow.h"
44 #include "rocrand/rocrand_uniform.h"
46 #include <hip/hip_runtime.h>
50 namespace rocrand_device
55 __forceinline__ __device__ __host__
56 float2 box_muller(
unsigned int x,
unsigned int y)
59 float u = fmaf(x, ROCRAND_2POW32_INV, ROCRAND_2POW32_INV);
60 float v = fmaf(y, ROCRAND_2POW32_INV_2PI, ROCRAND_2POW32_INV_2PI);
62 float s = sqrtf(-2.0f * logf(u));
63 #ifdef __HIP_DEVICE_COMPILE__
64 __sincosf(v, &result.x, &result.y);
68 result.x = sinf(v) * s;
69 result.y = cosf(v) * s;
74 __forceinline__ __device__ __host__
75 float2 box_muller(
unsigned long long v)
77 unsigned int x =
static_cast<unsigned int>(v);
78 unsigned int y =
static_cast<unsigned int>(v >> 32);
80 return box_muller(x, y);
83 __forceinline__ __device__ __host__
84 double2 box_muller_double(uint4 v)
87 unsigned long long int v1
88 = (
unsigned long long int)v.x ^ ((
unsigned long long int)v.y << (53 - 32));
89 double u = fma(
static_cast<double>(v1), ROCRAND_2POW53_INV_DOUBLE, ROCRAND_2POW53_INV_DOUBLE);
90 unsigned long long int v2
91 = (
unsigned long long int)v.z ^ ((
unsigned long long int)v.w << (53 - 32));
92 double w = fma(
static_cast<double>(v2),
93 ROCRAND_2POW53_INV_DOUBLE * 2.0,
94 ROCRAND_2POW53_INV_DOUBLE * 2.0);
95 double s = sqrt(-2.0 * log(u));
96 #ifdef __HIP_DEVICE_COMPILE__
97 sincospi(w, &result.x, &result.y);
101 result.x = sin(w * ROCRAND_PI_DOUBLE) * s;
102 result.y = cos(w * ROCRAND_PI_DOUBLE) * s;
107 __forceinline__ __device__ __host__
108 double2 box_muller_double(ulonglong2 v)
110 unsigned int x =
static_cast<unsigned int>(v.x);
111 unsigned int y =
static_cast<unsigned int>(v.x >> 32);
112 unsigned int z =
static_cast<unsigned int>(v.y);
113 unsigned int w =
static_cast<unsigned int>(v.y >> 32);
115 return box_muller_double(make_uint4(x, y, z, w));
118 __forceinline__ __device__ __host__
119 __half2 box_muller_half(
unsigned short x,
unsigned short y)
121 #if defined(ROCRAND_HALF_MATH_SUPPORTED)
122 __half u = __float2half(fmaf(x, ROCRAND_2POW16_INV, ROCRAND_2POW16_INV));
123 __half v = __float2half(fmaf(y, ROCRAND_2POW16_INV_2PI, ROCRAND_2POW16_INV_2PI));
124 __half s = hsqrt(__hmul(__float2half(-2.0f), hlog(u)));
125 return __half2{__hmul(hsin(v), s), __hmul(hcos(v), s)};
128 float u = fmaf(x, ROCRAND_2POW16_INV, ROCRAND_2POW16_INV);
129 float v = fmaf(y, ROCRAND_2POW16_INV_2PI, ROCRAND_2POW16_INV_2PI);
130 float s = sqrtf(-2.0f * logf(u));
131 #ifdef __HIP_DEVICE_COMPILE__
132 __sincosf(v, &r.x, &r.y);
139 return __half2{__float2half(r.x), __float2half(r.y)};
143 template<
typename state_type>
144 __forceinline__ __device__ __host__
145 float2 mrg_box_muller(
unsigned int x,
unsigned int y)
148 float u = rocrand_device::detail::mrg_uniform_distribution<state_type>(x);
149 float v = rocrand_device::detail::mrg_uniform_distribution<state_type>(y) * ROCRAND_2PI;
150 float s = sqrtf(-2.0f * logf(u));
151 #ifdef __HIP_DEVICE_COMPILE__
152 __sincosf(v, &result.x, &result.y);
156 result.x = sinf(v) * s;
157 result.y = cosf(v) * s;
162 template<
typename state_type>
163 __forceinline__ __device__ __host__
164 double2 mrg_box_muller_double(
unsigned int x,
unsigned int y)
167 double u = rocrand_device::detail::mrg_uniform_distribution<state_type>(x);
168 double v = rocrand_device::detail::mrg_uniform_distribution<state_type>(y) * 2.0;
169 double s = sqrt(-2.0 * log(u));
170 #ifdef __HIP_DEVICE_COMPILE__
171 sincospi(v, &result.x, &result.y);
175 result.x = sin(v * ROCRAND_PI_DOUBLE) * s;
176 result.y = cos(v * ROCRAND_PI_DOUBLE) * s;
181 __forceinline__ __device__ __host__
182 float roc_f_erfinv(
float x)
184 float tt1, tt2, lnx, sgn;
185 sgn = (x < 0.0f) ? -1.0f : 1.0f;
187 x = (1.0f - x) * (1.0f + x);
190 #ifdef __HIP_DEVICE_COMPILE__
196 #ifdef __HIP_DEVICE_COMPILE__
199 else if(std::isinf(lnx))
203 tt1 = 2.0f / (ROCRAND_PI * 0.147f) + 0.5f * lnx;
204 tt2 = 1.0f / (0.147f) * lnx;
206 return (sgn * sqrtf(-tt1 + sqrtf(tt1 * tt1 - tt2)));
209 __forceinline__ __device__ __host__
210 double roc_d_erfinv(
double x)
212 double tt1, tt2, lnx, sgn;
213 sgn = (x < 0.0) ? -1.0 : 1.0;
215 x = (1.0 - x) * (1.0 + x);
218 #ifdef __HIP_DEVICE_COMPILE__
224 #ifdef __HIP_DEVICE_COMPILE__
227 else if(std::isinf(lnx))
231 tt1 = 2.0 / (ROCRAND_PI_DOUBLE * 0.147) + 0.5 * lnx;
232 tt2 = 1.0 / (0.147) * lnx;
234 return (sgn * sqrt(-tt1 + sqrt(tt1 * tt1 - tt2)));
237 __forceinline__ __device__ __host__
238 float normal_distribution(
unsigned int x)
240 float p = ::rocrand_device::detail::uniform_distribution(x);
241 float v = ROCRAND_SQRT2 * ::rocrand_device::detail::roc_f_erfinv(2.0f * p - 1.0f);
245 __forceinline__ __device__ __host__
246 float normal_distribution(
unsigned long long int x)
248 float p = ::rocrand_device::detail::uniform_distribution(x);
249 float v = ROCRAND_SQRT2 * ::rocrand_device::detail::roc_f_erfinv(2.0f * p - 1.0f);
253 __forceinline__ __device__ __host__
254 float2 normal_distribution2(
unsigned int v1,
unsigned int v2)
256 return ::rocrand_device::detail::box_muller(v1, v2);
259 __forceinline__ __device__ __host__
260 float2 normal_distribution2(uint2 v)
262 return ::rocrand_device::detail::box_muller(v.x, v.y);
265 __forceinline__ __device__ __host__
266 float2 normal_distribution2(
unsigned long long v)
268 return ::rocrand_device::detail::box_muller(v);
271 __forceinline__ __device__ __host__
272 float4 normal_distribution4(uint4 v)
274 float2 r1 = ::rocrand_device::detail::box_muller(v.x, v.y);
275 float2 r2 = ::rocrand_device::detail::box_muller(v.z, v.w);
276 return float4{r1.x, r1.y, r2.x, r2.y};
279 __forceinline__ __device__ __host__
280 float4 normal_distribution4(longlong2 v)
282 float2 r1 = ::rocrand_device::detail::box_muller(v.x);
283 float2 r2 = ::rocrand_device::detail::box_muller(v.y);
284 return float4{r1.x, r1.y, r2.x, r2.y};
287 __forceinline__ __device__ __host__
288 float4 normal_distribution4(
unsigned long long v1,
unsigned long long v2)
290 float2 r1 = ::rocrand_device::detail::box_muller(v1);
291 float2 r2 = ::rocrand_device::detail::box_muller(v2);
292 return float4{r1.x, r1.y, r2.x, r2.y};
295 __forceinline__ __device__ __host__
296 double normal_distribution_double(
unsigned int x)
298 double p = ::rocrand_device::detail::uniform_distribution_double(x);
299 double v = ROCRAND_SQRT2 * ::rocrand_device::detail::roc_d_erfinv(2.0 * p - 1.0);
303 __forceinline__ __device__ __host__
304 double normal_distribution_double(
unsigned long long int x)
306 double p = ::rocrand_device::detail::uniform_distribution_double(x);
307 double v = ROCRAND_SQRT2 * ::rocrand_device::detail::roc_d_erfinv(2.0 * p - 1.0);
311 __forceinline__ __device__ __host__
312 double2 normal_distribution_double2(uint4 v)
314 return ::rocrand_device::detail::box_muller_double(v);
317 __forceinline__ __device__ __host__
318 double2 normal_distribution_double2(ulonglong2 v)
320 return ::rocrand_device::detail::box_muller_double(v);
323 __forceinline__ __device__ __host__
324 __half2 normal_distribution_half2(
unsigned int v)
326 return ::rocrand_device::detail::box_muller_half(
static_cast<unsigned short>(v),
327 static_cast<unsigned short>(v >> 16));
330 __forceinline__ __device__ __host__
331 __half2 normal_distribution_half2(
unsigned long long v)
333 return ::rocrand_device::detail::box_muller_half(
static_cast<unsigned short>(v),
334 static_cast<unsigned short>(v >> 32));
337 template<
typename state_type>
338 __forceinline__ __device__ __host__
339 float2 mrg_normal_distribution2(
unsigned int v1,
unsigned int v2)
341 return ::rocrand_device::detail::mrg_box_muller<state_type>(v1, v2);
344 template<
typename state_type>
345 __forceinline__ __device__ __host__
346 double2 mrg_normal_distribution_double2(
unsigned int v1,
unsigned int v2)
348 return ::rocrand_device::detail::mrg_box_muller_double<state_type>(v1, v2);
351 template<
typename state_type>
352 __forceinline__ __device__ __host__
353 __half2 mrg_normal_distribution_half2(
unsigned int v)
355 v = rocrand_device::detail::mrg_uniform_distribution_uint<state_type>(v);
356 return ::rocrand_device::detail::box_muller_half(
static_cast<unsigned short>(v),
357 static_cast<unsigned short>(v >> 16));
377 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
378 __forceinline__ __device__ __host__
381 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_philox4x32_10> bm_helper;
383 if(bm_helper::has_float(state))
385 return bm_helper::get_float(state);
391 float2 r = rocrand_device::detail::normal_distribution2(state1, state2);
392 bm_helper::save_float(state, r.y);
411 __forceinline__ __device__ __host__
417 return rocrand_device::detail::normal_distribution2(state1, state2);
434 __forceinline__ __device__ __host__
437 return rocrand_device::detail::normal_distribution4(
rocrand4(state));
454 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
455 __forceinline__ __device__ __host__
458 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_philox4x32_10> bm_helper;
460 if(bm_helper::has_double(state))
462 return bm_helper::get_double(state);
464 double2 r = rocrand_device::detail::normal_distribution_double2(
rocrand4(state));
465 bm_helper::save_double(state, r.y);
484 __forceinline__ __device__ __host__
487 return rocrand_device::detail::normal_distribution_double2(
rocrand4(state));
504 __forceinline__ __device__ __host__
508 r1 = rocrand_device::detail::normal_distribution_double2(
rocrand4(state));
509 r2 = rocrand_device::detail::normal_distribution_double2(
rocrand4(state));
510 return double4{r1.x, r1.y, r2.x, r2.y};
527 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
528 __forceinline__ __device__ __host__
531 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_mrg31k3p> bm_helper;
533 if(bm_helper::has_float(state))
535 return bm_helper::get_float(state);
538 auto state1 = state->next();
539 auto state2 = state->next();
542 = rocrand_device::detail::mrg_normal_distribution2<rocrand_state_mrg31k3p>(state1, state2);
543 bm_helper::save_float(state, r.y);
562 __forceinline__ __device__ __host__
565 auto state1 = state->next();
566 auto state2 = state->next();
568 return rocrand_device::detail::mrg_normal_distribution2<rocrand_state_mrg31k3p>(state1, state2);
585 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
586 __forceinline__ __device__ __host__
589 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_mrg31k3p> bm_helper;
591 if(bm_helper::has_double(state))
593 return bm_helper::get_double(state);
596 auto state1 = state->next();
597 auto state2 = state->next();
600 = rocrand_device::detail::mrg_normal_distribution_double2<rocrand_state_mrg31k3p>(state1,
602 bm_helper::save_double(state, r.y);
621 __forceinline__ __device__ __host__
624 auto state1 = state->next();
625 auto state2 = state->next();
627 return rocrand_device::detail::mrg_normal_distribution_double2<rocrand_state_mrg31k3p>(state1,
645 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
646 __forceinline__ __device__ __host__
649 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_mrg32k3a> bm_helper;
651 if(bm_helper::has_float(state))
653 return bm_helper::get_float(state);
656 auto state1 = state->next();
657 auto state2 = state->next();
660 = rocrand_device::detail::mrg_normal_distribution2<rocrand_state_mrg32k3a>(state1, state2);
661 bm_helper::save_float(state, r.y);
680 __forceinline__ __device__ __host__
683 auto state1 = state->next();
684 auto state2 = state->next();
686 return rocrand_device::detail::mrg_normal_distribution2<rocrand_state_mrg32k3a>(state1, state2);
703 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
704 __forceinline__ __device__ __host__
707 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_mrg32k3a> bm_helper;
709 if(bm_helper::has_double(state))
711 return bm_helper::get_double(state);
714 auto state1 = state->next();
715 auto state2 = state->next();
718 = rocrand_device::detail::mrg_normal_distribution_double2<rocrand_state_mrg32k3a>(state1,
720 bm_helper::save_double(state, r.y);
739 __forceinline__ __device__ __host__
742 auto state1 = state->next();
743 auto state2 = state->next();
745 return rocrand_device::detail::mrg_normal_distribution_double2<rocrand_state_mrg32k3a>(state1,
763 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
764 __forceinline__ __device__ __host__
767 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_xorwow> bm_helper;
769 if(bm_helper::has_float(state))
771 return bm_helper::get_float(state);
775 float2 r = rocrand_device::detail::normal_distribution2(state1, state2);
776 bm_helper::save_float(state, r.y);
795 __forceinline__ __device__ __host__
800 return rocrand_device::detail::normal_distribution2(state1, state2);
817 #ifndef ROCRAND_DETAIL_BM_NOT_IN_STATE
818 __forceinline__ __device__ __host__
821 typedef rocrand_device::detail::engine_boxmuller_helper<rocrand_state_xorwow> bm_helper;
823 if(bm_helper::has_double(state))
825 return bm_helper::get_double(state);
833 double2 r = rocrand_device::detail::normal_distribution_double2(
834 uint4{state1, state2, state3, state4});
835 bm_helper::save_double(state, r.y);
854 __forceinline__ __device__ __host__
862 return rocrand_device::detail::normal_distribution_double2(
863 uint4{state1, state2, state3, state4});
878 __forceinline__ __device__
881 return rocrand_device::detail::normal_distribution(
rocrand(state));
898 __forceinline__ __device__
903 return rocrand_device::detail::normal_distribution2(state1, state2);
918 __forceinline__ __device__
921 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
938 __forceinline__ __device__
946 return rocrand_device::detail::normal_distribution_double2(
947 uint4{state1, state2, state3, state4});
962 __forceinline__ __device__ __host__
965 return rocrand_device::detail::normal_distribution(
rocrand(state));
980 __forceinline__ __device__ __host__
983 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
998 __forceinline__ __device__ __host__
1001 return rocrand_device::detail::normal_distribution(
rocrand(state));
1016 __forceinline__ __device__ __host__
1019 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1034 __forceinline__ __device__ __host__
1037 return rocrand_device::detail::normal_distribution(
rocrand(state));
1052 __forceinline__ __device__ __host__
1055 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1070 __forceinline__ __device__ __host__
1073 return rocrand_device::detail::normal_distribution(
rocrand(state));
1088 __forceinline__ __device__ __host__
1091 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1106 __forceinline__ __device__ __host__
1109 return rocrand_device::detail::normal_distribution(
rocrand(state));
1126 __forceinline__ __device__ __host__
1132 return rocrand_device::detail::normal_distribution2(state1, state2);
1147 __forceinline__ __device__ __host__
1150 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1167 __forceinline__ __device__ __host__
1175 return rocrand_device::detail::normal_distribution_double2(
1176 uint4{state1, state2, state3, state4});
1191 __forceinline__ __device__ __host__
1194 return rocrand_device::detail::normal_distribution(
rocrand(state));
1211 __forceinline__ __device__ __host__
1214 return rocrand_device::detail::normal_distribution2(rocrand2(state));
1229 __forceinline__ __device__ __host__
1232 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1249 __forceinline__ __device__ __host__
1252 auto state1 = rocrand2(state);
1253 auto state2 = rocrand2(state);
1255 return rocrand_device::detail::normal_distribution_double2(
1256 uint4{state1.x, state1.y, state2.x, state2.y});
1271 __forceinline__ __device__ __host__
1274 return rocrand_device::detail::normal_distribution(
rocrand(state));
1291 __forceinline__ __device__ __host__
1294 return rocrand_device::detail::normal_distribution2(
rocrand(state));
1309 __forceinline__ __device__ __host__
1312 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1329 __forceinline__ __device__ __host__
1332 return rocrand_device::detail::normal_distribution_double2(rocrand2(state));
1347 __forceinline__ __device__ __host__
1350 return rocrand_device::detail::normal_distribution(
rocrand(state));
1367 __forceinline__ __device__ __host__
1373 return rocrand_device::detail::normal_distribution2(state1, state2);
1388 __forceinline__ __device__ __host__
1391 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1408 __forceinline__ __device__ __host__
1411 return rocrand_device::detail::normal_distribution_double2(
rocrand4(state));
1426 __forceinline__ __device__ __host__
1429 return rocrand_device::detail::normal_distribution(
rocrand(state));
1446 __forceinline__ __device__ __host__
1452 return rocrand_device::detail::normal_distribution2(state1, state2);
1467 __forceinline__ __device__ __host__
1470 return rocrand_device::detail::normal_distribution_double(
rocrand(state));
1487 __forceinline__ __device__ __host__
1493 return rocrand_device::detail::normal_distribution_double2(ulonglong2{state1, state2});
__forceinline__ __device__ __host__ double4 rocrand_normal_double4(rocrand_state_philox4x32_10 *state)
Returns four normally distributed double values.
Definition: rocrand_normal.h:505
__forceinline__ __device__ __host__ double2 rocrand_normal_double2(rocrand_state_philox4x32_10 *state)
Returns two normally distributed double values.
Definition: rocrand_normal.h:485
__forceinline__ __device__ __host__ float rocrand_normal(rocrand_state_philox4x32_10 *state)
Returns a normally distributed float value.
Definition: rocrand_normal.h:379
__forceinline__ __device__ __host__ uint4 rocrand4(rocrand_state_philox4x32_10 *state)
Returns four uniformly distributed random unsigned int values from [0; 2^32 - 1] range.
Definition: rocrand_philox4x32_10.h:377
__forceinline__ __device__ __host__ double rocrand_normal_double(rocrand_state_philox4x32_10 *state)
Returns a normally distributed double value.
Definition: rocrand_normal.h:456
__forceinline__ __device__ __host__ float4 rocrand_normal4(rocrand_state_philox4x32_10 *state)
Returns four normally distributed float values.
Definition: rocrand_normal.h:435
__forceinline__ __device__ __host__ unsigned int rocrand(rocrand_state_lfsr113 *state)
Returns uniformly distributed random unsigned int value from [0; 2^32 - 1] range.
Definition: rocrand_lfsr113.h:277
__forceinline__ __device__ __host__ float2 rocrand_normal2(rocrand_state_philox4x32_10 *state)
Returns two normally distributed float values.
Definition: rocrand_normal.h:412