1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
//! Private implementation details of public gather/scatter APIs.
#[cfg(not(bootstrap))]
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SupportedLaneCount};
#[cfg(bootstrap)]
use core::mem;

/// A vector of *const T.
#[derive(Debug, Copy, Clone)]
#[repr(simd)]
pub(crate) struct SimdConstPtr<T, const LANES: usize>([*const T; LANES]);

impl<T, const LANES: usize> SimdConstPtr<T, LANES>
where
    LaneCount<LANES>: SupportedLaneCount,
    T: Sized,
{
    #[inline]
    #[must_use]
    pub fn splat(ptr: *const T) -> Self {
        Self([ptr; LANES])
    }

    #[inline]
    #[must_use]
    pub fn wrapping_add(self, addend: Simd<usize, LANES>) -> Self {
        #[cfg(bootstrap)]
        // Safety: converting pointers to usize and vice-versa is safe
        // (even if using that pointer is not)
        unsafe {
            let x: Simd<usize, LANES> = mem::transmute_copy(&self);
            mem::transmute_copy(&{ x + (addend * Simd::splat(mem::size_of::<T>())) })
        }
        #[cfg(not(bootstrap))]
        // Safety: this intrinsic doesn't have a precondition
        unsafe { intrinsics::simd_arith_offset(self, addend) }
    }
}

/// A vector of *mut T. Be very careful around potential aliasing.
#[derive(Debug, Copy, Clone)]
#[repr(simd)]
pub(crate) struct SimdMutPtr<T, const LANES: usize>([*mut T; LANES]);

impl<T, const LANES: usize> SimdMutPtr<T, LANES>
where
    LaneCount<LANES>: SupportedLaneCount,
    T: Sized,
{
    #[inline]
    #[must_use]
    pub fn splat(ptr: *mut T) -> Self {
        Self([ptr; LANES])
    }

    #[inline]
    #[must_use]
    pub fn wrapping_add(self, addend: Simd<usize, LANES>) -> Self {
        #[cfg(bootstrap)]
        // Safety: converting pointers to usize and vice-versa is safe
        // (even if using that pointer is not)
        unsafe {
            let x: Simd<usize, LANES> = mem::transmute_copy(&self);
            mem::transmute_copy(&{ x + (addend * Simd::splat(mem::size_of::<T>())) })
        }
        #[cfg(not(bootstrap))]
        // Safety: this intrinsic doesn't have a precondition
        unsafe { intrinsics::simd_arith_offset(self, addend) }
    }
}