kernel/alloc/
kvec.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! Implementation of [`Vec`].
4
5// May not be needed in Rust 1.87.0 (pending beta backport).
6#![allow(clippy::ptr_eq)]
7
8use super::{
9    allocator::{KVmalloc, Kmalloc, Vmalloc},
10    layout::ArrayLayout,
11    AllocError, Allocator, Box, Flags,
12};
13use core::{
14    fmt,
15    marker::PhantomData,
16    mem::{ManuallyDrop, MaybeUninit},
17    ops::Deref,
18    ops::DerefMut,
19    ops::Index,
20    ops::IndexMut,
21    ptr,
22    ptr::NonNull,
23    slice,
24    slice::SliceIndex,
25};
26
27/// Create a [`KVec`] containing the arguments.
28///
29/// New memory is allocated with `GFP_KERNEL`.
30///
31/// # Examples
32///
33/// ```
34/// let mut v = kernel::kvec![];
35/// v.push(1, GFP_KERNEL)?;
36/// assert_eq!(v, [1]);
37///
38/// let mut v = kernel::kvec![1; 3]?;
39/// v.push(4, GFP_KERNEL)?;
40/// assert_eq!(v, [1, 1, 1, 4]);
41///
42/// let mut v = kernel::kvec![1, 2, 3]?;
43/// v.push(4, GFP_KERNEL)?;
44/// assert_eq!(v, [1, 2, 3, 4]);
45///
46/// # Ok::<(), Error>(())
47/// ```
48#[macro_export]
49macro_rules! kvec {
50    () => (
51        $crate::alloc::KVec::new()
52    );
53    ($elem:expr; $n:expr) => (
54        $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL)
55    );
56    ($($x:expr),+ $(,)?) => (
57        match $crate::alloc::KBox::new_uninit(GFP_KERNEL) {
58            Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))),
59            Err(e) => Err(e),
60        }
61    );
62}
63
64/// The kernel's [`Vec`] type.
65///
66/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g.
67/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec<T, A>`.
68///
69/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For
70/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist.
71///
72/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::<T>`; no memory is allocated.
73///
74/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the
75/// capacity of the vector (the number of elements that currently fit into the vector), its length
76/// (the number of elements that are currently stored in the vector) and the `Allocator` type used
77/// to allocate (and free) the backing buffer.
78///
79/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts
80/// and manually modified.
81///
82/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements
83/// are added to the vector.
84///
85/// # Invariants
86///
87/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for
88///   zero-sized types, is a dangling, well aligned pointer.
89///
90/// - `self.len` always represents the exact number of elements stored in the vector.
91///
92/// - `self.layout` represents the absolute number of elements that can be stored within the vector
93///   without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the
94///   backing buffer to be larger than `layout`.
95///
96/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer
97///   was allocated with (and must be freed with).
98pub struct Vec<T, A: Allocator> {
99    ptr: NonNull<T>,
100    /// Represents the actual buffer size as `cap` times `size_of::<T>` bytes.
101    ///
102    /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of
103    /// elements we can still store without reallocating.
104    layout: ArrayLayout<T>,
105    len: usize,
106    _p: PhantomData<A>,
107}
108
109/// Type alias for [`Vec`] with a [`Kmalloc`] allocator.
110///
111/// # Examples
112///
113/// ```
114/// let mut v = KVec::new();
115/// v.push(1, GFP_KERNEL)?;
116/// assert_eq!(&v, &[1]);
117///
118/// # Ok::<(), Error>(())
119/// ```
120pub type KVec<T> = Vec<T, Kmalloc>;
121
122/// Type alias for [`Vec`] with a [`Vmalloc`] allocator.
123///
124/// # Examples
125///
126/// ```
127/// let mut v = VVec::new();
128/// v.push(1, GFP_KERNEL)?;
129/// assert_eq!(&v, &[1]);
130///
131/// # Ok::<(), Error>(())
132/// ```
133pub type VVec<T> = Vec<T, Vmalloc>;
134
135/// Type alias for [`Vec`] with a [`KVmalloc`] allocator.
136///
137/// # Examples
138///
139/// ```
140/// let mut v = KVVec::new();
141/// v.push(1, GFP_KERNEL)?;
142/// assert_eq!(&v, &[1]);
143///
144/// # Ok::<(), Error>(())
145/// ```
146pub type KVVec<T> = Vec<T, KVmalloc>;
147
148// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements.
149unsafe impl<T, A> Send for Vec<T, A>
150where
151    T: Send,
152    A: Allocator,
153{
154}
155
156// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements.
157unsafe impl<T, A> Sync for Vec<T, A>
158where
159    T: Sync,
160    A: Allocator,
161{
162}
163
164impl<T, A> Vec<T, A>
165where
166    A: Allocator,
167{
168    #[inline]
169    const fn is_zst() -> bool {
170        core::mem::size_of::<T>() == 0
171    }
172
173    /// Returns the number of elements that can be stored within the vector without allocating
174    /// additional memory.
175    pub fn capacity(&self) -> usize {
176        if const { Self::is_zst() } {
177            usize::MAX
178        } else {
179            self.layout.len()
180        }
181    }
182
183    /// Returns the number of elements stored within the vector.
184    #[inline]
185    pub fn len(&self) -> usize {
186        self.len
187    }
188
189    /// Forcefully sets `self.len` to `new_len`.
190    ///
191    /// # Safety
192    ///
193    /// - `new_len` must be less than or equal to [`Self::capacity`].
194    /// - If `new_len` is greater than `self.len`, all elements within the interval
195    ///   [`self.len`,`new_len`) must be initialized.
196    #[inline]
197    pub unsafe fn set_len(&mut self, new_len: usize) {
198        debug_assert!(new_len <= self.capacity());
199        self.len = new_len;
200    }
201
202    /// Returns a slice of the entire vector.
203    #[inline]
204    pub fn as_slice(&self) -> &[T] {
205        self
206    }
207
208    /// Returns a mutable slice of the entire vector.
209    #[inline]
210    pub fn as_mut_slice(&mut self) -> &mut [T] {
211        self
212    }
213
214    /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a
215    /// dangling raw pointer.
216    #[inline]
217    pub fn as_mut_ptr(&mut self) -> *mut T {
218        self.ptr.as_ptr()
219    }
220
221    /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw
222    /// pointer.
223    #[inline]
224    pub fn as_ptr(&self) -> *const T {
225        self.ptr.as_ptr()
226    }
227
228    /// Returns `true` if the vector contains no elements, `false` otherwise.
229    ///
230    /// # Examples
231    ///
232    /// ```
233    /// let mut v = KVec::new();
234    /// assert!(v.is_empty());
235    ///
236    /// v.push(1, GFP_KERNEL);
237    /// assert!(!v.is_empty());
238    /// ```
239    #[inline]
240    pub fn is_empty(&self) -> bool {
241        self.len() == 0
242    }
243
244    /// Creates a new, empty `Vec<T, A>`.
245    ///
246    /// This method does not allocate by itself.
247    #[inline]
248    pub const fn new() -> Self {
249        // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet,
250        // - `ptr` is a properly aligned dangling pointer for type `T`,
251        // - `layout` is an empty `ArrayLayout` (zero capacity)
252        // - `len` is zero, since no elements can be or have been stored,
253        // - `A` is always valid.
254        Self {
255            ptr: NonNull::dangling(),
256            layout: ArrayLayout::empty(),
257            len: 0,
258            _p: PhantomData::<A>,
259        }
260    }
261
262    /// Returns a slice of `MaybeUninit<T>` for the remaining spare capacity of the vector.
263    pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<T>] {
264        // SAFETY:
265        // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is
266        //   guaranteed to be part of the same allocated object.
267        // - `self.len` can not overflow `isize`.
268        let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit<T>;
269
270        // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated
271        // and valid, but uninitialized.
272        unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) }
273    }
274
275    /// Appends an element to the back of the [`Vec`] instance.
276    ///
277    /// # Examples
278    ///
279    /// ```
280    /// let mut v = KVec::new();
281    /// v.push(1, GFP_KERNEL)?;
282    /// assert_eq!(&v, &[1]);
283    ///
284    /// v.push(2, GFP_KERNEL)?;
285    /// assert_eq!(&v, &[1, 2]);
286    /// # Ok::<(), Error>(())
287    /// ```
288    pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> {
289        self.reserve(1, flags)?;
290
291        // SAFETY:
292        // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is
293        //   guaranteed to be part of the same allocated object.
294        // - `self.len` can not overflow `isize`.
295        let ptr = unsafe { self.as_mut_ptr().add(self.len) };
296
297        // SAFETY:
298        // - `ptr` is properly aligned and valid for writes.
299        unsafe { core::ptr::write(ptr, v) };
300
301        // SAFETY: We just initialised the first spare entry, so it is safe to increase the length
302        // by 1. We also know that the new length is <= capacity because of the previous call to
303        // `reserve` above.
304        unsafe { self.set_len(self.len() + 1) };
305        Ok(())
306    }
307
308    /// Creates a new [`Vec`] instance with at least the given capacity.
309    ///
310    /// # Examples
311    ///
312    /// ```
313    /// let v = KVec::<u32>::with_capacity(20, GFP_KERNEL)?;
314    ///
315    /// assert!(v.capacity() >= 20);
316    /// # Ok::<(), Error>(())
317    /// ```
318    pub fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> {
319        let mut v = Vec::new();
320
321        v.reserve(capacity, flags)?;
322
323        Ok(v)
324    }
325
326    /// Creates a `Vec<T, A>` from a pointer, a length and a capacity using the allocator `A`.
327    ///
328    /// # Examples
329    ///
330    /// ```
331    /// let mut v = kernel::kvec![1, 2, 3]?;
332    /// v.reserve(1, GFP_KERNEL)?;
333    ///
334    /// let (mut ptr, mut len, cap) = v.into_raw_parts();
335    ///
336    /// // SAFETY: We've just reserved memory for another element.
337    /// unsafe { ptr.add(len).write(4) };
338    /// len += 1;
339    ///
340    /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and
341    /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it
342    /// // from the exact same raw parts.
343    /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) };
344    ///
345    /// assert_eq!(v, [1, 2, 3, 4]);
346    ///
347    /// # Ok::<(), Error>(())
348    /// ```
349    ///
350    /// # Safety
351    ///
352    /// If `T` is a ZST:
353    ///
354    /// - `ptr` must be a dangling, well aligned pointer.
355    ///
356    /// Otherwise:
357    ///
358    /// - `ptr` must have been allocated with the allocator `A`.
359    /// - `ptr` must satisfy or exceed the alignment requirements of `T`.
360    /// - `ptr` must point to memory with a size of at least `size_of::<T>() * capacity` bytes.
361    /// - The allocated size in bytes must not be larger than `isize::MAX`.
362    /// - `length` must be less than or equal to `capacity`.
363    /// - The first `length` elements must be initialized values of type `T`.
364    ///
365    /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for
366    /// `cap` and `len`.
367    pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self {
368        let layout = if Self::is_zst() {
369            ArrayLayout::empty()
370        } else {
371            // SAFETY: By the safety requirements of this function, `capacity * size_of::<T>()` is
372            // smaller than `isize::MAX`.
373            unsafe { ArrayLayout::new_unchecked(capacity) }
374        };
375
376        // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are
377        // covered by the safety requirements of this function.
378        Self {
379            // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid
380            // memory allocation, allocated with `A`.
381            ptr: unsafe { NonNull::new_unchecked(ptr) },
382            layout,
383            len: length,
384            _p: PhantomData::<A>,
385        }
386    }
387
388    /// Consumes the `Vec<T, A>` and returns its raw components `pointer`, `length` and `capacity`.
389    ///
390    /// This will not run the destructor of the contained elements and for non-ZSTs the allocation
391    /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the
392    /// elements and free the allocation, if any.
393    pub fn into_raw_parts(self) -> (*mut T, usize, usize) {
394        let mut me = ManuallyDrop::new(self);
395        let len = me.len();
396        let capacity = me.capacity();
397        let ptr = me.as_mut_ptr();
398        (ptr, len, capacity)
399    }
400
401    /// Ensures that the capacity exceeds the length by at least `additional` elements.
402    ///
403    /// # Examples
404    ///
405    /// ```
406    /// let mut v = KVec::new();
407    /// v.push(1, GFP_KERNEL)?;
408    ///
409    /// v.reserve(10, GFP_KERNEL)?;
410    /// let cap = v.capacity();
411    /// assert!(cap >= 10);
412    ///
413    /// v.reserve(10, GFP_KERNEL)?;
414    /// let new_cap = v.capacity();
415    /// assert_eq!(new_cap, cap);
416    ///
417    /// # Ok::<(), Error>(())
418    /// ```
419    pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> {
420        let len = self.len();
421        let cap = self.capacity();
422
423        if cap - len >= additional {
424            return Ok(());
425        }
426
427        if Self::is_zst() {
428            // The capacity is already `usize::MAX` for ZSTs, we can't go higher.
429            return Err(AllocError);
430        }
431
432        // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the
433        // multiplication by two won't overflow.
434        let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?);
435        let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?;
436
437        // SAFETY:
438        // - `ptr` is valid because it's either `None` or comes from a previous call to
439        //   `A::realloc`.
440        // - `self.layout` matches the `ArrayLayout` of the preceding allocation.
441        let ptr = unsafe {
442            A::realloc(
443                Some(self.ptr.cast()),
444                layout.into(),
445                self.layout.into(),
446                flags,
447            )?
448        };
449
450        // INVARIANT:
451        // - `layout` is some `ArrayLayout::<T>`,
452        // - `ptr` has been created by `A::realloc` from `layout`.
453        self.ptr = ptr.cast();
454        self.layout = layout;
455
456        Ok(())
457    }
458}
459
460impl<T: Clone, A: Allocator> Vec<T, A> {
461    /// Extend the vector by `n` clones of `value`.
462    pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> {
463        if n == 0 {
464            return Ok(());
465        }
466
467        self.reserve(n, flags)?;
468
469        let spare = self.spare_capacity_mut();
470
471        for item in spare.iter_mut().take(n - 1) {
472            item.write(value.clone());
473        }
474
475        // We can write the last element directly without cloning needlessly.
476        spare[n - 1].write(value);
477
478        // SAFETY:
479        // - `self.len() + n < self.capacity()` due to the call to reserve above,
480        // - the loop and the line above initialized the next `n` elements.
481        unsafe { self.set_len(self.len() + n) };
482
483        Ok(())
484    }
485
486    /// Pushes clones of the elements of slice into the [`Vec`] instance.
487    ///
488    /// # Examples
489    ///
490    /// ```
491    /// let mut v = KVec::new();
492    /// v.push(1, GFP_KERNEL)?;
493    ///
494    /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?;
495    /// assert_eq!(&v, &[1, 20, 30, 40]);
496    ///
497    /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?;
498    /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]);
499    /// # Ok::<(), Error>(())
500    /// ```
501    pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> {
502        self.reserve(other.len(), flags)?;
503        for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) {
504            slot.write(item.clone());
505        }
506
507        // SAFETY:
508        // - `other.len()` spare entries have just been initialized, so it is safe to increase
509        //   the length by the same number.
510        // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve`
511        //   call.
512        unsafe { self.set_len(self.len() + other.len()) };
513        Ok(())
514    }
515
516    /// Create a new `Vec<T, A>` and extend it by `n` clones of `value`.
517    pub fn from_elem(value: T, n: usize, flags: Flags) -> Result<Self, AllocError> {
518        let mut v = Self::with_capacity(n, flags)?;
519
520        v.extend_with(n, value, flags)?;
521
522        Ok(v)
523    }
524}
525
526impl<T, A> Drop for Vec<T, A>
527where
528    A: Allocator,
529{
530    fn drop(&mut self) {
531        // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant.
532        unsafe {
533            ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
534                self.as_mut_ptr(),
535                self.len,
536            ))
537        };
538
539        // SAFETY:
540        // - `self.ptr` was previously allocated with `A`.
541        // - `self.layout` matches the `ArrayLayout` of the preceding allocation.
542        unsafe { A::free(self.ptr.cast(), self.layout.into()) };
543    }
544}
545
546impl<T, A, const N: usize> From<Box<[T; N], A>> for Vec<T, A>
547where
548    A: Allocator,
549{
550    fn from(b: Box<[T; N], A>) -> Vec<T, A> {
551        let len = b.len();
552        let ptr = Box::into_raw(b);
553
554        // SAFETY:
555        // - `b` has been allocated with `A`,
556        // - `ptr` fulfills the alignment requirements for `T`,
557        // - `ptr` points to memory with at least a size of `size_of::<T>() * len`,
558        // - all elements within `b` are initialized values of `T`,
559        // - `len` does not exceed `isize::MAX`.
560        unsafe { Vec::from_raw_parts(ptr as _, len, len) }
561    }
562}
563
564impl<T> Default for KVec<T> {
565    #[inline]
566    fn default() -> Self {
567        Self::new()
568    }
569}
570
571impl<T: fmt::Debug, A: Allocator> fmt::Debug for Vec<T, A> {
572    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
573        fmt::Debug::fmt(&**self, f)
574    }
575}
576
577impl<T, A> Deref for Vec<T, A>
578where
579    A: Allocator,
580{
581    type Target = [T];
582
583    #[inline]
584    fn deref(&self) -> &[T] {
585        // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len`
586        // initialized elements of type `T`.
587        unsafe { slice::from_raw_parts(self.as_ptr(), self.len) }
588    }
589}
590
591impl<T, A> DerefMut for Vec<T, A>
592where
593    A: Allocator,
594{
595    #[inline]
596    fn deref_mut(&mut self) -> &mut [T] {
597        // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len`
598        // initialized elements of type `T`.
599        unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) }
600    }
601}
602
603impl<T: Eq, A> Eq for Vec<T, A> where A: Allocator {}
604
605impl<T, I: SliceIndex<[T]>, A> Index<I> for Vec<T, A>
606where
607    A: Allocator,
608{
609    type Output = I::Output;
610
611    #[inline]
612    fn index(&self, index: I) -> &Self::Output {
613        Index::index(&**self, index)
614    }
615}
616
617impl<T, I: SliceIndex<[T]>, A> IndexMut<I> for Vec<T, A>
618where
619    A: Allocator,
620{
621    #[inline]
622    fn index_mut(&mut self, index: I) -> &mut Self::Output {
623        IndexMut::index_mut(&mut **self, index)
624    }
625}
626
627macro_rules! impl_slice_eq {
628    ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => {
629        $(
630            impl<T, U, $($vars)*> PartialEq<$rhs> for $lhs
631            where
632                T: PartialEq<U>,
633            {
634                #[inline]
635                fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] }
636            }
637        )*
638    }
639}
640
641impl_slice_eq! {
642    [A1: Allocator, A2: Allocator] Vec<T, A1>, Vec<U, A2>,
643    [A: Allocator] Vec<T, A>, &[U],
644    [A: Allocator] Vec<T, A>, &mut [U],
645    [A: Allocator] &[T], Vec<U, A>,
646    [A: Allocator] &mut [T], Vec<U, A>,
647    [A: Allocator] Vec<T, A>, [U],
648    [A: Allocator] [T], Vec<U, A>,
649    [A: Allocator, const N: usize] Vec<T, A>, [U; N],
650    [A: Allocator, const N: usize] Vec<T, A>, &[U; N],
651}
652
653impl<'a, T, A> IntoIterator for &'a Vec<T, A>
654where
655    A: Allocator,
656{
657    type Item = &'a T;
658    type IntoIter = slice::Iter<'a, T>;
659
660    fn into_iter(self) -> Self::IntoIter {
661        self.iter()
662    }
663}
664
665impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec<T, A>
666where
667    A: Allocator,
668{
669    type Item = &'a mut T;
670    type IntoIter = slice::IterMut<'a, T>;
671
672    fn into_iter(self) -> Self::IntoIter {
673        self.iter_mut()
674    }
675}
676
677/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector.
678///
679/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the
680/// [`IntoIterator`] trait).
681///
682/// # Examples
683///
684/// ```
685/// let v = kernel::kvec![0, 1, 2]?;
686/// let iter = v.into_iter();
687///
688/// # Ok::<(), Error>(())
689/// ```
690pub struct IntoIter<T, A: Allocator> {
691    ptr: *mut T,
692    buf: NonNull<T>,
693    len: usize,
694    layout: ArrayLayout<T>,
695    _p: PhantomData<A>,
696}
697
698impl<T, A> IntoIter<T, A>
699where
700    A: Allocator,
701{
702    fn into_raw_parts(self) -> (*mut T, NonNull<T>, usize, usize) {
703        let me = ManuallyDrop::new(self);
704        let ptr = me.ptr;
705        let buf = me.buf;
706        let len = me.len;
707        let cap = me.layout.len();
708        (ptr, buf, len, cap)
709    }
710
711    /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`.
712    ///
713    /// # Examples
714    ///
715    /// ```
716    /// let v = kernel::kvec![1, 2, 3]?;
717    /// let mut it = v.into_iter();
718    ///
719    /// assert_eq!(it.next(), Some(1));
720    ///
721    /// let v = it.collect(GFP_KERNEL);
722    /// assert_eq!(v, [2, 3]);
723    ///
724    /// # Ok::<(), Error>(())
725    /// ```
726    ///
727    /// # Implementation details
728    ///
729    /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait
730    /// in the kernel, namely:
731    ///
732    /// - Rust's specialization feature is unstable. This prevents us to optimize for the special
733    ///   case where `I::IntoIter` equals `Vec`'s `IntoIter` type.
734    /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator`
735    ///   doesn't require this type to be `'static`.
736    /// - `FromIterator::from_iter` does return `Self` instead of `Result<Self, AllocError>`, hence
737    ///   we can't properly handle allocation failures.
738    /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation
739    ///   flags.
740    ///
741    /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a
742    /// `Vec` again.
743    ///
744    /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing
745    /// buffer. However, this backing buffer may be shrunk to the actual count of elements.
746    pub fn collect(self, flags: Flags) -> Vec<T, A> {
747        let old_layout = self.layout;
748        let (mut ptr, buf, len, mut cap) = self.into_raw_parts();
749        let has_advanced = ptr != buf.as_ptr();
750
751        if has_advanced {
752            // Copy the contents we have advanced to at the beginning of the buffer.
753            //
754            // SAFETY:
755            // - `ptr` is valid for reads of `len * size_of::<T>()` bytes,
756            // - `buf.as_ptr()` is valid for writes of `len * size_of::<T>()` bytes,
757            // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to
758            //   each other,
759            // - both `ptr` and `buf.ptr()` are properly aligned.
760            unsafe { ptr::copy(ptr, buf.as_ptr(), len) };
761            ptr = buf.as_ptr();
762
763            // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()`.
764            let layout = unsafe { ArrayLayout::<T>::new_unchecked(len) };
765
766            // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed to be
767            // smaller than `cap`. Depending on `alloc` this operation may shrink the buffer or leaves
768            // it as it is.
769            ptr = match unsafe {
770                A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags)
771            } {
772                // If we fail to shrink, which likely can't even happen, continue with the existing
773                // buffer.
774                Err(_) => ptr,
775                Ok(ptr) => {
776                    cap = len;
777                    ptr.as_ptr().cast()
778                }
779            };
780        }
781
782        // SAFETY: If the iterator has been advanced, the advanced elements have been copied to
783        // the beginning of the buffer and `len` has been adjusted accordingly.
784        //
785        // - `ptr` is guaranteed to point to the start of the backing buffer.
786        // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`.
787        // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original
788        //   `Vec`.
789        unsafe { Vec::from_raw_parts(ptr, len, cap) }
790    }
791}
792
793impl<T, A> Iterator for IntoIter<T, A>
794where
795    A: Allocator,
796{
797    type Item = T;
798
799    /// # Examples
800    ///
801    /// ```
802    /// let v = kernel::kvec![1, 2, 3]?;
803    /// let mut it = v.into_iter();
804    ///
805    /// assert_eq!(it.next(), Some(1));
806    /// assert_eq!(it.next(), Some(2));
807    /// assert_eq!(it.next(), Some(3));
808    /// assert_eq!(it.next(), None);
809    ///
810    /// # Ok::<(), Error>(())
811    /// ```
812    fn next(&mut self) -> Option<T> {
813        if self.len == 0 {
814            return None;
815        }
816
817        let current = self.ptr;
818
819        // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr`
820        // by one guarantees that.
821        unsafe { self.ptr = self.ptr.add(1) };
822
823        self.len -= 1;
824
825        // SAFETY: `current` is guaranteed to point at a valid element within the buffer.
826        Some(unsafe { current.read() })
827    }
828
829    /// # Examples
830    ///
831    /// ```
832    /// let v: KVec<u32> = kernel::kvec![1, 2, 3]?;
833    /// let mut iter = v.into_iter();
834    /// let size = iter.size_hint().0;
835    ///
836    /// iter.next();
837    /// assert_eq!(iter.size_hint().0, size - 1);
838    ///
839    /// iter.next();
840    /// assert_eq!(iter.size_hint().0, size - 2);
841    ///
842    /// iter.next();
843    /// assert_eq!(iter.size_hint().0, size - 3);
844    ///
845    /// # Ok::<(), Error>(())
846    /// ```
847    fn size_hint(&self) -> (usize, Option<usize>) {
848        (self.len, Some(self.len))
849    }
850}
851
852impl<T, A> Drop for IntoIter<T, A>
853where
854    A: Allocator,
855{
856    fn drop(&mut self) {
857        // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant.
858        unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) };
859
860        // SAFETY:
861        // - `self.buf` was previously allocated with `A`.
862        // - `self.layout` matches the `ArrayLayout` of the preceding allocation.
863        unsafe { A::free(self.buf.cast(), self.layout.into()) };
864    }
865}
866
867impl<T, A> IntoIterator for Vec<T, A>
868where
869    A: Allocator,
870{
871    type Item = T;
872    type IntoIter = IntoIter<T, A>;
873
874    /// Consumes the `Vec<T, A>` and creates an `Iterator`, which moves each value out of the
875    /// vector (from start to end).
876    ///
877    /// # Examples
878    ///
879    /// ```
880    /// let v = kernel::kvec![1, 2]?;
881    /// let mut v_iter = v.into_iter();
882    ///
883    /// let first_element: Option<u32> = v_iter.next();
884    ///
885    /// assert_eq!(first_element, Some(1));
886    /// assert_eq!(v_iter.next(), Some(2));
887    /// assert_eq!(v_iter.next(), None);
888    ///
889    /// # Ok::<(), Error>(())
890    /// ```
891    ///
892    /// ```
893    /// let v = kernel::kvec![];
894    /// let mut v_iter = v.into_iter();
895    ///
896    /// let first_element: Option<u32> = v_iter.next();
897    ///
898    /// assert_eq!(first_element, None);
899    ///
900    /// # Ok::<(), Error>(())
901    /// ```
902    #[inline]
903    fn into_iter(self) -> Self::IntoIter {
904        let buf = self.ptr;
905        let layout = self.layout;
906        let (ptr, len, _) = self.into_raw_parts();
907
908        IntoIter {
909            ptr,
910            buf,
911            len,
912            layout,
913            _p: PhantomData::<A>,
914        }
915    }
916}