axhal/
percpu.rs

1//! CPU-local data structures.
2
3#[percpu::def_percpu]
4static CPU_ID: usize = 0;
5
6#[percpu::def_percpu]
7static IS_BSP: bool = false;
8
9#[percpu::def_percpu]
10static CURRENT_TASK_PTR: usize = 0;
11
12/// Returns the ID of the current CPU.
13#[inline]
14pub fn this_cpu_id() -> usize {
15    CPU_ID.read_current()
16}
17
18/// Returns whether the current CPU is the primary CPU (aka the bootstrap
19/// processor or BSP)
20#[inline]
21pub fn this_cpu_is_bsp() -> bool {
22    IS_BSP.read_current()
23}
24
25/// Caches the pointer to the current task in the `SP_EL0` register.
26///
27/// In aarch64 architecture, we use `SP_EL0` as the read cache for
28/// the current task pointer. And this function will update this cache.
29#[cfg(target_arch = "aarch64")]
30unsafe fn cache_current_task_ptr<T>(ptr: *const T) {
31    use aarch64_cpu::registers::{SP_EL0, Writeable};
32    SP_EL0.set(ptr as u64);
33}
34
35/// Gets the pointer to the current task with preemption-safety.
36///
37/// Preemption may be enabled when calling this function. This function will
38/// guarantee the correctness even the current task is preempted.
39#[inline]
40pub fn current_task_ptr<T>() -> *const T {
41    #[cfg(target_arch = "x86_64")]
42    unsafe {
43        // on x86, only one instruction is needed to read the per-CPU task pointer from `gs:[off]`.
44        CURRENT_TASK_PTR.read_current_raw() as _
45    }
46    #[cfg(any(
47        target_arch = "riscv32",
48        target_arch = "riscv64",
49        target_arch = "loongarch64"
50    ))]
51    unsafe {
52        // on RISC-V and LA64, reading `CURRENT_TASK_PTR` requires multiple instruction, so we disable local IRQs.
53        let _guard = kernel_guard::IrqSave::new();
54        CURRENT_TASK_PTR.read_current_raw() as _
55    }
56    #[cfg(target_arch = "aarch64")]
57    {
58        // on ARM64, we use `SP_EL0` to cache the current task pointer.
59        use aarch64_cpu::registers::{Readable, SP_EL0};
60        let sp_el0 = SP_EL0.get();
61        if sp_el0 != 0 {
62            // use the cached value
63            sp_el0 as _
64        } else {
65            // `SP_EL0` will be reset to 0 on each interrupt or exception,
66            // call the slow path and cache the value.
67            unsafe {
68                let _guard = kernel_guard::IrqSave::new();
69                let ptr = CURRENT_TASK_PTR.read_current_raw() as _;
70                cache_current_task_ptr(ptr);
71                ptr
72            }
73        }
74    }
75}
76
77/// Sets the pointer to the current task with preemption-safety.
78///
79/// Preemption may be enabled when calling this function. This function will
80/// guarantee the correctness even the current task is preempted.
81///
82/// # Safety
83///
84/// The given `ptr` must be pointed to a valid task structure.
85#[inline]
86pub unsafe fn set_current_task_ptr<T>(ptr: *const T) {
87    #[cfg(target_arch = "x86_64")]
88    {
89        unsafe { CURRENT_TASK_PTR.write_current_raw(ptr as usize) }
90    }
91    #[cfg(any(
92        target_arch = "riscv32",
93        target_arch = "riscv64",
94        target_arch = "loongarch64"
95    ))]
96    {
97        let _guard = kernel_guard::IrqSave::new();
98        unsafe { CURRENT_TASK_PTR.write_current_raw(ptr as usize) }
99    }
100    #[cfg(target_arch = "aarch64")]
101    {
102        let _guard = kernel_guard::IrqSave::new();
103        unsafe {
104            CURRENT_TASK_PTR.write_current_raw(ptr as usize);
105            cache_current_task_ptr(ptr);
106        }
107    }
108}
109
110#[allow(dead_code)]
111pub(crate) fn init_primary(cpu_id: usize) {
112    percpu::init();
113    percpu::init_percpu_reg(cpu_id);
114    unsafe {
115        CPU_ID.write_current_raw(cpu_id);
116        IS_BSP.write_current_raw(true);
117    }
118}
119
120#[allow(dead_code)]
121pub(crate) fn init_secondary(cpu_id: usize) {
122    percpu::init_percpu_reg(cpu_id);
123    unsafe {
124        CPU_ID.write_current_raw(cpu_id);
125        IS_BSP.write_current_raw(false);
126    }
127}
axhal/percpu.rs

axhal/
percpu.rs
