axhal/
percpu.rs

1//! CPU-local data structures.
2
3#[percpu::def_percpu]
4static CPU_ID: usize = 0;
5
6#[percpu::def_percpu]
7static IS_BSP: bool = false;
8
9#[percpu::def_percpu]
10static CURRENT_TASK_PTR: usize = 0;
11
12/// Returns the ID of the current CPU.
13#[inline]
14pub fn this_cpu_id() -> usize {
15    CPU_ID.read_current()
16}
17
18/// Returns whether the current CPU is the primary CPU (aka the bootstrap
19/// processor or BSP)
20#[inline]
21pub fn this_cpu_is_bsp() -> bool {
22    IS_BSP.read_current()
23}
24
25/// Caches the pointer to the current task in the `SP_EL0` register.
26///
27/// In aarch64 architecture, we use `SP_EL0` as the read cache for
28/// the current task pointer. And this function will update this cache.
29#[cfg(target_arch = "aarch64")]
30unsafe fn cache_current_task_ptr<T>(ptr: *const T) {
31    use aarch64_cpu::registers::{SP_EL0, Writeable};
32    SP_EL0.set(ptr as u64);
33}
34
35/// Gets the pointer to the current task with preemption-safety.
36///
37/// Preemption may be enabled when calling this function. This function will
38/// guarantee the correctness even the current task is preempted.
39#[inline]
40pub fn current_task_ptr<T>() -> *const T {
41    #[cfg(target_arch = "x86_64")]
42    unsafe {
43        // on x86, only one instruction is needed to read the per-CPU task pointer from `gs:[off]`.
44        CURRENT_TASK_PTR.read_current_raw() as _
45    }
46    #[cfg(any(
47        target_arch = "riscv32",
48        target_arch = "riscv64",
49        target_arch = "loongarch64"
50    ))]
51    unsafe {
52        // on RISC-V and LA64, reading `CURRENT_TASK_PTR` requires multiple instruction, so we disable local IRQs.
53        let _guard = kernel_guard::IrqSave::new();
54        CURRENT_TASK_PTR.read_current_raw() as _
55    }
56    #[cfg(target_arch = "aarch64")]
57    {
58        // on ARM64, we use `SP_EL0` to cache the current task pointer.
59        use aarch64_cpu::registers::{Readable, SP_EL0};
60        let sp_el0 = SP_EL0.get();
61        if sp_el0 != 0 {
62            // use the cached value
63            sp_el0 as _
64        } else {
65            // `SP_EL0` will be reset to 0 on each interrupt or exception,
66            // call the slow path and cache the value.
67            unsafe {
68                let _guard = kernel_guard::IrqSave::new();
69                let ptr = CURRENT_TASK_PTR.read_current_raw() as _;
70                cache_current_task_ptr(ptr);
71                ptr
72            }
73        }
74    }
75    #[cfg(target_arch = "arm")]
76    unsafe {
77        let _guard = kernel_guard::IrqSave::new();
78        CURRENT_TASK_PTR.read_current_raw() as _
79    }
80}
81
82/// Sets the pointer to the current task with preemption-safety.
83///
84/// Preemption may be enabled when calling this function. This function will
85/// guarantee the correctness even the current task is preempted.
86///
87/// # Safety
88///
89/// The given `ptr` must be pointed to a valid task structure.
90#[inline]
91pub unsafe fn set_current_task_ptr<T>(ptr: *const T) {
92    #[cfg(target_arch = "x86_64")]
93    {
94        unsafe { CURRENT_TASK_PTR.write_current_raw(ptr as usize) }
95    }
96    #[cfg(any(
97        target_arch = "riscv32",
98        target_arch = "riscv64",
99        target_arch = "loongarch64"
100    ))]
101    {
102        let _guard = kernel_guard::IrqSave::new();
103        unsafe { CURRENT_TASK_PTR.write_current_raw(ptr as usize) }
104    }
105    #[cfg(target_arch = "aarch64")]
106    {
107        let _guard = kernel_guard::IrqSave::new();
108        unsafe {
109            CURRENT_TASK_PTR.write_current_raw(ptr as usize);
110            cache_current_task_ptr(ptr);
111        }
112    }
113    #[cfg(target_arch = "arm")]
114    {
115        let _guard = kernel_guard::IrqSave::new();
116        unsafe { CURRENT_TASK_PTR.write_current_raw(ptr as usize) }
117    }
118}
119
120#[allow(dead_code)]
121pub(crate) fn init_primary(cpu_id: usize) {
122    percpu::init_in_place().expect("failed to initialize percpu area");
123    percpu::init_percpu_reg(cpu_id);
124    unsafe {
125        CPU_ID.write_current_raw(cpu_id);
126        IS_BSP.write_current_raw(true);
127    }
128}
129
130#[allow(dead_code)]
131pub(crate) fn init_secondary(cpu_id: usize) {
132    percpu::init_percpu_reg(cpu_id);
133    unsafe {
134        CPU_ID.write_current_raw(cpu_id);
135        IS_BSP.write_current_raw(false);
136    }
137}
axhal/percpu.rs

axhal/
percpu.rs
