std/sys/sync/mutex/
futex.rs

Help
1use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
2use crate::sys::futex::{self, futex_wait, futex_wake};
3
4type Futex = futex::SmallFutex;
5type State = futex::SmallPrimitive;
6
7pub struct Mutex {
8    futex: Futex,
9}
10
11const UNLOCKED: State = 0;
12const LOCKED: State = 1; // locked, no other threads waiting
13const CONTENDED: State = 2; // locked, and other threads waiting (contended)
14
15impl Mutex {
16    #[inline]
17    pub const fn new() -> Self {
18        Self { futex: Futex::new(UNLOCKED) }
19    }
20
21    #[inline]
22    // Make this a diagnostic item for Miri's concurrency model checker.
23    #[cfg_attr(not(test), rustc_diagnostic_item = "sys_mutex_try_lock")]
24    pub fn try_lock(&self) -> bool {
25        self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed).is_ok()
26    }
27
28    #[inline]
29    // Make this a diagnostic item for Miri's concurrency model checker.
30    #[cfg_attr(not(test), rustc_diagnostic_item = "sys_mutex_lock")]
31    pub fn lock(&self) {
32        if self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed).is_err() {
33            self.lock_contended();
34        }
35    }
36
37    #[cold]
38    fn lock_contended(&self) {
39        // Spin first to speed things up if the lock is released quickly.
40        let mut state = self.spin();
41
42        // If it's unlocked now, attempt to take the lock
43        // without marking it as contended.
44        if state == UNLOCKED {
45            match self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed) {
46                Ok(_) => return, // Locked!
47                Err(s) => state = s,
48            }
49        }
50
51        loop {
52            // Put the lock in contended state.
53            // We avoid an unnecessary write if it as already set to CONTENDED,
54            // to be friendlier for the caches.
55            if state != CONTENDED && self.futex.swap(CONTENDED, Acquire) == UNLOCKED {
56                // We changed it from UNLOCKED to CONTENDED, so we just successfully locked it.
57                return;
58            }
59
60            // Wait for the futex to change state, assuming it is still CONTENDED.
61            futex_wait(&self.futex, CONTENDED, None);
62
63            // Spin again after waking up.
64            state = self.spin();
65        }
66    }
67
68    fn spin(&self) -> State {
69        let mut spin = 100;
70        loop {
71            // We only use `load` (and not `swap` or `compare_exchange`)
72            // while spinning, to be easier on the caches.
73            let state = self.futex.load(Relaxed);
74
75            // We stop spinning when the mutex is UNLOCKED,
76            // but also when it's CONTENDED.
77            if state != LOCKED || spin == 0 {
78                return state;
79            }
80
81            crate::hint::spin_loop();
82            spin -= 1;
83        }
84    }
85
86    #[inline]
87    // Make this a diagnostic item for Miri's concurrency model checker.
88    #[cfg_attr(not(test), rustc_diagnostic_item = "sys_mutex_unlock")]
89    pub unsafe fn unlock(&self) {
90        if self.futex.swap(UNLOCKED, Release) == CONTENDED {
91            // We only wake up one thread. When that thread locks the mutex, it
92            // will mark the mutex as CONTENDED (see lock_contended above),
93            // which makes sure that any other waiting threads will also be
94            // woken up eventually.
95            self.wake();
96        }
97    }
98
99    #[cold]
100    fn wake(&self) {
101        futex_wake(&self.futex);
102    }
103}
std/sys/sync/mutex/futex.rs

std/sys/sync/mutex/
futex.rs
