1/* Low-level lock implementation. Generic futex-based version.
2 Copyright (C) 2005-2020 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library. If not, see
17 <https://www.gnu.org/licenses/>. */
18
19#ifndef _LOWLEVELLOCK_H
20#define _LOWLEVELLOCK_H 1
21
22#include <atomic.h>
23#include <lowlevellock-futex.h>
24#include <time.h>
25
26/* Low-level locks use a combination of atomic operations (to acquire and
27 release lock ownership) and futex operations (to block until the state
28 of a lock changes). A lock can be in one of three states:
29 0: not acquired,
30 1: acquired with no waiters; no other threads are blocked or about to block
31 for changes to the lock state,
32 >1: acquired, possibly with waiters; there may be other threads blocked or
33 about to block for changes to the lock state.
34
35 We expect that the common case is an uncontended lock, so we just need
36 to transition the lock between states 0 and 1; releasing the lock does
37 not need to wake any other blocked threads. If the lock is contended
38 and a thread decides to block using a futex operation, then this thread
39 needs to first change the state to >1; if this state is observed during
40 lock release, the releasing thread will wake one of the potentially
41 blocked threads.
42
43 Much of this code takes a 'private' parameter. This may be:
44 LLL_PRIVATE: lock only shared within a process
45 LLL_SHARED: lock may be shared across processes.
46
47 Condition variables contain an optimization for broadcasts that requeues
48 waiting threads on a lock's futex. Therefore, there is a special
49 variant of the locks (whose name contains "cond") that makes sure to
50 always set the lock state to >1 and not just 1.
51
52 Robust locks set the lock to the id of the owner. This allows detection
53 of the case where the owner exits without releasing the lock. Flags are
54 OR'd with the owner id to record additional information about lock state.
55 Therefore the states of robust locks are:
56 0: not acquired
57 id: acquired (by user identified by id & FUTEX_TID_MASK)
58
59 The following flags may be set in the robust lock value:
60 FUTEX_WAITERS - possibly has waiters
61 FUTEX_OWNER_DIED - owning user has exited without releasing the futex. */
62
63
64/* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return
65 0. Otherwise leave lock unchanged and return non-zero to indicate that the
66 lock was not acquired. */
67#define __lll_trylock(lock) \
68 __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0))
69#define lll_trylock(lock) \
70 __lll_trylock (&(lock))
71
72/* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and
73 return 0. Otherwise leave lock unchanged and return non-zero to indicate
74 that the lock was not acquired. */
75#define lll_cond_trylock(lock) \
76 __glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 2, 0))
77
78extern void __lll_lock_wait_private (int *futex) attribute_hidden;
79extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
80
81/* This is an expression rather than a statement even though its value is
82 void, so that it can be used in a comma expression or as an expression
83 that's cast to void. */
84/* The inner conditional compiles to a call to __lll_lock_wait_private if
85 private is known at compile time to be LLL_PRIVATE, and to a call to
86 __lll_lock_wait otherwise. */
87/* If FUTEX is 0 (not acquired), set to 1 (acquired with no waiters) and
88 return. Otherwise, ensure that it is >1 (acquired, possibly with waiters)
89 and then block until we acquire the lock, at which point FUTEX will still be
90 >1. The lock is always acquired on return. */
91#define __lll_lock(futex, private) \
92 ((void) \
93 ({ \
94 int *__futex = (futex); \
95 if (__glibc_unlikely \
96 (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \
97 { \
98 if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
99 __lll_lock_wait_private (__futex); \
100 else \
101 __lll_lock_wait (__futex, private); \
102 } \
103 }))
104#define lll_lock(futex, private) \
105 __lll_lock (&(futex), private)
106
107
108/* This is an expression rather than a statement even though its value is
109 void, so that it can be used in a comma expression or as an expression
110 that's cast to void. */
111/* Unconditionally set FUTEX to 2 (acquired, possibly with waiters). If FUTEX
112 was 0 (not acquired) then return. Otherwise, block until the lock is
113 acquired, at which point FUTEX is 2 (acquired, possibly with waiters). The
114 lock is always acquired on return. */
115#define __lll_cond_lock(futex, private) \
116 ((void) \
117 ({ \
118 int *__futex = (futex); \
119 if (__glibc_unlikely (atomic_exchange_acq (__futex, 2) != 0)) \
120 __lll_lock_wait (__futex, private); \
121 }))
122#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
123
124
125extern int __lll_clocklock_wait (int *futex, int val, clockid_t,
126 const struct timespec *,
127 int private) attribute_hidden;
128
129#define lll_timedwait(futex, val, clockid, abstime, private) \
130 __lll_clocklock_wait (futex, val, clockid, abstime, private)
131
132/* As __lll_lock, but with an absolute timeout measured against the clock
133 specified in CLOCKID. If the timeout occurs then return ETIMEDOUT. If
134 ABSTIME is invalid, return EINVAL. */
135#define __lll_clocklock(futex, clockid, abstime, private) \
136 ({ \
137 int *__futex = (futex); \
138 int __val = 0; \
139 \
140 if (__glibc_unlikely \
141 (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \
142 { \
143 while (atomic_exchange_acq (futex, 2) != 0) \
144 { \
145 __val = __lll_clocklock_wait (__futex, 2, clockid, \
146 abstime, private); \
147 if (__val == EINVAL || __val == ETIMEDOUT) \
148 break; \
149 } \
150 } \
151 __val; \
152 })
153#define lll_clocklock(futex, clockid, abstime, private) \
154 __lll_clocklock (&(futex), clockid, abstime, private)
155
156
157/* This is an expression rather than a statement even though its value is
158 void, so that it can be used in a comma expression or as an expression
159 that's cast to void. */
160/* Unconditionally set FUTEX to 0 (not acquired), releasing the lock. If FUTEX
161 was >1 (acquired, possibly with waiters), then wake any waiters. The waiter
162 that acquires the lock will set FUTEX to >1.
163 Evaluate PRIVATE before releasing the lock so that we do not violate the
164 mutex destruction requirements. Specifically, we need to ensure that
165 another thread can destroy the mutex (and reuse its memory) once it
166 acquires the lock and when there will be no further lock acquisitions;
167 thus, we must not access the lock after releasing it, or those accesses
168 could be concurrent with mutex destruction or reuse of the memory. */
169#define __lll_unlock(futex, private) \
170 ((void) \
171 ({ \
172 int *__futex = (futex); \
173 int __private = (private); \
174 int __oldval = atomic_exchange_rel (__futex, 0); \
175 if (__glibc_unlikely (__oldval > 1)) \
176 lll_futex_wake (__futex, 1, __private); \
177 }))
178#define lll_unlock(futex, private) \
179 __lll_unlock (&(futex), private)
180
181
182#define lll_islocked(futex) \
183 ((futex) != LLL_LOCK_INITIALIZER)
184
185
186/* Our internal lock implementation is identical to the binary-compatible
187 mutex implementation. */
188
189/* Initializers for lock. */
190#define LLL_LOCK_INITIALIZER (0)
191#define LLL_LOCK_INITIALIZER_LOCKED (1)
192
193#endif /* lowlevellock.h */
194