1/* Copyright (C) 2002-2020 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include "pthreadP.h"
23#include <lowlevellock.h>
24#include <futex-internal.h>
25
26#ifndef lll_trylock_elision
27#define lll_trylock_elision(a,t) lll_trylock(a)
28#endif
29
30#ifndef FORCE_ELISION
31#define FORCE_ELISION(m, s)
32#endif
33
34int
35__pthread_mutex_trylock (pthread_mutex_t *mutex)
36{
37 int oldval;
38 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
39
40 /* See concurrency notes regarding mutex type which is loaded from __kind
41 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
42 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
43 PTHREAD_MUTEX_TIMED_NP))
44 {
45 /* Recursive mutex. */
46 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
47 case PTHREAD_MUTEX_RECURSIVE_NP:
48 /* Check whether we already hold the mutex. */
49 if (mutex->__data.__owner == id)
50 {
51 /* Just bump the counter. */
52 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
53 /* Overflow of the counter. */
54 return EAGAIN;
55
56 ++mutex->__data.__count;
57 return 0;
58 }
59
60 if (lll_trylock (mutex->__data.__lock) == 0)
61 {
62 /* Record the ownership. */
63 mutex->__data.__owner = id;
64 mutex->__data.__count = 1;
65 ++mutex->__data.__nusers;
66 return 0;
67 }
68 break;
69
70 case PTHREAD_MUTEX_TIMED_ELISION_NP:
71 elision: __attribute__((unused))
72 if (lll_trylock_elision (mutex->__data.__lock,
73 mutex->__data.__elision) != 0)
74 break;
75 /* Don't record the ownership. */
76 return 0;
77
78 case PTHREAD_MUTEX_TIMED_NP:
79 FORCE_ELISION (mutex, goto elision);
80 /*FALL THROUGH*/
81 case PTHREAD_MUTEX_ADAPTIVE_NP:
82 case PTHREAD_MUTEX_ERRORCHECK_NP:
83 if (lll_trylock (mutex->__data.__lock) != 0)
84 break;
85
86 /* Record the ownership. */
87 mutex->__data.__owner = id;
88 ++mutex->__data.__nusers;
89
90 return 0;
91
92 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
93 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
94 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
95 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
96 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
97 &mutex->__data.__list.__next);
98 /* We need to set op_pending before starting the operation. Also
99 see comments at ENQUEUE_MUTEX. */
100 __asm ("" ::: "memory");
101
102 oldval = mutex->__data.__lock;
103 do
104 {
105 again:
106 if ((oldval & FUTEX_OWNER_DIED) != 0)
107 {
108 /* The previous owner died. Try locking the mutex. */
109 int newval = id | (oldval & FUTEX_WAITERS);
110
111 newval
112 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
113 newval, oldval);
114
115 if (newval != oldval)
116 {
117 oldval = newval;
118 goto again;
119 }
120
121 /* We got the mutex. */
122 mutex->__data.__count = 1;
123 /* But it is inconsistent unless marked otherwise. */
124 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
125
126 /* We must not enqueue the mutex before we have acquired it.
127 Also see comments at ENQUEUE_MUTEX. */
128 __asm ("" ::: "memory");
129 ENQUEUE_MUTEX (mutex);
130 /* We need to clear op_pending after we enqueue the mutex. */
131 __asm ("" ::: "memory");
132 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
133
134 /* Note that we deliberately exit here. If we fall
135 through to the end of the function __nusers would be
136 incremented which is not correct because the old
137 owner has to be discounted. */
138 return EOWNERDEAD;
139 }
140
141 /* Check whether we already hold the mutex. */
142 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
143 {
144 int kind = PTHREAD_MUTEX_TYPE (mutex);
145 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
146 {
147 /* We do not need to ensure ordering wrt another memory
148 access. Also see comments at ENQUEUE_MUTEX. */
149 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
150 NULL);
151 return EDEADLK;
152 }
153
154 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
155 {
156 /* We do not need to ensure ordering wrt another memory
157 access. */
158 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
159 NULL);
160
161 /* Just bump the counter. */
162 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
163 /* Overflow of the counter. */
164 return EAGAIN;
165
166 ++mutex->__data.__count;
167
168 return 0;
169 }
170 }
171
172 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
173 id, 0);
174 if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
175 {
176 /* We haven't acquired the lock as it is already acquired by
177 another owner. We do not need to ensure ordering wrt another
178 memory access. */
179 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
180
181 return EBUSY;
182 }
183
184 if (__builtin_expect (mutex->__data.__owner
185 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
186 {
187 /* This mutex is now not recoverable. */
188 mutex->__data.__count = 0;
189 if (oldval == id)
190 lll_unlock (mutex->__data.__lock,
191 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
192 /* FIXME This violates the mutex destruction requirements. See
193 __pthread_mutex_unlock_full. */
194 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
195 return ENOTRECOVERABLE;
196 }
197 }
198 while ((oldval & FUTEX_OWNER_DIED) != 0);
199
200 /* We must not enqueue the mutex before we have acquired it.
201 Also see comments at ENQUEUE_MUTEX. */
202 __asm ("" ::: "memory");
203 ENQUEUE_MUTEX (mutex);
204 /* We need to clear op_pending after we enqueue the mutex. */
205 __asm ("" ::: "memory");
206 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
207
208 mutex->__data.__owner = id;
209 ++mutex->__data.__nusers;
210 mutex->__data.__count = 1;
211
212 return 0;
213
214 /* The PI support requires the Linux futex system call. If that's not
215 available, pthread_mutex_init should never have allowed the type to
216 be set. So it will get the default case for an invalid type. */
217#ifdef __NR_futex
218 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
219 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
220 case PTHREAD_MUTEX_PI_NORMAL_NP:
221 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
222 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
223 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
224 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
225 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
226 {
227 int kind, robust;
228 {
229 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
230 in sysdeps/nptl/bits/thread-shared-types.h. */
231 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
232 kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
233 robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
234 }
235
236 if (robust)
237 {
238 /* Note: robust PI futexes are signaled by setting bit 0. */
239 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
240 (void *) (((uintptr_t) &mutex->__data.__list.__next)
241 | 1));
242 /* We need to set op_pending before starting the operation. Also
243 see comments at ENQUEUE_MUTEX. */
244 __asm ("" ::: "memory");
245 }
246
247 oldval = mutex->__data.__lock;
248
249 /* Check whether we already hold the mutex. */
250 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
251 {
252 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
253 {
254 /* We do not need to ensure ordering wrt another memory
255 access. */
256 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
257 return EDEADLK;
258 }
259
260 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
261 {
262 /* We do not need to ensure ordering wrt another memory
263 access. */
264 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
265
266 /* Just bump the counter. */
267 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
268 /* Overflow of the counter. */
269 return EAGAIN;
270
271 ++mutex->__data.__count;
272
273 return 0;
274 }
275 }
276
277 oldval
278 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
279 id, 0);
280
281 if (oldval != 0)
282 {
283 if ((oldval & FUTEX_OWNER_DIED) == 0)
284 {
285 /* We haven't acquired the lock as it is already acquired by
286 another owner. We do not need to ensure ordering wrt another
287 memory access. */
288 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
289
290 return EBUSY;
291 }
292
293 assert (robust);
294
295 /* The mutex owner died. The kernel will now take care of
296 everything. */
297 int private = (robust
298 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
299 : PTHREAD_MUTEX_PSHARED (mutex));
300 int e = INTERNAL_SYSCALL_CALL (futex, &mutex->__data.__lock,
301 __lll_private_flag (FUTEX_TRYLOCK_PI,
302 private), 0, 0);
303
304 if (INTERNAL_SYSCALL_ERROR_P (e)
305 && INTERNAL_SYSCALL_ERRNO (e) == EWOULDBLOCK)
306 {
307 /* The kernel has not yet finished the mutex owner death.
308 We do not need to ensure ordering wrt another memory
309 access. */
310 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
311
312 return EBUSY;
313 }
314
315 oldval = mutex->__data.__lock;
316 }
317
318 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
319 {
320 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
321
322 /* We got the mutex. */
323 mutex->__data.__count = 1;
324 /* But it is inconsistent unless marked otherwise. */
325 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
326
327 /* We must not enqueue the mutex before we have acquired it.
328 Also see comments at ENQUEUE_MUTEX. */
329 __asm ("" ::: "memory");
330 ENQUEUE_MUTEX (mutex);
331 /* We need to clear op_pending after we enqueue the mutex. */
332 __asm ("" ::: "memory");
333 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
334
335 /* Note that we deliberately exit here. If we fall
336 through to the end of the function __nusers would be
337 incremented which is not correct because the old owner
338 has to be discounted. */
339 return EOWNERDEAD;
340 }
341
342 if (robust
343 && __builtin_expect (mutex->__data.__owner
344 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
345 {
346 /* This mutex is now not recoverable. */
347 mutex->__data.__count = 0;
348
349 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
350 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
351
352 /* To the kernel, this will be visible after the kernel has
353 acquired the mutex in the syscall. */
354 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
355 return ENOTRECOVERABLE;
356 }
357
358 if (robust)
359 {
360 /* We must not enqueue the mutex before we have acquired it.
361 Also see comments at ENQUEUE_MUTEX. */
362 __asm ("" ::: "memory");
363 ENQUEUE_MUTEX_PI (mutex);
364 /* We need to clear op_pending after we enqueue the mutex. */
365 __asm ("" ::: "memory");
366 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
367 }
368
369 mutex->__data.__owner = id;
370 ++mutex->__data.__nusers;
371 mutex->__data.__count = 1;
372
373 return 0;
374 }
375#endif /* __NR_futex. */
376
377 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
378 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
379 case PTHREAD_MUTEX_PP_NORMAL_NP:
380 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
381 {
382 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
383 in sysdeps/nptl/bits/thread-shared-types.h. */
384 int kind = atomic_load_relaxed (&(mutex->__data.__kind))
385 & PTHREAD_MUTEX_KIND_MASK_NP;
386
387 oldval = mutex->__data.__lock;
388
389 /* Check whether we already hold the mutex. */
390 if (mutex->__data.__owner == id)
391 {
392 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
393 return EDEADLK;
394
395 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
396 {
397 /* Just bump the counter. */
398 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
399 /* Overflow of the counter. */
400 return EAGAIN;
401
402 ++mutex->__data.__count;
403
404 return 0;
405 }
406 }
407
408 int oldprio = -1, ceilval;
409 do
410 {
411 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
412 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
413
414 if (__pthread_current_priority () > ceiling)
415 {
416 if (oldprio != -1)
417 __pthread_tpp_change_priority (oldprio, -1);
418 return EINVAL;
419 }
420
421 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
422 if (retval)
423 return retval;
424
425 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
426 oldprio = ceiling;
427
428 oldval
429 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
430 ceilval | 1, ceilval);
431
432 if (oldval == ceilval)
433 break;
434 }
435 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
436
437 if (oldval != ceilval)
438 {
439 __pthread_tpp_change_priority (oldprio, -1);
440 break;
441 }
442
443 assert (mutex->__data.__owner == 0);
444 /* Record the ownership. */
445 mutex->__data.__owner = id;
446 ++mutex->__data.__nusers;
447 mutex->__data.__count = 1;
448
449 return 0;
450 }
451 break;
452
453 default:
454 /* Correct code cannot set any other type. */
455 return EINVAL;
456 }
457
458 return EBUSY;
459}
460
461#ifndef __pthread_mutex_trylock
462#ifndef pthread_mutex_trylock
463weak_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
464hidden_def (__pthread_mutex_trylock)
465#endif
466#endif
467