1/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
3#ident "$Id$"
4/*======
5This file is part of PerconaFT.
6
7
8Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
9
10 PerconaFT is free software: you can redistribute it and/or modify
11 it under the terms of the GNU General Public License, version 2,
12 as published by the Free Software Foundation.
13
14 PerconaFT is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
21
22----------------------------------------
23
24 PerconaFT is free software: you can redistribute it and/or modify
25 it under the terms of the GNU Affero General Public License, version 3,
26 as published by the Free Software Foundation.
27
28 PerconaFT is distributed in the hope that it will be useful,
29 but WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 GNU Affero General Public License for more details.
32
33 You should have received a copy of the GNU Affero General Public License
34 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
35======= */
36
37#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
38
39#pragma once
40
41#include <pthread.h>
42#include <time.h>
43#include <stdint.h>
44
45#include "toku_portability.h"
46#include "toku_assert.h"
47
48// TODO: some things moved toku_instrumentation.h, not necessarily the best
49// place
50typedef pthread_attr_t toku_pthread_attr_t;
51typedef pthread_t toku_pthread_t;
52typedef pthread_mutex_t toku_pthread_mutex_t;
53typedef pthread_condattr_t toku_pthread_condattr_t;
54typedef pthread_cond_t toku_pthread_cond_t;
55typedef pthread_rwlockattr_t toku_pthread_rwlockattr_t;
56typedef pthread_key_t toku_pthread_key_t;
57typedef struct timespec toku_timespec_t;
58
59// TODO: break this include loop
60#include <pthread.h>
61typedef pthread_mutexattr_t toku_pthread_mutexattr_t;
62
63struct toku_mutex_t {
64 pthread_mutex_t pmutex;
65 struct PSI_mutex
66 *psi_mutex; /* The performance schema instrumentation hook */
67#if TOKU_PTHREAD_DEBUG
68 pthread_t owner; // = pthread_self(); // for debugging
69 bool locked;
70 bool valid;
71 pfs_key_t instr_key_id;
72#endif
73};
74
75struct toku_cond_t {
76 pthread_cond_t pcond;
77 struct PSI_cond *psi_cond;
78#if TOKU_PTHREAD_DEBUG
79 pfs_key_t instr_key_id;
80#endif
81};
82
83#ifdef TOKU_PTHREAD_DEBUG
84#define TOKU_COND_INITIALIZER \
85 { \
86 .pcond = PTHREAD_COND_INITIALIZER, .psi_cond = nullptr, \
87 .instr_key_id = 0 \
88 }
89#else
90#define TOKU_COND_INITIALIZER \
91 { .pcond = PTHREAD_COND_INITIALIZER, .psi_cond = nullptr }
92#endif
93
94struct toku_pthread_rwlock_t {
95 pthread_rwlock_t rwlock;
96 struct PSI_rwlock *psi_rwlock;
97#if TOKU_PTHREAD_DEBUG
98 pfs_key_t instr_key_id;
99#endif
100};
101
102typedef struct toku_mutex_aligned {
103 toku_mutex_t aligned_mutex __attribute__((__aligned__(64)));
104} toku_mutex_aligned_t;
105
106// Initializing with {} will fill in a struct with all zeros.
107// But you may also need a pragma to suppress the warnings, as follows
108//
109// #pragma GCC diagnostic push
110// #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
111// toku_mutex_t foo = ZERO_MUTEX_INITIALIZER;
112// #pragma GCC diagnostic pop
113//
114// In general it will be a lot of busy work to make this codebase compile
115// cleanly with -Wmissing-field-initializers
116
117#define ZERO_MUTEX_INITIALIZER \
118 {}
119
120#if TOKU_PTHREAD_DEBUG
121#define TOKU_MUTEX_INITIALIZER \
122 { \
123 .pmutex = PTHREAD_MUTEX_INITIALIZER, .psi_mutex = nullptr, .owner = 0, \
124 .locked = false, .valid = true, .instr_key_id = 0 \
125 }
126#else
127#define TOKU_MUTEX_INITIALIZER \
128 { .pmutex = PTHREAD_MUTEX_INITIALIZER, .psi_mutex = nullptr }
129#endif
130
131// Darwin doesn't provide adaptive mutexes
132#if defined(__APPLE__)
133#define TOKU_MUTEX_ADAPTIVE PTHREAD_MUTEX_DEFAULT
134#if TOKU_PTHREAD_DEBUG
135#define TOKU_ADAPTIVE_MUTEX_INITIALIZER \
136 { \
137 .pmutex = PTHREAD_MUTEX_INITIALIZER, .psi_mutex = nullptr, .owner = 0, \
138 .locked = false, .valid = true, .instr_key_id = 0 \
139 }
140#else
141#define TOKU_ADAPTIVE_MUTEX_INITIALIZER \
142 { .pmutex = PTHREAD_MUTEX_INITIALIZER, .psi_mutex = nullptr }
143#endif
144#else // __FreeBSD__, __linux__, at least
145#define TOKU_MUTEX_ADAPTIVE PTHREAD_MUTEX_ADAPTIVE_NP
146#if TOKU_PTHREAD_DEBUG
147#define TOKU_ADAPTIVE_MUTEX_INITIALIZER \
148 { \
149 .pmutex = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP, .psi_mutex = nullptr, \
150 .owner = 0, .locked = false, .valid = true, .instr_key_id = 0 \
151 }
152#else
153#define TOKU_ADAPTIVE_MUTEX_INITIALIZER \
154 { .pmutex = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP, .psi_mutex = nullptr }
155#endif
156#endif
157
158// Different OSes implement mutexes as different amounts of nested structs.
159// C++ will fill out all missing values with zeroes if you provide at least one
160// zero, but it needs the right amount of nesting.
161#if defined(__FreeBSD__)
162#define ZERO_COND_INITIALIZER \
163 { 0 }
164#elif defined(__APPLE__)
165#define ZERO_COND_INITIALIZER \
166 { \
167 { 0 } \
168 }
169#else // __linux__, at least
170#define ZERO_COND_INITIALIZER \
171 {}
172#endif
173
174static inline void toku_mutexattr_init(toku_pthread_mutexattr_t *attr) {
175 int r = pthread_mutexattr_init(attr);
176 assert_zero(r);
177}
178
179static inline void
180toku_mutexattr_settype(toku_pthread_mutexattr_t *attr, int type) {
181 int r = pthread_mutexattr_settype(attr, type);
182 assert_zero(r);
183}
184
185static inline void
186toku_mutexattr_destroy(toku_pthread_mutexattr_t *attr) {
187 int r = pthread_mutexattr_destroy(attr);
188 assert_zero(r);
189}
190
191#if TOKU_PTHREAD_DEBUG
192static inline void toku_mutex_assert_locked(const toku_mutex_t *mutex) {
193 invariant(mutex->locked);
194 invariant(mutex->owner == pthread_self());
195}
196#else
197static inline void
198toku_mutex_assert_locked(const toku_mutex_t *mutex __attribute__((unused))) {
199}
200#endif
201
202// asserting that a mutex is unlocked only makes sense
203// if the calling thread can guaruntee that no other threads
204// are trying to lock this mutex at the time of the assertion
205//
206// a good example of this is a tree with mutexes on each node.
207// when a node is locked the caller knows that no other threads
208// can be trying to lock its childrens' mutexes. the children
209// are in one of two fixed states: locked or unlocked.
210#if TOKU_PTHREAD_DEBUG
211static inline void
212toku_mutex_assert_unlocked(toku_mutex_t *mutex) {
213 invariant(mutex->owner == 0);
214 invariant(!mutex->locked);
215}
216#else
217static inline void toku_mutex_assert_unlocked(toku_mutex_t *mutex
218 __attribute__((unused))) {}
219#endif
220
221#define toku_mutex_lock(M) \
222 toku_mutex_lock_with_source_location(M, __FILE__, __LINE__)
223
224static inline void toku_cond_init(toku_cond_t *cond,
225 const toku_pthread_condattr_t *attr) {
226 int r = pthread_cond_init(&cond->pcond, attr);
227 assert_zero(r);
228}
229
230#define toku_mutex_trylock(M) \
231 toku_mutex_trylock_with_source_location(M, __FILE__, __LINE__)
232
233inline void toku_mutex_unlock(toku_mutex_t *mutex) {
234#if TOKU_PTHREAD_DEBUG
235 invariant(mutex->owner == pthread_self());
236 invariant(mutex->valid);
237 invariant(mutex->locked);
238 mutex->locked = false;
239 mutex->owner = 0;
240#endif
241 toku_instr_mutex_unlock(mutex->psi_mutex);
242 int r = pthread_mutex_unlock(&mutex->pmutex);
243 assert_zero(r);
244}
245
246inline void toku_mutex_lock_with_source_location(toku_mutex_t *mutex,
247 const char *src_file,
248 int src_line) {
249
250 toku_mutex_instrumentation mutex_instr;
251 toku_instr_mutex_lock_start(mutex_instr, *mutex, src_file, src_line);
252
253 const int r = pthread_mutex_lock(&mutex->pmutex);
254 toku_instr_mutex_lock_end(mutex_instr, r);
255
256 assert_zero(r);
257#if TOKU_PTHREAD_DEBUG
258 invariant(mutex->valid);
259 invariant(!mutex->locked);
260 invariant(mutex->owner == 0);
261 mutex->locked = true;
262 mutex->owner = pthread_self();
263#endif
264}
265
266inline int toku_mutex_trylock_with_source_location(toku_mutex_t *mutex,
267 const char *src_file,
268 int src_line) {
269
270 toku_mutex_instrumentation mutex_instr;
271 toku_instr_mutex_trylock_start(mutex_instr, *mutex, src_file, src_line);
272
273 const int r = pthread_mutex_lock(&mutex->pmutex);
274 toku_instr_mutex_lock_end(mutex_instr, r);
275
276#if TOKU_PTHREAD_DEBUG
277 if (r == 0) {
278 invariant(mutex->valid);
279 invariant(!mutex->locked);
280 invariant(mutex->owner == 0);
281 mutex->locked = true;
282 mutex->owner = pthread_self();
283 }
284#endif
285 return r;
286}
287
288#define toku_cond_wait(C, M) \
289 toku_cond_wait_with_source_location(C, M, __FILE__, __LINE__)
290
291#define toku_cond_timedwait(C, M, W) \
292 toku_cond_timedwait_with_source_location(C, M, W, __FILE__, __LINE__)
293
294inline void toku_cond_init(const toku_instr_key &key,
295 toku_cond_t *cond,
296 const pthread_condattr_t *attr) {
297 toku_instr_cond_init(key, *cond);
298 int r = pthread_cond_init(&cond->pcond, attr);
299 assert_zero(r);
300}
301
302inline void toku_cond_destroy(toku_cond_t *cond) {
303 toku_instr_cond_destroy(cond->psi_cond);
304 int r = pthread_cond_destroy(&cond->pcond);
305 assert_zero(r);
306}
307
308inline void toku_cond_wait_with_source_location(toku_cond_t *cond,
309 toku_mutex_t *mutex,
310 const char *src_file,
311 uint src_line) {
312
313#if TOKU_PTHREAD_DEBUG
314 invariant(mutex->locked);
315 mutex->locked = false;
316 mutex->owner = 0;
317#endif
318
319 /* Instrumentation start */
320 toku_cond_instrumentation cond_instr;
321 toku_instr_cond_wait_start(cond_instr,
322 toku_instr_cond_op::cond_wait,
323 *cond,
324 *mutex,
325 src_file,
326 src_line);
327
328 /* Instrumented code */
329 const int r = pthread_cond_wait(&cond->pcond, &mutex->pmutex);
330
331 /* Instrumentation end */
332 toku_instr_cond_wait_end(cond_instr, r);
333
334 assert_zero(r);
335#if TOKU_PTHREAD_DEBUG
336 invariant(!mutex->locked);
337 mutex->locked = true;
338 mutex->owner = pthread_self();
339#endif
340}
341
342inline int toku_cond_timedwait_with_source_location(toku_cond_t *cond,
343 toku_mutex_t *mutex,
344 toku_timespec_t *wakeup_at,
345 const char *src_file,
346 uint src_line) {
347#if TOKU_PTHREAD_DEBUG
348 invariant(mutex->locked);
349 mutex->locked = false;
350 mutex->owner = 0;
351#endif
352
353 /* Instrumentation start */
354 toku_cond_instrumentation cond_instr;
355 toku_instr_cond_wait_start(cond_instr,
356 toku_instr_cond_op::cond_timedwait,
357 *cond,
358 *mutex,
359 src_file,
360 src_line);
361
362 /* Instrumented code */
363 const int r = pthread_cond_timedwait(
364 &cond->pcond, &mutex->pmutex, wakeup_at);
365
366 /* Instrumentation end */
367 toku_instr_cond_wait_end(cond_instr, r);
368
369#if TOKU_PTHREAD_DEBUG
370 invariant(!mutex->locked);
371 mutex->locked = true;
372 mutex->owner = pthread_self();
373#endif
374 return r;
375}
376
377inline void toku_cond_signal(toku_cond_t *cond) {
378 toku_instr_cond_signal(*cond);
379 const int r = pthread_cond_signal(&cond->pcond);
380 assert_zero(r);
381}
382
383inline void toku_cond_broadcast(toku_cond_t *cond) {
384 toku_instr_cond_broadcast(*cond);
385 const int r = pthread_cond_broadcast(&cond->pcond);
386 assert_zero(r);
387}
388
389inline void toku_mutex_init(const toku_instr_key &key,
390 toku_mutex_t *mutex,
391 const toku_pthread_mutexattr_t *attr) {
392#if TOKU_PTHREAD_DEBUG
393 mutex->valid = true;
394#endif
395 toku_instr_mutex_init(key, *mutex);
396 const int r = pthread_mutex_init(&mutex->pmutex, attr);
397 assert_zero(r);
398#if TOKU_PTHREAD_DEBUG
399 mutex->locked = false;
400 invariant(mutex->valid);
401 mutex->valid = true;
402 mutex->owner = 0;
403#endif
404}
405
406inline void toku_mutex_destroy(toku_mutex_t *mutex) {
407#if TOKU_PTHREAD_DEBUG
408 invariant(mutex->valid);
409 mutex->valid = false;
410 invariant(!mutex->locked);
411#endif
412 toku_instr_mutex_destroy(mutex->psi_mutex);
413 int r = pthread_mutex_destroy(&mutex->pmutex);
414 assert_zero(r);
415}
416
417#define toku_pthread_rwlock_rdlock(RW) \
418 toku_pthread_rwlock_rdlock_with_source_location(RW, __FILE__, __LINE__)
419
420#define toku_pthread_rwlock_wrlock(RW) \
421 toku_pthread_rwlock_wrlock_with_source_location(RW, __FILE__, __LINE__)
422
423inline void toku_pthread_rwlock_init(
424 const toku_instr_key &key,
425 toku_pthread_rwlock_t *__restrict rwlock,
426 const toku_pthread_rwlockattr_t *__restrict attr) {
427 toku_instr_rwlock_init(key, *rwlock);
428 int r = pthread_rwlock_init(&rwlock->rwlock, attr);
429 assert_zero(r);
430}
431
432inline void toku_pthread_rwlock_destroy(toku_pthread_rwlock_t *rwlock) {
433 toku_instr_rwlock_destroy(rwlock->psi_rwlock);
434 int r = pthread_rwlock_destroy(&rwlock->rwlock);
435 assert_zero(r);
436}
437
438inline void toku_pthread_rwlock_rdlock_with_source_location(
439 toku_pthread_rwlock_t *rwlock,
440 const char *src_file,
441 uint src_line) {
442
443 /* Instrumentation start */
444 toku_rwlock_instrumentation rwlock_instr;
445 toku_instr_rwlock_rdlock_wait_start(
446 rwlock_instr, *rwlock, src_file, src_line);
447 /* Instrumented code */
448 const int r = pthread_rwlock_rdlock(&rwlock->rwlock);
449
450 /* Instrumentation end */
451 toku_instr_rwlock_rdlock_wait_end(rwlock_instr, r);
452
453 assert_zero(r);
454}
455
456inline void toku_pthread_rwlock_wrlock_with_source_location(
457 toku_pthread_rwlock_t *rwlock,
458 const char *src_file,
459 uint src_line) {
460
461 /* Instrumentation start */
462 toku_rwlock_instrumentation rwlock_instr;
463 toku_instr_rwlock_wrlock_wait_start(
464 rwlock_instr, *rwlock, src_file, src_line);
465 /* Instrumented code */
466 const int r = pthread_rwlock_wrlock(&rwlock->rwlock);
467
468 /* Instrumentation end */
469 toku_instr_rwlock_wrlock_wait_end(rwlock_instr, r);
470
471 assert_zero(r);
472}
473
474inline void toku_pthread_rwlock_rdunlock(toku_pthread_rwlock_t *rwlock) {
475 toku_instr_rwlock_unlock(*rwlock);
476 const int r = pthread_rwlock_unlock(&rwlock->rwlock);
477 assert_zero(r);
478}
479
480inline void toku_pthread_rwlock_wrunlock(toku_pthread_rwlock_t *rwlock) {
481 toku_instr_rwlock_unlock(*rwlock);
482 const int r = pthread_rwlock_unlock(&rwlock->rwlock);
483 assert_zero(r);
484}
485
486static inline int toku_pthread_join(toku_pthread_t thread, void **value_ptr) {
487 return pthread_join(thread, value_ptr);
488}
489
490static inline int
491toku_pthread_detach(toku_pthread_t thread) {
492 return pthread_detach(thread);
493}
494
495static inline int
496toku_pthread_key_create(toku_pthread_key_t *key, void (*destroyf)(void *)) {
497 return pthread_key_create(key, destroyf);
498}
499
500static inline int
501toku_pthread_key_delete(toku_pthread_key_t key) {
502 return pthread_key_delete(key);
503}
504
505static inline void *
506toku_pthread_getspecific(toku_pthread_key_t key) {
507 return pthread_getspecific(key);
508}
509
510static inline int toku_pthread_setspecific(toku_pthread_key_t key, void *data) {
511 return pthread_setspecific(key, data);
512}
513
514int toku_pthread_yield(void) __attribute__((__visibility__("default")));
515
516static inline toku_pthread_t toku_pthread_self(void) { return pthread_self(); }
517
518static inline void *toku_pthread_done(void *exit_value) {
519 toku_instr_delete_current_thread();
520 pthread_exit(exit_value);
521}
522