1/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
3#ident "$Id$"
4/*======
5This file is part of PerconaFT.
6
7
8Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
9
10 PerconaFT is free software: you can redistribute it and/or modify
11 it under the terms of the GNU General Public License, version 2,
12 as published by the Free Software Foundation.
13
14 PerconaFT is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
21
22----------------------------------------
23
24 PerconaFT is free software: you can redistribute it and/or modify
25 it under the terms of the GNU Affero General Public License, version 3,
26 as published by the Free Software Foundation.
27
28 PerconaFT is distributed in the hope that it will be useful,
29 but WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 GNU Affero General Public License for more details.
32
33 You should have received a copy of the GNU Affero General Public License
34 along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
35======= */
36
37#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
38
39#include <toku_assert.h>
40
41#include <util/context.h>
42#include <util/frwlock.h>
43
44toku_instr_key *frwlock_m_wait_read_key;
45
46namespace toku {
47
48 static __thread int thread_local_tid = -1;
49 static int get_local_tid() {
50 if (thread_local_tid == -1) {
51 thread_local_tid = toku_os_gettid();
52 }
53 return thread_local_tid;
54 }
55
56 void frwlock::init(toku_mutex_t *const mutex
57#if defined(TOKU_MYSQL_WITH_PFS)
58 ,
59 const toku_instr_key &rwlock_instr_key
60#endif
61 ) {
62 m_mutex = mutex;
63
64 m_num_readers = 0;
65 m_num_writers = 0;
66 m_num_want_write = 0;
67 m_num_want_read = 0;
68 m_num_signaled_readers = 0;
69 m_num_expensive_want_write = 0;
70#if defined(TOKU_MYSQL_WITH_PFS)
71 toku_pthread_rwlock_init(rwlock_instr_key, &m_rwlock, nullptr);
72#endif
73 toku_cond_init(toku_uninstrumented, &m_wait_read, nullptr);
74 m_queue_item_read.cond = &m_wait_read;
75 m_queue_item_read.next = nullptr;
76 m_wait_read_is_in_queue = false;
77 m_current_writer_expensive = false;
78 m_read_wait_expensive = false;
79 m_current_writer_tid = -1;
80 m_blocking_writer_context_id = CTX_INVALID;
81
82 m_wait_head = nullptr;
83 m_wait_tail = nullptr;
84 }
85
86 void frwlock::deinit(void) {
87 toku_cond_destroy(&m_wait_read);
88#if defined(TOKU_MYSQL_WITH_PFS)
89 toku_pthread_rwlock_destroy(&m_rwlock);
90#endif
91 }
92
93 bool frwlock::queue_is_empty(void) const { return m_wait_head == nullptr; }
94
95 void frwlock::enq_item(queue_item *const item) {
96 paranoid_invariant_null(item->next);
97 if (m_wait_tail != nullptr) {
98 m_wait_tail->next = item;
99 } else {
100 paranoid_invariant_null(m_wait_head);
101 m_wait_head = item;
102 }
103 m_wait_tail = item;
104 }
105
106 toku_cond_t *frwlock::deq_item(void) {
107 paranoid_invariant_notnull(m_wait_head);
108 paranoid_invariant_notnull(m_wait_tail);
109 queue_item *item = m_wait_head;
110 m_wait_head = m_wait_head->next;
111 if (m_wait_tail == item) {
112 m_wait_tail = nullptr;
113 }
114 return item->cond;
115 }
116
117 // Prerequisite: Holds m_mutex.
118 void frwlock::write_lock(bool expensive) {
119#if defined(TOKU_MYSQL_WITH_PFS)
120 /* Instrumentation start */
121 toku_rwlock_instrumentation rwlock_instr;
122 toku_instr_rwlock_wrlock_wait_start(
123 rwlock_instr, m_rwlock, __FILE__, __LINE__);
124#endif
125
126 toku_mutex_assert_locked(m_mutex);
127 if (this->try_write_lock(expensive)) {
128#if defined(TOKU_MYSQL_WITH_PFS)
129 /* Instrumentation end */
130 toku_instr_rwlock_wrlock_wait_end(rwlock_instr, 0);
131#endif
132 return;
133 }
134
135 toku_cond_t cond = TOKU_COND_INITIALIZER;
136 queue_item item = {.cond = &cond, .next = nullptr};
137 this->enq_item(&item);
138
139 // Wait for our turn.
140 ++m_num_want_write;
141 if (expensive) {
142 ++m_num_expensive_want_write;
143 }
144 if (m_num_writers == 0 && m_num_want_write == 1) {
145 // We are the first to want a write lock. No new readers can get the
146 // lock.
147 // Set our thread id and context for proper instrumentation.
148 // see: toku_context_note_frwlock_contention()
149 m_current_writer_tid = get_local_tid();
150 m_blocking_writer_context_id = toku_thread_get_context()->get_id();
151 }
152 toku_cond_wait(&cond, m_mutex);
153 toku_cond_destroy(&cond);
154
155 // Now it's our turn.
156 paranoid_invariant(m_num_want_write > 0);
157 paranoid_invariant_zero(m_num_readers);
158 paranoid_invariant_zero(m_num_writers);
159 paranoid_invariant_zero(m_num_signaled_readers);
160
161 // Not waiting anymore; grab the lock.
162 --m_num_want_write;
163 if (expensive) {
164 --m_num_expensive_want_write;
165 }
166 m_num_writers = 1;
167 m_current_writer_expensive = expensive;
168 m_current_writer_tid = get_local_tid();
169 m_blocking_writer_context_id = toku_thread_get_context()->get_id();
170
171#if defined(TOKU_MYSQL_WITH_PFS)
172 /* Instrumentation end */
173 toku_instr_rwlock_wrlock_wait_end(rwlock_instr, 0);
174#endif
175 }
176
177 bool frwlock::try_write_lock(bool expensive) {
178 toku_mutex_assert_locked(m_mutex);
179 if (m_num_readers > 0 || m_num_writers > 0 ||
180 m_num_signaled_readers > 0 || m_num_want_write > 0) {
181 return false;
182 }
183 // No one holds the lock. Grant the write lock.
184 paranoid_invariant_zero(m_num_want_write);
185 paranoid_invariant_zero(m_num_want_read);
186 m_num_writers = 1;
187 m_current_writer_expensive = expensive;
188 m_current_writer_tid = get_local_tid();
189 m_blocking_writer_context_id = toku_thread_get_context()->get_id();
190 return true;
191 }
192
193 void frwlock::read_lock(void) {
194#if defined(TOKU_MYSQL_WITH_PFS)
195 /* Instrumentation start */
196 toku_rwlock_instrumentation rwlock_instr;
197 toku_instr_rwlock_rdlock_wait_start(
198 rwlock_instr, m_rwlock, __FILE__, __LINE__);
199#endif
200 toku_mutex_assert_locked(m_mutex);
201 if (m_num_writers > 0 || m_num_want_write > 0) {
202 if (!m_wait_read_is_in_queue) {
203 // Throw the read cond_t onto the queue.
204 paranoid_invariant(m_num_signaled_readers == m_num_want_read);
205 m_queue_item_read.next = nullptr;
206 this->enq_item(&m_queue_item_read);
207 m_wait_read_is_in_queue = true;
208 paranoid_invariant(!m_read_wait_expensive);
209 m_read_wait_expensive = (m_current_writer_expensive ||
210 (m_num_expensive_want_write > 0));
211 }
212
213 // Note this contention event in engine status.
214 toku_context_note_frwlock_contention(
215 toku_thread_get_context()->get_id(),
216 m_blocking_writer_context_id);
217
218 // Wait for our turn.
219 ++m_num_want_read;
220 toku_cond_wait(&m_wait_read, m_mutex);
221
222 // Now it's our turn.
223 paranoid_invariant_zero(m_num_writers);
224 paranoid_invariant(m_num_want_read > 0);
225 paranoid_invariant(m_num_signaled_readers > 0);
226
227 // Not waiting anymore; grab the lock.
228 --m_num_want_read;
229 --m_num_signaled_readers;
230 }
231 ++m_num_readers;
232#if defined(TOKU_MYSQL_WITH_PFS)
233 /* Instrumentation end */
234 toku_instr_rwlock_rdlock_wait_end(rwlock_instr, 0);
235#endif
236 }
237
238 bool frwlock::try_read_lock(void) {
239 toku_mutex_assert_locked(m_mutex);
240 if (m_num_writers > 0 || m_num_want_write > 0) {
241 return false;
242 }
243 // No writer holds the lock.
244 // No writers are waiting.
245 // Grant the read lock.
246 ++m_num_readers;
247 return true;
248 }
249
250 void frwlock::maybe_signal_next_writer(void) {
251 if (m_num_want_write > 0 && m_num_signaled_readers == 0 &&
252 m_num_readers == 0) {
253 toku_cond_t *cond = this->deq_item();
254 paranoid_invariant(cond != &m_wait_read);
255 // Grant write lock to waiting writer.
256 paranoid_invariant(m_num_want_write > 0);
257 toku_cond_signal(cond);
258 }
259 }
260
261 void frwlock::read_unlock(void) {
262#ifdef TOKU_MYSQL_WITH_PFS
263 toku_instr_rwlock_unlock(m_rwlock);
264#endif
265 toku_mutex_assert_locked(m_mutex);
266 paranoid_invariant(m_num_writers == 0);
267 paranoid_invariant(m_num_readers > 0);
268 --m_num_readers;
269 this->maybe_signal_next_writer();
270 }
271
272 bool frwlock::read_lock_is_expensive(void) {
273 toku_mutex_assert_locked(m_mutex);
274 if (m_wait_read_is_in_queue) {
275 return m_read_wait_expensive;
276 } else {
277 return m_current_writer_expensive ||
278 (m_num_expensive_want_write > 0);
279 }
280 }
281
282 void frwlock::maybe_signal_or_broadcast_next(void) {
283 paranoid_invariant(m_num_signaled_readers == 0);
284
285 if (this->queue_is_empty()) {
286 paranoid_invariant(m_num_want_write == 0);
287 paranoid_invariant(m_num_want_read == 0);
288 return;
289 }
290 toku_cond_t *cond = this->deq_item();
291 if (cond == &m_wait_read) {
292 // Grant read locks to all waiting readers
293 paranoid_invariant(m_wait_read_is_in_queue);
294 paranoid_invariant(m_num_want_read > 0);
295 m_num_signaled_readers = m_num_want_read;
296 m_wait_read_is_in_queue = false;
297 m_read_wait_expensive = false;
298 toku_cond_broadcast(cond);
299 } else {
300 // Grant write lock to waiting writer.
301 paranoid_invariant(m_num_want_write > 0);
302 toku_cond_signal(cond);
303 }
304 }
305
306 void frwlock::write_unlock(void) {
307#if defined(TOKU_MYSQL_WITH_PFS)
308 toku_instr_rwlock_unlock(m_rwlock);
309#endif
310 toku_mutex_assert_locked(m_mutex);
311 paranoid_invariant(m_num_writers == 1);
312 m_num_writers = 0;
313 m_current_writer_expensive = false;
314 m_current_writer_tid = -1;
315 m_blocking_writer_context_id = CTX_INVALID;
316 this->maybe_signal_or_broadcast_next();
317 }
318 bool frwlock::write_lock_is_expensive(void) {
319 toku_mutex_assert_locked(m_mutex);
320 return (m_num_expensive_want_write > 0) || (m_current_writer_expensive);
321 }
322
323 uint32_t frwlock::users(void) const {
324 toku_mutex_assert_locked(m_mutex);
325 return m_num_readers + m_num_writers + m_num_want_read +
326 m_num_want_write;
327 }
328 uint32_t frwlock::blocked_users(void) const {
329 toku_mutex_assert_locked(m_mutex);
330 return m_num_want_read + m_num_want_write;
331 }
332 uint32_t frwlock::writers(void) const {
333 // this is sometimes called as "assert(lock->writers())" when we
334 // assume we have the write lock. if that's the assumption, we may
335 // not own the mutex, so we don't assert_locked here
336 return m_num_writers;
337 }
338 uint32_t frwlock::blocked_writers(void) const {
339 toku_mutex_assert_locked(m_mutex);
340 return m_num_want_write;
341 }
342 uint32_t frwlock::readers(void) const {
343 toku_mutex_assert_locked(m_mutex);
344 return m_num_readers;
345 }
346 uint32_t frwlock::blocked_readers(void) const {
347 toku_mutex_assert_locked(m_mutex);
348 return m_num_want_read;
349 }
350
351} // namespace toku
352