1/*
2 * cf_mutex.c
3 *
4 * Copyright (C) 2017 Aerospike, Inc.
5 *
6 * Portions may be licensed to Aerospike, Inc. under one or more contributor
7 * license agreements.
8 *
9 * This program is free software: you can redistribute it and/or modify it under
10 * the terms of the GNU Affero General Public License as published by the Free
11 * Software Foundation, either version 3 of the License, or (at your option) any
12 * later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
16 * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
17 * details.
18 *
19 * You should have received a copy of the GNU Affero General Public License
20 * along with this program. If not, see http://www.gnu.org/licenses/
21 */
22
23
24//==========================================================
25// Includes.
26//
27
28#include <cf_mutex.h>
29
30#include <stdbool.h>
31#include <stdio.h>
32#include <stdint.h>
33#include <unistd.h>
34
35#include <linux/futex.h>
36#include <sys/syscall.h>
37
38#include "fault.h"
39
40
41//==========================================================
42// Typedefs & constants.
43//
44
45#define FUTEX_SPIN_MAX 100
46
47
48//==========================================================
49// Inlines & macros.
50//
51
52inline static void
53sys_futex(void *uaddr, int op, int val)
54{
55 syscall(SYS_futex, uaddr, op, val, NULL, NULL, 0);
56}
57
58#define xchg(__ptr, __val) __sync_lock_test_and_set(__ptr, __val)
59#define cmpxchg(__ptr, __cmp, __set) __sync_val_compare_and_swap(__ptr, __cmp, __set)
60#define cpu_relax() asm volatile("pause\n": : :"memory")
61#define unlikely(__expr) __builtin_expect(!! (__expr), 0)
62#define likely(__expr) __builtin_expect(!! (__expr), 1)
63
64
65//==========================================================
66// Public API - cf_mutex.
67//
68
69void
70cf_mutex_lock(cf_mutex *m)
71{
72 if (likely(cmpxchg((uint32_t *)m, 0, 1) == 0)) {
73 return; // was not locked
74 }
75
76 if (m->u32 == 2) {
77 sys_futex(m, FUTEX_WAIT_PRIVATE, 2);
78 }
79
80 while (xchg((uint32_t *)m, 2) != 0) {
81 sys_futex(m, FUTEX_WAIT_PRIVATE, 2);
82 }
83}
84
85void
86cf_mutex_unlock(cf_mutex *m)
87{
88 uint32_t check = xchg((uint32_t *)m, 0);
89
90 if (unlikely(check == 2)) {
91 sys_futex(m, FUTEX_WAKE_PRIVATE, 1);
92 }
93 else if (unlikely(check == 0)) {
94 cf_crash(CF_MISC, "cf_mutex_unlock() on already unlocked mutex");
95 }
96}
97
98// Return true if lock success.
99bool
100cf_mutex_trylock(cf_mutex *m)
101{
102 if (cmpxchg((uint32_t *)m, 0, 1) == 0) {
103 return true; // was not locked
104 }
105
106 return false;
107}
108
109void
110cf_mutex_lock_spin(cf_mutex *m)
111{
112 for (int i = 0; i < FUTEX_SPIN_MAX; i++) {
113 if (cmpxchg((uint32_t *)m, 0, 1) == 0) {
114 return; // was not locked
115 }
116
117 cpu_relax();
118 }
119
120 if (m->u32 == 2) {
121 sys_futex(m, FUTEX_WAIT_PRIVATE, 2);
122 }
123
124 while (xchg((uint32_t *)m, 2) != 0) {
125 sys_futex(m, FUTEX_WAIT_PRIVATE, 2);
126 }
127}
128
129void
130cf_mutex_unlock_spin(cf_mutex *m)
131{
132 uint32_t check = xchg((uint32_t *)m, 0);
133
134 if (unlikely(check == 2)) {
135 // Spin and hope someone takes the lock.
136 for (int i = 0; i < FUTEX_SPIN_MAX; i++) {
137 if (m->u32 != 0) {
138 if (cmpxchg((uint32_t *)m, 1, 2) == 0) {
139 break;
140 }
141
142 return; // someone else took the lock
143 }
144
145 cpu_relax();
146 }
147
148 sys_futex(m, FUTEX_WAKE_PRIVATE, 1);
149 }
150 else if (unlikely(check == 0)) {
151 cf_crash(CF_MISC, "cf_mutex_unlock_spin() on already unlocked mutex");
152 }
153}
154
155
156//==========================================================
157// Public API - cf_condition.
158//
159
160void
161cf_condition_wait(cf_condition *c, cf_mutex *m)
162{
163 uint32_t seq = c->seq;
164
165 cf_mutex_unlock(m);
166 sys_futex(&c->seq, FUTEX_WAIT_PRIVATE, seq);
167 cf_mutex_lock(m);
168}
169
170void
171cf_condition_signal(cf_condition *c)
172{
173 __sync_fetch_and_add(&c->seq, 1);
174 sys_futex(&c->seq, FUTEX_WAKE_PRIVATE, 1);
175}
176