1 | /* |
2 | * Atomic operations on 64-bit quantities. |
3 | * |
4 | * Copyright (C) 2017 Red Hat, Inc. |
5 | * |
6 | * Author: Paolo Bonzini <pbonzini@redhat.com> |
7 | * |
8 | * This work is licensed under the terms of the GNU GPL, version 2 or later. |
9 | * See the COPYING file in the top-level directory. |
10 | */ |
11 | |
12 | #ifndef QEMU_STATS64_H |
13 | #define QEMU_STATS64_H |
14 | |
15 | #include "qemu/atomic.h" |
16 | |
17 | /* This provides atomic operations on 64-bit type, using a reader-writer |
18 | * spinlock on architectures that do not have 64-bit accesses. Even on |
19 | * those architectures, it tries hard not to take the lock. |
20 | */ |
21 | |
22 | typedef struct Stat64 { |
23 | #ifdef CONFIG_ATOMIC64 |
24 | uint64_t value; |
25 | #else |
26 | uint32_t low, high; |
27 | uint32_t lock; |
28 | #endif |
29 | } Stat64; |
30 | |
31 | #ifdef CONFIG_ATOMIC64 |
32 | static inline void stat64_init(Stat64 *s, uint64_t value) |
33 | { |
34 | /* This is not guaranteed to be atomic! */ |
35 | *s = (Stat64) { value }; |
36 | } |
37 | |
38 | static inline uint64_t stat64_get(const Stat64 *s) |
39 | { |
40 | return atomic_read__nocheck(&s->value); |
41 | } |
42 | |
43 | static inline void stat64_add(Stat64 *s, uint64_t value) |
44 | { |
45 | atomic_add(&s->value, value); |
46 | } |
47 | |
48 | static inline void stat64_min(Stat64 *s, uint64_t value) |
49 | { |
50 | uint64_t orig = atomic_read__nocheck(&s->value); |
51 | while (orig > value) { |
52 | orig = atomic_cmpxchg__nocheck(&s->value, orig, value); |
53 | } |
54 | } |
55 | |
56 | static inline void stat64_max(Stat64 *s, uint64_t value) |
57 | { |
58 | uint64_t orig = atomic_read__nocheck(&s->value); |
59 | while (orig < value) { |
60 | orig = atomic_cmpxchg__nocheck(&s->value, orig, value); |
61 | } |
62 | } |
63 | #else |
64 | uint64_t stat64_get(const Stat64 *s); |
65 | bool stat64_min_slow(Stat64 *s, uint64_t value); |
66 | bool stat64_max_slow(Stat64 *s, uint64_t value); |
67 | bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high); |
68 | |
69 | static inline void stat64_init(Stat64 *s, uint64_t value) |
70 | { |
71 | /* This is not guaranteed to be atomic! */ |
72 | *s = (Stat64) { .low = value, .high = value >> 32, .lock = 0 }; |
73 | } |
74 | |
75 | static inline void stat64_add(Stat64 *s, uint64_t value) |
76 | { |
77 | uint32_t low, high; |
78 | high = value >> 32; |
79 | low = (uint32_t) value; |
80 | if (!low) { |
81 | if (high) { |
82 | atomic_add(&s->high, high); |
83 | } |
84 | return; |
85 | } |
86 | |
87 | for (;;) { |
88 | uint32_t orig = s->low; |
89 | uint32_t result = orig + low; |
90 | uint32_t old; |
91 | |
92 | if (result < low || high) { |
93 | /* If the high part is affected, take the lock. */ |
94 | if (stat64_add32_carry(s, low, high)) { |
95 | return; |
96 | } |
97 | continue; |
98 | } |
99 | |
100 | /* No carry, try with a 32-bit cmpxchg. The result is independent of |
101 | * the high 32 bits, so it can race just fine with stat64_add32_carry |
102 | * and even stat64_get! |
103 | */ |
104 | old = atomic_cmpxchg(&s->low, orig, result); |
105 | if (orig == old) { |
106 | return; |
107 | } |
108 | } |
109 | } |
110 | |
111 | static inline void stat64_min(Stat64 *s, uint64_t value) |
112 | { |
113 | uint32_t low, high; |
114 | uint32_t orig_low, orig_high; |
115 | |
116 | high = value >> 32; |
117 | low = (uint32_t) value; |
118 | do { |
119 | orig_high = atomic_read(&s->high); |
120 | if (orig_high < high) { |
121 | return; |
122 | } |
123 | |
124 | if (orig_high == high) { |
125 | /* High 32 bits are equal. Read low after high, otherwise we |
126 | * can get a false positive (e.g. 0x1235,0x0000 changes to |
127 | * 0x1234,0x8000 and we read it as 0x1234,0x0000). Pairs with |
128 | * the write barrier in stat64_min_slow. |
129 | */ |
130 | smp_rmb(); |
131 | orig_low = atomic_read(&s->low); |
132 | if (orig_low <= low) { |
133 | return; |
134 | } |
135 | |
136 | /* See if we were lucky and a writer raced against us. The |
137 | * barrier is theoretically unnecessary, but if we remove it |
138 | * we may miss being lucky. |
139 | */ |
140 | smp_rmb(); |
141 | orig_high = atomic_read(&s->high); |
142 | if (orig_high < high) { |
143 | return; |
144 | } |
145 | } |
146 | |
147 | /* If the value changes in any way, we have to take the lock. */ |
148 | } while (!stat64_min_slow(s, value)); |
149 | } |
150 | |
151 | static inline void stat64_max(Stat64 *s, uint64_t value) |
152 | { |
153 | uint32_t low, high; |
154 | uint32_t orig_low, orig_high; |
155 | |
156 | high = value >> 32; |
157 | low = (uint32_t) value; |
158 | do { |
159 | orig_high = atomic_read(&s->high); |
160 | if (orig_high > high) { |
161 | return; |
162 | } |
163 | |
164 | if (orig_high == high) { |
165 | /* High 32 bits are equal. Read low after high, otherwise we |
166 | * can get a false positive (e.g. 0x1234,0x8000 changes to |
167 | * 0x1235,0x0000 and we read it as 0x1235,0x8000). Pairs with |
168 | * the write barrier in stat64_max_slow. |
169 | */ |
170 | smp_rmb(); |
171 | orig_low = atomic_read(&s->low); |
172 | if (orig_low >= low) { |
173 | return; |
174 | } |
175 | |
176 | /* See if we were lucky and a writer raced against us. The |
177 | * barrier is theoretically unnecessary, but if we remove it |
178 | * we may miss being lucky. |
179 | */ |
180 | smp_rmb(); |
181 | orig_high = atomic_read(&s->high); |
182 | if (orig_high > high) { |
183 | return; |
184 | } |
185 | } |
186 | |
187 | /* If the value changes in any way, we have to take the lock. */ |
188 | } while (!stat64_max_slow(s, value)); |
189 | } |
190 | |
191 | #endif |
192 | |
193 | #endif |
194 | |