1 | //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the code for emitting atomic operations. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "CGCall.h" |
14 | #include "CGRecordLayout.h" |
15 | #include "CodeGenFunction.h" |
16 | #include "CodeGenModule.h" |
17 | #include "TargetInfo.h" |
18 | #include "clang/AST/ASTContext.h" |
19 | #include "clang/CodeGen/CGFunctionInfo.h" |
20 | #include "clang/Frontend/FrontendDiagnostic.h" |
21 | #include "llvm/ADT/DenseMap.h" |
22 | #include "llvm/IR/DataLayout.h" |
23 | #include "llvm/IR/Intrinsics.h" |
24 | #include "llvm/IR/Operator.h" |
25 | |
26 | using namespace clang; |
27 | using namespace CodeGen; |
28 | |
29 | namespace { |
30 | class AtomicInfo { |
31 | CodeGenFunction &CGF; |
32 | QualType AtomicTy; |
33 | QualType ValueTy; |
34 | uint64_t AtomicSizeInBits; |
35 | uint64_t ValueSizeInBits; |
36 | CharUnits AtomicAlign; |
37 | CharUnits ValueAlign; |
38 | TypeEvaluationKind EvaluationKind; |
39 | bool UseLibcall; |
40 | LValue LVal; |
41 | CGBitFieldInfo BFI; |
42 | public: |
43 | AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) |
44 | : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), |
45 | EvaluationKind(TEK_Scalar), UseLibcall(true) { |
46 | assert(!lvalue.isGlobalReg()); |
47 | ASTContext &C = CGF.getContext(); |
48 | if (lvalue.isSimple()) { |
49 | AtomicTy = lvalue.getType(); |
50 | if (auto *ATy = AtomicTy->getAs<AtomicType>()) |
51 | ValueTy = ATy->getValueType(); |
52 | else |
53 | ValueTy = AtomicTy; |
54 | EvaluationKind = CGF.getEvaluationKind(ValueTy); |
55 | |
56 | uint64_t ValueAlignInBits; |
57 | uint64_t AtomicAlignInBits; |
58 | TypeInfo ValueTI = C.getTypeInfo(ValueTy); |
59 | ValueSizeInBits = ValueTI.Width; |
60 | ValueAlignInBits = ValueTI.Align; |
61 | |
62 | TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); |
63 | AtomicSizeInBits = AtomicTI.Width; |
64 | AtomicAlignInBits = AtomicTI.Align; |
65 | |
66 | assert(ValueSizeInBits <= AtomicSizeInBits); |
67 | assert(ValueAlignInBits <= AtomicAlignInBits); |
68 | |
69 | AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); |
70 | ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); |
71 | if (lvalue.getAlignment().isZero()) |
72 | lvalue.setAlignment(AtomicAlign); |
73 | |
74 | LVal = lvalue; |
75 | } else if (lvalue.isBitField()) { |
76 | ValueTy = lvalue.getType(); |
77 | ValueSizeInBits = C.getTypeSize(ValueTy); |
78 | auto &OrigBFI = lvalue.getBitFieldInfo(); |
79 | auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); |
80 | AtomicSizeInBits = C.toBits( |
81 | C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) |
82 | .alignTo(lvalue.getAlignment())); |
83 | llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); |
84 | auto OffsetInChars = |
85 | (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * |
86 | lvalue.getAlignment(); |
87 | llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( |
88 | CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); |
89 | StoragePtr = CGF.Builder.CreateAddrSpaceCast( |
90 | StoragePtr, llvm::PointerType::getUnqual(CGF.getLLVMContext()), |
91 | "atomic_bitfield_base" ); |
92 | BFI = OrigBFI; |
93 | BFI.Offset = Offset; |
94 | BFI.StorageSize = AtomicSizeInBits; |
95 | BFI.StorageOffset += OffsetInChars; |
96 | llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); |
97 | LVal = LValue::MakeBitfield( |
98 | Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, |
99 | lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); |
100 | AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); |
101 | if (AtomicTy.isNull()) { |
102 | llvm::APInt Size( |
103 | /*numBits=*/32, |
104 | C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); |
105 | AtomicTy = |
106 | C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, |
107 | /*IndexTypeQuals=*/0); |
108 | } |
109 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
110 | } else if (lvalue.isVectorElt()) { |
111 | ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); |
112 | ValueSizeInBits = C.getTypeSize(ValueTy); |
113 | AtomicTy = lvalue.getType(); |
114 | AtomicSizeInBits = C.getTypeSize(AtomicTy); |
115 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
116 | LVal = lvalue; |
117 | } else { |
118 | assert(lvalue.isExtVectorElt()); |
119 | ValueTy = lvalue.getType(); |
120 | ValueSizeInBits = C.getTypeSize(ValueTy); |
121 | AtomicTy = ValueTy = CGF.getContext().getExtVectorType( |
122 | lvalue.getType(), cast<llvm::FixedVectorType>( |
123 | lvalue.getExtVectorAddress().getElementType()) |
124 | ->getNumElements()); |
125 | AtomicSizeInBits = C.getTypeSize(AtomicTy); |
126 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
127 | LVal = lvalue; |
128 | } |
129 | UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( |
130 | AtomicSizeInBits, C.toBits(lvalue.getAlignment())); |
131 | } |
132 | |
133 | QualType getAtomicType() const { return AtomicTy; } |
134 | QualType getValueType() const { return ValueTy; } |
135 | CharUnits getAtomicAlignment() const { return AtomicAlign; } |
136 | uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } |
137 | uint64_t getValueSizeInBits() const { return ValueSizeInBits; } |
138 | TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } |
139 | bool shouldUseLibcall() const { return UseLibcall; } |
140 | const LValue &getAtomicLValue() const { return LVal; } |
141 | llvm::Value *getAtomicPointer() const { |
142 | if (LVal.isSimple()) |
143 | return LVal.getPointer(CGF); |
144 | else if (LVal.isBitField()) |
145 | return LVal.getBitFieldPointer(); |
146 | else if (LVal.isVectorElt()) |
147 | return LVal.getVectorPointer(); |
148 | assert(LVal.isExtVectorElt()); |
149 | return LVal.getExtVectorPointer(); |
150 | } |
151 | Address getAtomicAddress() const { |
152 | llvm::Type *ElTy; |
153 | if (LVal.isSimple()) |
154 | ElTy = LVal.getAddress(CGF).getElementType(); |
155 | else if (LVal.isBitField()) |
156 | ElTy = LVal.getBitFieldAddress().getElementType(); |
157 | else if (LVal.isVectorElt()) |
158 | ElTy = LVal.getVectorAddress().getElementType(); |
159 | else |
160 | ElTy = LVal.getExtVectorAddress().getElementType(); |
161 | return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); |
162 | } |
163 | |
164 | Address getAtomicAddressAsAtomicIntPointer() const { |
165 | return castToAtomicIntPointer(getAtomicAddress()); |
166 | } |
167 | |
168 | /// Is the atomic size larger than the underlying value type? |
169 | /// |
170 | /// Note that the absence of padding does not mean that atomic |
171 | /// objects are completely interchangeable with non-atomic |
172 | /// objects: we might have promoted the alignment of a type |
173 | /// without making it bigger. |
174 | bool hasPadding() const { |
175 | return (ValueSizeInBits != AtomicSizeInBits); |
176 | } |
177 | |
178 | bool emitMemSetZeroIfNecessary() const; |
179 | |
180 | llvm::Value *getAtomicSizeValue() const { |
181 | CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); |
182 | return CGF.CGM.getSize(size); |
183 | } |
184 | |
185 | /// Cast the given pointer to an integer pointer suitable for atomic |
186 | /// operations if the source. |
187 | Address castToAtomicIntPointer(Address Addr) const; |
188 | |
189 | /// If Addr is compatible with the iN that will be used for an atomic |
190 | /// operation, bitcast it. Otherwise, create a temporary that is suitable |
191 | /// and copy the value across. |
192 | Address convertToAtomicIntPointer(Address Addr) const; |
193 | |
194 | /// Turn an atomic-layout object into an r-value. |
195 | RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, |
196 | SourceLocation loc, bool AsValue) const; |
197 | |
198 | /// Converts a rvalue to integer value. |
199 | llvm::Value *convertRValueToInt(RValue RVal) const; |
200 | |
201 | RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, |
202 | AggValueSlot ResultSlot, |
203 | SourceLocation Loc, bool AsValue) const; |
204 | |
205 | /// Copy an atomic r-value into atomic-layout memory. |
206 | void emitCopyIntoMemory(RValue rvalue) const; |
207 | |
208 | /// Project an l-value down to the value field. |
209 | LValue projectValue() const { |
210 | assert(LVal.isSimple()); |
211 | Address addr = getAtomicAddress(); |
212 | if (hasPadding()) |
213 | addr = CGF.Builder.CreateStructGEP(addr, 0); |
214 | |
215 | return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), |
216 | LVal.getBaseInfo(), LVal.getTBAAInfo()); |
217 | } |
218 | |
219 | /// Emits atomic load. |
220 | /// \returns Loaded value. |
221 | RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, |
222 | bool AsValue, llvm::AtomicOrdering AO, |
223 | bool IsVolatile); |
224 | |
225 | /// Emits atomic compare-and-exchange sequence. |
226 | /// \param Expected Expected value. |
227 | /// \param Desired Desired value. |
228 | /// \param Success Atomic ordering for success operation. |
229 | /// \param Failure Atomic ordering for failed operation. |
230 | /// \param IsWeak true if atomic operation is weak, false otherwise. |
231 | /// \returns Pair of values: previous value from storage (value type) and |
232 | /// boolean flag (i1 type) with true if success and false otherwise. |
233 | std::pair<RValue, llvm::Value *> |
234 | EmitAtomicCompareExchange(RValue Expected, RValue Desired, |
235 | llvm::AtomicOrdering Success = |
236 | llvm::AtomicOrdering::SequentiallyConsistent, |
237 | llvm::AtomicOrdering Failure = |
238 | llvm::AtomicOrdering::SequentiallyConsistent, |
239 | bool IsWeak = false); |
240 | |
241 | /// Emits atomic update. |
242 | /// \param AO Atomic ordering. |
243 | /// \param UpdateOp Update operation for the current lvalue. |
244 | void EmitAtomicUpdate(llvm::AtomicOrdering AO, |
245 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
246 | bool IsVolatile); |
247 | /// Emits atomic update. |
248 | /// \param AO Atomic ordering. |
249 | void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, |
250 | bool IsVolatile); |
251 | |
252 | /// Materialize an atomic r-value in atomic-layout memory. |
253 | Address materializeRValue(RValue rvalue) const; |
254 | |
255 | /// Creates temp alloca for intermediate operations on atomic value. |
256 | Address CreateTempAlloca() const; |
257 | private: |
258 | bool requiresMemSetZero(llvm::Type *type) const; |
259 | |
260 | |
261 | /// Emits atomic load as a libcall. |
262 | void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, |
263 | llvm::AtomicOrdering AO, bool IsVolatile); |
264 | /// Emits atomic load as LLVM instruction. |
265 | llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); |
266 | /// Emits atomic compare-and-exchange op as a libcall. |
267 | llvm::Value *EmitAtomicCompareExchangeLibcall( |
268 | llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, |
269 | llvm::AtomicOrdering Success = |
270 | llvm::AtomicOrdering::SequentiallyConsistent, |
271 | llvm::AtomicOrdering Failure = |
272 | llvm::AtomicOrdering::SequentiallyConsistent); |
273 | /// Emits atomic compare-and-exchange op as LLVM instruction. |
274 | std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( |
275 | llvm::Value *ExpectedVal, llvm::Value *DesiredVal, |
276 | llvm::AtomicOrdering Success = |
277 | llvm::AtomicOrdering::SequentiallyConsistent, |
278 | llvm::AtomicOrdering Failure = |
279 | llvm::AtomicOrdering::SequentiallyConsistent, |
280 | bool IsWeak = false); |
281 | /// Emit atomic update as libcalls. |
282 | void |
283 | EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, |
284 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
285 | bool IsVolatile); |
286 | /// Emit atomic update as LLVM instructions. |
287 | void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, |
288 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
289 | bool IsVolatile); |
290 | /// Emit atomic update as libcalls. |
291 | void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, |
292 | bool IsVolatile); |
293 | /// Emit atomic update as LLVM instructions. |
294 | void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, |
295 | bool IsVolatile); |
296 | }; |
297 | } |
298 | |
299 | Address AtomicInfo::CreateTempAlloca() const { |
300 | Address TempAlloca = CGF.CreateMemTemp( |
301 | (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy |
302 | : AtomicTy, |
303 | getAtomicAlignment(), |
304 | "atomic-temp" ); |
305 | // Cast to pointer to value type for bitfields. |
306 | if (LVal.isBitField()) |
307 | return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( |
308 | TempAlloca, getAtomicAddress().getType(), |
309 | getAtomicAddress().getElementType()); |
310 | return TempAlloca; |
311 | } |
312 | |
313 | static RValue emitAtomicLibcall(CodeGenFunction &CGF, |
314 | StringRef fnName, |
315 | QualType resultType, |
316 | CallArgList &args) { |
317 | const CGFunctionInfo &fnInfo = |
318 | CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); |
319 | llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); |
320 | llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); |
321 | fnAttrB.addAttribute(llvm::Attribute::NoUnwind); |
322 | fnAttrB.addAttribute(llvm::Attribute::WillReturn); |
323 | llvm::AttributeList fnAttrs = llvm::AttributeList::get( |
324 | CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); |
325 | |
326 | llvm::FunctionCallee fn = |
327 | CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); |
328 | auto callee = CGCallee::forDirect(fn); |
329 | return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); |
330 | } |
331 | |
332 | /// Does a store of the given IR type modify the full expected width? |
333 | static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, |
334 | uint64_t expectedSize) { |
335 | return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); |
336 | } |
337 | |
338 | /// Does the atomic type require memsetting to zero before initialization? |
339 | /// |
340 | /// The IR type is provided as a way of making certain queries faster. |
341 | bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { |
342 | // If the atomic type has size padding, we definitely need a memset. |
343 | if (hasPadding()) return true; |
344 | |
345 | // Otherwise, do some simple heuristics to try to avoid it: |
346 | switch (getEvaluationKind()) { |
347 | // For scalars and complexes, check whether the store size of the |
348 | // type uses the full size. |
349 | case TEK_Scalar: |
350 | return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); |
351 | case TEK_Complex: |
352 | return !isFullSizeType(CGF.CGM, type->getStructElementType(0), |
353 | AtomicSizeInBits / 2); |
354 | |
355 | // Padding in structs has an undefined bit pattern. User beware. |
356 | case TEK_Aggregate: |
357 | return false; |
358 | } |
359 | llvm_unreachable("bad evaluation kind" ); |
360 | } |
361 | |
362 | bool AtomicInfo::emitMemSetZeroIfNecessary() const { |
363 | assert(LVal.isSimple()); |
364 | Address addr = LVal.getAddress(CGF); |
365 | if (!requiresMemSetZero(addr.getElementType())) |
366 | return false; |
367 | |
368 | CGF.Builder.CreateMemSet( |
369 | addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), |
370 | CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), |
371 | LVal.getAlignment().getAsAlign()); |
372 | return true; |
373 | } |
374 | |
375 | static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, |
376 | Address Dest, Address Ptr, |
377 | Address Val1, Address Val2, |
378 | uint64_t Size, |
379 | llvm::AtomicOrdering SuccessOrder, |
380 | llvm::AtomicOrdering FailureOrder, |
381 | llvm::SyncScope::ID Scope) { |
382 | // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. |
383 | llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); |
384 | llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); |
385 | |
386 | llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( |
387 | Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, |
388 | Scope); |
389 | Pair->setVolatile(E->isVolatile()); |
390 | Pair->setWeak(IsWeak); |
391 | |
392 | // Cmp holds the result of the compare-exchange operation: true on success, |
393 | // false on failure. |
394 | llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); |
395 | llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); |
396 | |
397 | // This basic block is used to hold the store instruction if the operation |
398 | // failed. |
399 | llvm::BasicBlock *StoreExpectedBB = |
400 | CGF.createBasicBlock("cmpxchg.store_expected" , CGF.CurFn); |
401 | |
402 | // This basic block is the exit point of the operation, we should end up |
403 | // here regardless of whether or not the operation succeeded. |
404 | llvm::BasicBlock *ContinueBB = |
405 | CGF.createBasicBlock("cmpxchg.continue" , CGF.CurFn); |
406 | |
407 | // Update Expected if Expected isn't equal to Old, otherwise branch to the |
408 | // exit point. |
409 | CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); |
410 | |
411 | CGF.Builder.SetInsertPoint(StoreExpectedBB); |
412 | // Update the memory at Expected with Old's value. |
413 | CGF.Builder.CreateStore(Old, Val1); |
414 | // Finally, branch to the exit point. |
415 | CGF.Builder.CreateBr(ContinueBB); |
416 | |
417 | CGF.Builder.SetInsertPoint(ContinueBB); |
418 | // Update the memory at Dest with Cmp's value. |
419 | CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); |
420 | } |
421 | |
422 | /// Given an ordering required on success, emit all possible cmpxchg |
423 | /// instructions to cope with the provided (but possibly only dynamically known) |
424 | /// FailureOrder. |
425 | static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, |
426 | bool IsWeak, Address Dest, Address Ptr, |
427 | Address Val1, Address Val2, |
428 | llvm::Value *FailureOrderVal, |
429 | uint64_t Size, |
430 | llvm::AtomicOrdering SuccessOrder, |
431 | llvm::SyncScope::ID Scope) { |
432 | llvm::AtomicOrdering FailureOrder; |
433 | if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { |
434 | auto FOS = FO->getSExtValue(); |
435 | if (!llvm::isValidAtomicOrderingCABI(FOS)) |
436 | FailureOrder = llvm::AtomicOrdering::Monotonic; |
437 | else |
438 | switch ((llvm::AtomicOrderingCABI)FOS) { |
439 | case llvm::AtomicOrderingCABI::relaxed: |
440 | // 31.7.2.18: "The failure argument shall not be memory_order_release |
441 | // nor memory_order_acq_rel". Fallback to monotonic. |
442 | case llvm::AtomicOrderingCABI::release: |
443 | case llvm::AtomicOrderingCABI::acq_rel: |
444 | FailureOrder = llvm::AtomicOrdering::Monotonic; |
445 | break; |
446 | case llvm::AtomicOrderingCABI::consume: |
447 | case llvm::AtomicOrderingCABI::acquire: |
448 | FailureOrder = llvm::AtomicOrdering::Acquire; |
449 | break; |
450 | case llvm::AtomicOrderingCABI::seq_cst: |
451 | FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; |
452 | break; |
453 | } |
454 | // Prior to c++17, "the failure argument shall be no stronger than the |
455 | // success argument". This condition has been lifted and the only |
456 | // precondition is 31.7.2.18. Effectively treat this as a DR and skip |
457 | // language version checks. |
458 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
459 | FailureOrder, Scope); |
460 | return; |
461 | } |
462 | |
463 | // Create all the relevant BB's |
464 | auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail" , CGF.CurFn); |
465 | auto *AcquireBB = CGF.createBasicBlock("acquire_fail" , CGF.CurFn); |
466 | auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail" , CGF.CurFn); |
467 | auto *ContBB = CGF.createBasicBlock("atomic.continue" , CGF.CurFn); |
468 | |
469 | // MonotonicBB is arbitrarily chosen as the default case; in practice, this |
470 | // doesn't matter unless someone is crazy enough to use something that |
471 | // doesn't fold to a constant for the ordering. |
472 | llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); |
473 | // Implemented as acquire, since it's the closest in LLVM. |
474 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), |
475 | AcquireBB); |
476 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), |
477 | AcquireBB); |
478 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), |
479 | SeqCstBB); |
480 | |
481 | // Emit all the different atomics |
482 | CGF.Builder.SetInsertPoint(MonotonicBB); |
483 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, |
484 | Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); |
485 | CGF.Builder.CreateBr(ContBB); |
486 | |
487 | CGF.Builder.SetInsertPoint(AcquireBB); |
488 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
489 | llvm::AtomicOrdering::Acquire, Scope); |
490 | CGF.Builder.CreateBr(ContBB); |
491 | |
492 | CGF.Builder.SetInsertPoint(SeqCstBB); |
493 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
494 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
495 | CGF.Builder.CreateBr(ContBB); |
496 | |
497 | CGF.Builder.SetInsertPoint(ContBB); |
498 | } |
499 | |
500 | /// Duplicate the atomic min/max operation in conventional IR for the builtin |
501 | /// variants that return the new rather than the original value. |
502 | static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, |
503 | AtomicExpr::AtomicOp Op, |
504 | bool IsSigned, |
505 | llvm::Value *OldVal, |
506 | llvm::Value *RHS) { |
507 | llvm::CmpInst::Predicate Pred; |
508 | switch (Op) { |
509 | default: |
510 | llvm_unreachable("Unexpected min/max operation" ); |
511 | case AtomicExpr::AO__atomic_max_fetch: |
512 | Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; |
513 | break; |
514 | case AtomicExpr::AO__atomic_min_fetch: |
515 | Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; |
516 | break; |
517 | } |
518 | llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst" ); |
519 | return Builder.CreateSelect(Cmp, OldVal, RHS, "newval" ); |
520 | } |
521 | |
522 | static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, |
523 | Address Ptr, Address Val1, Address Val2, |
524 | llvm::Value *IsWeak, llvm::Value *FailureOrder, |
525 | uint64_t Size, llvm::AtomicOrdering Order, |
526 | llvm::SyncScope::ID Scope) { |
527 | llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; |
528 | bool PostOpMinMax = false; |
529 | unsigned PostOp = 0; |
530 | |
531 | switch (E->getOp()) { |
532 | case AtomicExpr::AO__c11_atomic_init: |
533 | case AtomicExpr::AO__opencl_atomic_init: |
534 | llvm_unreachable("Already handled!" ); |
535 | |
536 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
537 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
538 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
539 | emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, |
540 | FailureOrder, Size, Order, Scope); |
541 | return; |
542 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
543 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
544 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
545 | emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, |
546 | FailureOrder, Size, Order, Scope); |
547 | return; |
548 | case AtomicExpr::AO__atomic_compare_exchange: |
549 | case AtomicExpr::AO__atomic_compare_exchange_n: { |
550 | if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { |
551 | emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, |
552 | Val1, Val2, FailureOrder, Size, Order, Scope); |
553 | } else { |
554 | // Create all the relevant BB's |
555 | llvm::BasicBlock *StrongBB = |
556 | CGF.createBasicBlock("cmpxchg.strong" , CGF.CurFn); |
557 | llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak" , CGF.CurFn); |
558 | llvm::BasicBlock *ContBB = |
559 | CGF.createBasicBlock("cmpxchg.continue" , CGF.CurFn); |
560 | |
561 | llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); |
562 | SI->addCase(CGF.Builder.getInt1(false), StrongBB); |
563 | |
564 | CGF.Builder.SetInsertPoint(StrongBB); |
565 | emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, |
566 | FailureOrder, Size, Order, Scope); |
567 | CGF.Builder.CreateBr(ContBB); |
568 | |
569 | CGF.Builder.SetInsertPoint(WeakBB); |
570 | emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, |
571 | FailureOrder, Size, Order, Scope); |
572 | CGF.Builder.CreateBr(ContBB); |
573 | |
574 | CGF.Builder.SetInsertPoint(ContBB); |
575 | } |
576 | return; |
577 | } |
578 | case AtomicExpr::AO__c11_atomic_load: |
579 | case AtomicExpr::AO__opencl_atomic_load: |
580 | case AtomicExpr::AO__hip_atomic_load: |
581 | case AtomicExpr::AO__atomic_load_n: |
582 | case AtomicExpr::AO__atomic_load: { |
583 | llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); |
584 | Load->setAtomic(Order, Scope); |
585 | Load->setVolatile(E->isVolatile()); |
586 | CGF.Builder.CreateStore(Load, Dest); |
587 | return; |
588 | } |
589 | |
590 | case AtomicExpr::AO__c11_atomic_store: |
591 | case AtomicExpr::AO__opencl_atomic_store: |
592 | case AtomicExpr::AO__hip_atomic_store: |
593 | case AtomicExpr::AO__atomic_store: |
594 | case AtomicExpr::AO__atomic_store_n: { |
595 | llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); |
596 | llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); |
597 | Store->setAtomic(Order, Scope); |
598 | Store->setVolatile(E->isVolatile()); |
599 | return; |
600 | } |
601 | |
602 | case AtomicExpr::AO__c11_atomic_exchange: |
603 | case AtomicExpr::AO__hip_atomic_exchange: |
604 | case AtomicExpr::AO__opencl_atomic_exchange: |
605 | case AtomicExpr::AO__atomic_exchange_n: |
606 | case AtomicExpr::AO__atomic_exchange: |
607 | Op = llvm::AtomicRMWInst::Xchg; |
608 | break; |
609 | |
610 | case AtomicExpr::AO__atomic_add_fetch: |
611 | PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd |
612 | : llvm::Instruction::Add; |
613 | [[fallthrough]]; |
614 | case AtomicExpr::AO__c11_atomic_fetch_add: |
615 | case AtomicExpr::AO__hip_atomic_fetch_add: |
616 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
617 | case AtomicExpr::AO__atomic_fetch_add: |
618 | Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd |
619 | : llvm::AtomicRMWInst::Add; |
620 | break; |
621 | |
622 | case AtomicExpr::AO__atomic_sub_fetch: |
623 | PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub |
624 | : llvm::Instruction::Sub; |
625 | [[fallthrough]]; |
626 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
627 | case AtomicExpr::AO__hip_atomic_fetch_sub: |
628 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
629 | case AtomicExpr::AO__atomic_fetch_sub: |
630 | Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub |
631 | : llvm::AtomicRMWInst::Sub; |
632 | break; |
633 | |
634 | case AtomicExpr::AO__atomic_min_fetch: |
635 | PostOpMinMax = true; |
636 | [[fallthrough]]; |
637 | case AtomicExpr::AO__c11_atomic_fetch_min: |
638 | case AtomicExpr::AO__hip_atomic_fetch_min: |
639 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
640 | case AtomicExpr::AO__atomic_fetch_min: |
641 | Op = E->getValueType()->isFloatingType() |
642 | ? llvm::AtomicRMWInst::FMin |
643 | : (E->getValueType()->isSignedIntegerType() |
644 | ? llvm::AtomicRMWInst::Min |
645 | : llvm::AtomicRMWInst::UMin); |
646 | break; |
647 | |
648 | case AtomicExpr::AO__atomic_max_fetch: |
649 | PostOpMinMax = true; |
650 | [[fallthrough]]; |
651 | case AtomicExpr::AO__c11_atomic_fetch_max: |
652 | case AtomicExpr::AO__hip_atomic_fetch_max: |
653 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
654 | case AtomicExpr::AO__atomic_fetch_max: |
655 | Op = E->getValueType()->isFloatingType() |
656 | ? llvm::AtomicRMWInst::FMax |
657 | : (E->getValueType()->isSignedIntegerType() |
658 | ? llvm::AtomicRMWInst::Max |
659 | : llvm::AtomicRMWInst::UMax); |
660 | break; |
661 | |
662 | case AtomicExpr::AO__atomic_and_fetch: |
663 | PostOp = llvm::Instruction::And; |
664 | [[fallthrough]]; |
665 | case AtomicExpr::AO__c11_atomic_fetch_and: |
666 | case AtomicExpr::AO__hip_atomic_fetch_and: |
667 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
668 | case AtomicExpr::AO__atomic_fetch_and: |
669 | Op = llvm::AtomicRMWInst::And; |
670 | break; |
671 | |
672 | case AtomicExpr::AO__atomic_or_fetch: |
673 | PostOp = llvm::Instruction::Or; |
674 | [[fallthrough]]; |
675 | case AtomicExpr::AO__c11_atomic_fetch_or: |
676 | case AtomicExpr::AO__hip_atomic_fetch_or: |
677 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
678 | case AtomicExpr::AO__atomic_fetch_or: |
679 | Op = llvm::AtomicRMWInst::Or; |
680 | break; |
681 | |
682 | case AtomicExpr::AO__atomic_xor_fetch: |
683 | PostOp = llvm::Instruction::Xor; |
684 | [[fallthrough]]; |
685 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
686 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
687 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
688 | case AtomicExpr::AO__atomic_fetch_xor: |
689 | Op = llvm::AtomicRMWInst::Xor; |
690 | break; |
691 | |
692 | case AtomicExpr::AO__atomic_nand_fetch: |
693 | PostOp = llvm::Instruction::And; // the NOT is special cased below |
694 | [[fallthrough]]; |
695 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
696 | case AtomicExpr::AO__atomic_fetch_nand: |
697 | Op = llvm::AtomicRMWInst::Nand; |
698 | break; |
699 | } |
700 | |
701 | llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); |
702 | llvm::AtomicRMWInst *RMWI = |
703 | CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); |
704 | RMWI->setVolatile(E->isVolatile()); |
705 | |
706 | // For __atomic_*_fetch operations, perform the operation again to |
707 | // determine the value which was written. |
708 | llvm::Value *Result = RMWI; |
709 | if (PostOpMinMax) |
710 | Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), |
711 | E->getValueType()->isSignedIntegerType(), |
712 | RMWI, LoadVal1); |
713 | else if (PostOp) |
714 | Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, |
715 | LoadVal1); |
716 | if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) |
717 | Result = CGF.Builder.CreateNot(Result); |
718 | CGF.Builder.CreateStore(Result, Dest); |
719 | } |
720 | |
721 | // This function emits any expression (scalar, complex, or aggregate) |
722 | // into a temporary alloca. |
723 | static Address |
724 | EmitValToTemp(CodeGenFunction &CGF, Expr *E) { |
725 | Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp" ); |
726 | CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), |
727 | /*Init*/ true); |
728 | return DeclPtr; |
729 | } |
730 | |
731 | static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, |
732 | Address Ptr, Address Val1, Address Val2, |
733 | llvm::Value *IsWeak, llvm::Value *FailureOrder, |
734 | uint64_t Size, llvm::AtomicOrdering Order, |
735 | llvm::Value *Scope) { |
736 | auto ScopeModel = Expr->getScopeModel(); |
737 | |
738 | // LLVM atomic instructions always have synch scope. If clang atomic |
739 | // expression has no scope operand, use default LLVM synch scope. |
740 | if (!ScopeModel) { |
741 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
742 | Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("" )); |
743 | return; |
744 | } |
745 | |
746 | // Handle constant scope. |
747 | if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { |
748 | auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( |
749 | CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), |
750 | Order, CGF.CGM.getLLVMContext()); |
751 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
752 | Order, SCID); |
753 | return; |
754 | } |
755 | |
756 | // Handle non-constant scope. |
757 | auto &Builder = CGF.Builder; |
758 | auto Scopes = ScopeModel->getRuntimeValues(); |
759 | llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; |
760 | for (auto S : Scopes) |
761 | BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); |
762 | |
763 | llvm::BasicBlock *ContBB = |
764 | CGF.createBasicBlock("atomic.scope.continue" , CGF.CurFn); |
765 | |
766 | auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); |
767 | // If unsupported synch scope is encountered at run time, assume a fallback |
768 | // synch scope value. |
769 | auto FallBack = ScopeModel->getFallBackValue(); |
770 | llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); |
771 | for (auto S : Scopes) { |
772 | auto *B = BB[S]; |
773 | if (S != FallBack) |
774 | SI->addCase(Builder.getInt32(S), B); |
775 | |
776 | Builder.SetInsertPoint(B); |
777 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
778 | Order, |
779 | CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), |
780 | ScopeModel->map(S), |
781 | Order, |
782 | CGF.getLLVMContext())); |
783 | Builder.CreateBr(ContBB); |
784 | } |
785 | |
786 | Builder.SetInsertPoint(ContBB); |
787 | } |
788 | |
789 | static void |
790 | AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, |
791 | bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, |
792 | SourceLocation Loc, CharUnits SizeInChars) { |
793 | if (UseOptimizedLibcall) { |
794 | // Load value and pass it to the function directly. |
795 | CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); |
796 | int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); |
797 | ValTy = |
798 | CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); |
799 | llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits); |
800 | Address Ptr = Address(Val, ITy, Align); |
801 | Val = CGF.EmitLoadOfScalar(Ptr, false, |
802 | CGF.getContext().getPointerType(ValTy), |
803 | Loc); |
804 | // Coerce the value into an appropriately sized integer type. |
805 | Args.add(RValue::get(Val), ValTy); |
806 | } else { |
807 | // Non-optimized functions always take a reference. |
808 | Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy); |
809 | } |
810 | } |
811 | |
812 | RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { |
813 | QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); |
814 | QualType MemTy = AtomicTy; |
815 | if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) |
816 | MemTy = AT->getValueType(); |
817 | llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; |
818 | |
819 | Address Val1 = Address::invalid(); |
820 | Address Val2 = Address::invalid(); |
821 | Address Dest = Address::invalid(); |
822 | Address Ptr = EmitPointerWithAlignment(E->getPtr()); |
823 | |
824 | if (E->getOp() == AtomicExpr::AO__c11_atomic_init || |
825 | E->getOp() == AtomicExpr::AO__opencl_atomic_init) { |
826 | LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); |
827 | EmitAtomicInit(E->getVal1(), lvalue); |
828 | return RValue::get(nullptr); |
829 | } |
830 | |
831 | auto TInfo = getContext().getTypeInfoInChars(AtomicTy); |
832 | uint64_t Size = TInfo.Width.getQuantity(); |
833 | unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); |
834 | |
835 | bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; |
836 | bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; |
837 | bool UseLibcall = Misaligned | Oversized; |
838 | bool ShouldCastToIntPtrTy = true; |
839 | |
840 | CharUnits MaxInlineWidth = |
841 | getContext().toCharUnitsFromBits(MaxInlineWidthInBits); |
842 | |
843 | DiagnosticsEngine &Diags = CGM.getDiags(); |
844 | |
845 | if (Misaligned) { |
846 | Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) |
847 | << (int)TInfo.Width.getQuantity() |
848 | << (int)Ptr.getAlignment().getQuantity(); |
849 | } |
850 | |
851 | if (Oversized) { |
852 | Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) |
853 | << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); |
854 | } |
855 | |
856 | llvm::Value *Order = EmitScalarExpr(E->getOrder()); |
857 | llvm::Value *Scope = |
858 | E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; |
859 | |
860 | switch (E->getOp()) { |
861 | case AtomicExpr::AO__c11_atomic_init: |
862 | case AtomicExpr::AO__opencl_atomic_init: |
863 | llvm_unreachable("Already handled above with EmitAtomicInit!" ); |
864 | |
865 | case AtomicExpr::AO__c11_atomic_load: |
866 | case AtomicExpr::AO__opencl_atomic_load: |
867 | case AtomicExpr::AO__hip_atomic_load: |
868 | case AtomicExpr::AO__atomic_load_n: |
869 | break; |
870 | |
871 | case AtomicExpr::AO__atomic_load: |
872 | Dest = EmitPointerWithAlignment(E->getVal1()); |
873 | break; |
874 | |
875 | case AtomicExpr::AO__atomic_store: |
876 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
877 | break; |
878 | |
879 | case AtomicExpr::AO__atomic_exchange: |
880 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
881 | Dest = EmitPointerWithAlignment(E->getVal2()); |
882 | break; |
883 | |
884 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
885 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
886 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
887 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
888 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
889 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
890 | case AtomicExpr::AO__atomic_compare_exchange_n: |
891 | case AtomicExpr::AO__atomic_compare_exchange: |
892 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
893 | if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) |
894 | Val2 = EmitPointerWithAlignment(E->getVal2()); |
895 | else |
896 | Val2 = EmitValToTemp(*this, E->getVal2()); |
897 | OrderFail = EmitScalarExpr(E->getOrderFail()); |
898 | if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || |
899 | E->getOp() == AtomicExpr::AO__atomic_compare_exchange) |
900 | IsWeak = EmitScalarExpr(E->getWeak()); |
901 | break; |
902 | |
903 | case AtomicExpr::AO__c11_atomic_fetch_add: |
904 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
905 | case AtomicExpr::AO__hip_atomic_fetch_add: |
906 | case AtomicExpr::AO__hip_atomic_fetch_sub: |
907 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
908 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
909 | if (MemTy->isPointerType()) { |
910 | // For pointer arithmetic, we're required to do a bit of math: |
911 | // adding 1 to an int* is not the same as adding 1 to a uintptr_t. |
912 | // ... but only for the C11 builtins. The GNU builtins expect the |
913 | // user to multiply by sizeof(T). |
914 | QualType Val1Ty = E->getVal1()->getType(); |
915 | llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); |
916 | CharUnits PointeeIncAmt = |
917 | getContext().getTypeSizeInChars(MemTy->getPointeeType()); |
918 | Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); |
919 | auto Temp = CreateMemTemp(Val1Ty, ".atomictmp" ); |
920 | Val1 = Temp; |
921 | EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); |
922 | break; |
923 | } |
924 | [[fallthrough]]; |
925 | case AtomicExpr::AO__atomic_fetch_add: |
926 | case AtomicExpr::AO__atomic_fetch_max: |
927 | case AtomicExpr::AO__atomic_fetch_min: |
928 | case AtomicExpr::AO__atomic_fetch_sub: |
929 | case AtomicExpr::AO__atomic_add_fetch: |
930 | case AtomicExpr::AO__atomic_max_fetch: |
931 | case AtomicExpr::AO__atomic_min_fetch: |
932 | case AtomicExpr::AO__atomic_sub_fetch: |
933 | case AtomicExpr::AO__c11_atomic_fetch_max: |
934 | case AtomicExpr::AO__c11_atomic_fetch_min: |
935 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
936 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
937 | case AtomicExpr::AO__hip_atomic_fetch_max: |
938 | case AtomicExpr::AO__hip_atomic_fetch_min: |
939 | ShouldCastToIntPtrTy = !MemTy->isFloatingType(); |
940 | [[fallthrough]]; |
941 | |
942 | case AtomicExpr::AO__c11_atomic_store: |
943 | case AtomicExpr::AO__c11_atomic_exchange: |
944 | case AtomicExpr::AO__opencl_atomic_store: |
945 | case AtomicExpr::AO__hip_atomic_store: |
946 | case AtomicExpr::AO__opencl_atomic_exchange: |
947 | case AtomicExpr::AO__hip_atomic_exchange: |
948 | case AtomicExpr::AO__atomic_store_n: |
949 | case AtomicExpr::AO__atomic_exchange_n: |
950 | case AtomicExpr::AO__c11_atomic_fetch_and: |
951 | case AtomicExpr::AO__c11_atomic_fetch_or: |
952 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
953 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
954 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
955 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
956 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
957 | case AtomicExpr::AO__atomic_fetch_and: |
958 | case AtomicExpr::AO__hip_atomic_fetch_and: |
959 | case AtomicExpr::AO__atomic_fetch_or: |
960 | case AtomicExpr::AO__hip_atomic_fetch_or: |
961 | case AtomicExpr::AO__atomic_fetch_xor: |
962 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
963 | case AtomicExpr::AO__atomic_fetch_nand: |
964 | case AtomicExpr::AO__atomic_and_fetch: |
965 | case AtomicExpr::AO__atomic_or_fetch: |
966 | case AtomicExpr::AO__atomic_xor_fetch: |
967 | case AtomicExpr::AO__atomic_nand_fetch: |
968 | Val1 = EmitValToTemp(*this, E->getVal1()); |
969 | break; |
970 | } |
971 | |
972 | QualType RValTy = E->getType().getUnqualifiedType(); |
973 | |
974 | // The inlined atomics only function on iN types, where N is a power of 2. We |
975 | // need to make sure (via temporaries if necessary) that all incoming values |
976 | // are compatible. |
977 | LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); |
978 | AtomicInfo Atomics(*this, AtomicVal); |
979 | |
980 | if (ShouldCastToIntPtrTy) { |
981 | Ptr = Atomics.castToAtomicIntPointer(Ptr); |
982 | if (Val1.isValid()) |
983 | Val1 = Atomics.convertToAtomicIntPointer(Val1); |
984 | if (Val2.isValid()) |
985 | Val2 = Atomics.convertToAtomicIntPointer(Val2); |
986 | } |
987 | if (Dest.isValid()) { |
988 | if (ShouldCastToIntPtrTy) |
989 | Dest = Atomics.castToAtomicIntPointer(Dest); |
990 | } else if (E->isCmpXChg()) |
991 | Dest = CreateMemTemp(RValTy, "cmpxchg.bool" ); |
992 | else if (!RValTy->isVoidType()) { |
993 | Dest = Atomics.CreateTempAlloca(); |
994 | if (ShouldCastToIntPtrTy) |
995 | Dest = Atomics.castToAtomicIntPointer(Dest); |
996 | } |
997 | |
998 | // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . |
999 | if (UseLibcall) { |
1000 | bool UseOptimizedLibcall = false; |
1001 | switch (E->getOp()) { |
1002 | case AtomicExpr::AO__c11_atomic_init: |
1003 | case AtomicExpr::AO__opencl_atomic_init: |
1004 | llvm_unreachable("Already handled above with EmitAtomicInit!" ); |
1005 | |
1006 | case AtomicExpr::AO__c11_atomic_fetch_add: |
1007 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
1008 | case AtomicExpr::AO__atomic_fetch_add: |
1009 | case AtomicExpr::AO__hip_atomic_fetch_add: |
1010 | case AtomicExpr::AO__c11_atomic_fetch_and: |
1011 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
1012 | case AtomicExpr::AO__hip_atomic_fetch_and: |
1013 | case AtomicExpr::AO__atomic_fetch_and: |
1014 | case AtomicExpr::AO__c11_atomic_fetch_or: |
1015 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
1016 | case AtomicExpr::AO__hip_atomic_fetch_or: |
1017 | case AtomicExpr::AO__atomic_fetch_or: |
1018 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
1019 | case AtomicExpr::AO__atomic_fetch_nand: |
1020 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
1021 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
1022 | case AtomicExpr::AO__atomic_fetch_sub: |
1023 | case AtomicExpr::AO__hip_atomic_fetch_sub: |
1024 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
1025 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
1026 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
1027 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
1028 | case AtomicExpr::AO__atomic_fetch_xor: |
1029 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
1030 | case AtomicExpr::AO__c11_atomic_fetch_max: |
1031 | case AtomicExpr::AO__c11_atomic_fetch_min: |
1032 | case AtomicExpr::AO__atomic_add_fetch: |
1033 | case AtomicExpr::AO__atomic_and_fetch: |
1034 | case AtomicExpr::AO__atomic_nand_fetch: |
1035 | case AtomicExpr::AO__atomic_or_fetch: |
1036 | case AtomicExpr::AO__atomic_sub_fetch: |
1037 | case AtomicExpr::AO__atomic_xor_fetch: |
1038 | case AtomicExpr::AO__atomic_fetch_max: |
1039 | case AtomicExpr::AO__hip_atomic_fetch_max: |
1040 | case AtomicExpr::AO__atomic_fetch_min: |
1041 | case AtomicExpr::AO__hip_atomic_fetch_min: |
1042 | case AtomicExpr::AO__atomic_max_fetch: |
1043 | case AtomicExpr::AO__atomic_min_fetch: |
1044 | // For these, only library calls for certain sizes exist. |
1045 | UseOptimizedLibcall = true; |
1046 | break; |
1047 | |
1048 | case AtomicExpr::AO__atomic_load: |
1049 | case AtomicExpr::AO__atomic_store: |
1050 | case AtomicExpr::AO__atomic_exchange: |
1051 | case AtomicExpr::AO__atomic_compare_exchange: |
1052 | // Use the generic version if we don't know that the operand will be |
1053 | // suitably aligned for the optimized version. |
1054 | if (Misaligned) |
1055 | break; |
1056 | [[fallthrough]]; |
1057 | case AtomicExpr::AO__c11_atomic_load: |
1058 | case AtomicExpr::AO__c11_atomic_store: |
1059 | case AtomicExpr::AO__c11_atomic_exchange: |
1060 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
1061 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
1062 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
1063 | case AtomicExpr::AO__opencl_atomic_load: |
1064 | case AtomicExpr::AO__hip_atomic_load: |
1065 | case AtomicExpr::AO__opencl_atomic_store: |
1066 | case AtomicExpr::AO__hip_atomic_store: |
1067 | case AtomicExpr::AO__opencl_atomic_exchange: |
1068 | case AtomicExpr::AO__hip_atomic_exchange: |
1069 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
1070 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
1071 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
1072 | case AtomicExpr::AO__atomic_load_n: |
1073 | case AtomicExpr::AO__atomic_store_n: |
1074 | case AtomicExpr::AO__atomic_exchange_n: |
1075 | case AtomicExpr::AO__atomic_compare_exchange_n: |
1076 | // Only use optimized library calls for sizes for which they exist. |
1077 | // FIXME: Size == 16 optimized library functions exist too. |
1078 | if (Size == 1 || Size == 2 || Size == 4 || Size == 8) |
1079 | UseOptimizedLibcall = true; |
1080 | break; |
1081 | } |
1082 | |
1083 | CallArgList Args; |
1084 | if (!UseOptimizedLibcall) { |
1085 | // For non-optimized library calls, the size is the first parameter |
1086 | Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), |
1087 | getContext().getSizeType()); |
1088 | } |
1089 | // Atomic address is the first or second parameter |
1090 | // The OpenCL atomic library functions only accept pointer arguments to |
1091 | // generic address space. |
1092 | auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { |
1093 | if (!E->isOpenCL()) |
1094 | return V; |
1095 | auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); |
1096 | if (AS == LangAS::opencl_generic) |
1097 | return V; |
1098 | auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); |
1099 | auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS); |
1100 | |
1101 | return getTargetHooks().performAddrSpaceCast( |
1102 | *this, V, AS, LangAS::opencl_generic, DestType, false); |
1103 | }; |
1104 | |
1105 | Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), |
1106 | E->getPtr()->getType())), |
1107 | getContext().VoidPtrTy); |
1108 | |
1109 | std::string LibCallName; |
1110 | QualType LoweredMemTy = |
1111 | MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; |
1112 | QualType RetTy; |
1113 | bool HaveRetTy = false; |
1114 | llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; |
1115 | bool PostOpMinMax = false; |
1116 | switch (E->getOp()) { |
1117 | case AtomicExpr::AO__c11_atomic_init: |
1118 | case AtomicExpr::AO__opencl_atomic_init: |
1119 | llvm_unreachable("Already handled!" ); |
1120 | |
1121 | // There is only one libcall for compare an exchange, because there is no |
1122 | // optimisation benefit possible from a libcall version of a weak compare |
1123 | // and exchange. |
1124 | // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, |
1125 | // void *desired, int success, int failure) |
1126 | // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, |
1127 | // int success, int failure) |
1128 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
1129 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
1130 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
1131 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
1132 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
1133 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
1134 | case AtomicExpr::AO__atomic_compare_exchange: |
1135 | case AtomicExpr::AO__atomic_compare_exchange_n: |
1136 | LibCallName = "__atomic_compare_exchange" ; |
1137 | RetTy = getContext().BoolTy; |
1138 | HaveRetTy = true; |
1139 | Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), |
1140 | E->getVal1()->getType())), |
1141 | getContext().VoidPtrTy); |
1142 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), |
1143 | MemTy, E->getExprLoc(), TInfo.Width); |
1144 | Args.add(RValue::get(Order), getContext().IntTy); |
1145 | Order = OrderFail; |
1146 | break; |
1147 | // void __atomic_exchange(size_t size, void *mem, void *val, void *return, |
1148 | // int order) |
1149 | // T __atomic_exchange_N(T *mem, T val, int order) |
1150 | case AtomicExpr::AO__c11_atomic_exchange: |
1151 | case AtomicExpr::AO__opencl_atomic_exchange: |
1152 | case AtomicExpr::AO__atomic_exchange_n: |
1153 | case AtomicExpr::AO__atomic_exchange: |
1154 | case AtomicExpr::AO__hip_atomic_exchange: |
1155 | LibCallName = "__atomic_exchange" ; |
1156 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1157 | MemTy, E->getExprLoc(), TInfo.Width); |
1158 | break; |
1159 | // void __atomic_store(size_t size, void *mem, void *val, int order) |
1160 | // void __atomic_store_N(T *mem, T val, int order) |
1161 | case AtomicExpr::AO__c11_atomic_store: |
1162 | case AtomicExpr::AO__opencl_atomic_store: |
1163 | case AtomicExpr::AO__hip_atomic_store: |
1164 | case AtomicExpr::AO__atomic_store: |
1165 | case AtomicExpr::AO__atomic_store_n: |
1166 | LibCallName = "__atomic_store" ; |
1167 | RetTy = getContext().VoidTy; |
1168 | HaveRetTy = true; |
1169 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1170 | MemTy, E->getExprLoc(), TInfo.Width); |
1171 | break; |
1172 | // void __atomic_load(size_t size, void *mem, void *return, int order) |
1173 | // T __atomic_load_N(T *mem, int order) |
1174 | case AtomicExpr::AO__c11_atomic_load: |
1175 | case AtomicExpr::AO__opencl_atomic_load: |
1176 | case AtomicExpr::AO__hip_atomic_load: |
1177 | case AtomicExpr::AO__atomic_load: |
1178 | case AtomicExpr::AO__atomic_load_n: |
1179 | LibCallName = "__atomic_load" ; |
1180 | break; |
1181 | // T __atomic_add_fetch_N(T *mem, T val, int order) |
1182 | // T __atomic_fetch_add_N(T *mem, T val, int order) |
1183 | case AtomicExpr::AO__atomic_add_fetch: |
1184 | PostOp = llvm::Instruction::Add; |
1185 | [[fallthrough]]; |
1186 | case AtomicExpr::AO__c11_atomic_fetch_add: |
1187 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
1188 | case AtomicExpr::AO__atomic_fetch_add: |
1189 | case AtomicExpr::AO__hip_atomic_fetch_add: |
1190 | LibCallName = "__atomic_fetch_add" ; |
1191 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1192 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1193 | break; |
1194 | // T __atomic_and_fetch_N(T *mem, T val, int order) |
1195 | // T __atomic_fetch_and_N(T *mem, T val, int order) |
1196 | case AtomicExpr::AO__atomic_and_fetch: |
1197 | PostOp = llvm::Instruction::And; |
1198 | [[fallthrough]]; |
1199 | case AtomicExpr::AO__c11_atomic_fetch_and: |
1200 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
1201 | case AtomicExpr::AO__hip_atomic_fetch_and: |
1202 | case AtomicExpr::AO__atomic_fetch_and: |
1203 | LibCallName = "__atomic_fetch_and" ; |
1204 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1205 | MemTy, E->getExprLoc(), TInfo.Width); |
1206 | break; |
1207 | // T __atomic_or_fetch_N(T *mem, T val, int order) |
1208 | // T __atomic_fetch_or_N(T *mem, T val, int order) |
1209 | case AtomicExpr::AO__atomic_or_fetch: |
1210 | PostOp = llvm::Instruction::Or; |
1211 | [[fallthrough]]; |
1212 | case AtomicExpr::AO__c11_atomic_fetch_or: |
1213 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
1214 | case AtomicExpr::AO__hip_atomic_fetch_or: |
1215 | case AtomicExpr::AO__atomic_fetch_or: |
1216 | LibCallName = "__atomic_fetch_or" ; |
1217 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1218 | MemTy, E->getExprLoc(), TInfo.Width); |
1219 | break; |
1220 | // T __atomic_sub_fetch_N(T *mem, T val, int order) |
1221 | // T __atomic_fetch_sub_N(T *mem, T val, int order) |
1222 | case AtomicExpr::AO__atomic_sub_fetch: |
1223 | PostOp = llvm::Instruction::Sub; |
1224 | [[fallthrough]]; |
1225 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
1226 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
1227 | case AtomicExpr::AO__hip_atomic_fetch_sub: |
1228 | case AtomicExpr::AO__atomic_fetch_sub: |
1229 | LibCallName = "__atomic_fetch_sub" ; |
1230 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1231 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1232 | break; |
1233 | // T __atomic_xor_fetch_N(T *mem, T val, int order) |
1234 | // T __atomic_fetch_xor_N(T *mem, T val, int order) |
1235 | case AtomicExpr::AO__atomic_xor_fetch: |
1236 | PostOp = llvm::Instruction::Xor; |
1237 | [[fallthrough]]; |
1238 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
1239 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
1240 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
1241 | case AtomicExpr::AO__atomic_fetch_xor: |
1242 | LibCallName = "__atomic_fetch_xor" ; |
1243 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1244 | MemTy, E->getExprLoc(), TInfo.Width); |
1245 | break; |
1246 | case AtomicExpr::AO__atomic_min_fetch: |
1247 | PostOpMinMax = true; |
1248 | [[fallthrough]]; |
1249 | case AtomicExpr::AO__c11_atomic_fetch_min: |
1250 | case AtomicExpr::AO__atomic_fetch_min: |
1251 | case AtomicExpr::AO__hip_atomic_fetch_min: |
1252 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
1253 | LibCallName = E->getValueType()->isSignedIntegerType() |
1254 | ? "__atomic_fetch_min" |
1255 | : "__atomic_fetch_umin" ; |
1256 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1257 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1258 | break; |
1259 | case AtomicExpr::AO__atomic_max_fetch: |
1260 | PostOpMinMax = true; |
1261 | [[fallthrough]]; |
1262 | case AtomicExpr::AO__c11_atomic_fetch_max: |
1263 | case AtomicExpr::AO__atomic_fetch_max: |
1264 | case AtomicExpr::AO__hip_atomic_fetch_max: |
1265 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
1266 | LibCallName = E->getValueType()->isSignedIntegerType() |
1267 | ? "__atomic_fetch_max" |
1268 | : "__atomic_fetch_umax" ; |
1269 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1270 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1271 | break; |
1272 | // T __atomic_nand_fetch_N(T *mem, T val, int order) |
1273 | // T __atomic_fetch_nand_N(T *mem, T val, int order) |
1274 | case AtomicExpr::AO__atomic_nand_fetch: |
1275 | PostOp = llvm::Instruction::And; // the NOT is special cased below |
1276 | [[fallthrough]]; |
1277 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
1278 | case AtomicExpr::AO__atomic_fetch_nand: |
1279 | LibCallName = "__atomic_fetch_nand" ; |
1280 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1281 | MemTy, E->getExprLoc(), TInfo.Width); |
1282 | break; |
1283 | } |
1284 | |
1285 | if (E->isOpenCL()) { |
1286 | LibCallName = std::string("__opencl" ) + |
1287 | StringRef(LibCallName).drop_front(1).str(); |
1288 | |
1289 | } |
1290 | // Optimized functions have the size in their name. |
1291 | if (UseOptimizedLibcall) |
1292 | LibCallName += "_" + llvm::utostr(Size); |
1293 | // By default, assume we return a value of the atomic type. |
1294 | if (!HaveRetTy) { |
1295 | if (UseOptimizedLibcall) { |
1296 | // Value is returned directly. |
1297 | // The function returns an appropriately sized integer type. |
1298 | RetTy = getContext().getIntTypeForBitwidth( |
1299 | getContext().toBits(TInfo.Width), /*Signed=*/false); |
1300 | } else { |
1301 | // Value is returned through parameter before the order. |
1302 | RetTy = getContext().VoidTy; |
1303 | Args.add(RValue::get(Dest.getPointer()), getContext().VoidPtrTy); |
1304 | } |
1305 | } |
1306 | // order is always the last parameter |
1307 | Args.add(RValue::get(Order), |
1308 | getContext().IntTy); |
1309 | if (E->isOpenCL()) |
1310 | Args.add(RValue::get(Scope), getContext().IntTy); |
1311 | |
1312 | // PostOp is only needed for the atomic_*_fetch operations, and |
1313 | // thus is only needed for and implemented in the |
1314 | // UseOptimizedLibcall codepath. |
1315 | assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); |
1316 | |
1317 | RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); |
1318 | // The value is returned directly from the libcall. |
1319 | if (E->isCmpXChg()) |
1320 | return Res; |
1321 | |
1322 | // The value is returned directly for optimized libcalls but the expr |
1323 | // provided an out-param. |
1324 | if (UseOptimizedLibcall && Res.getScalarVal()) { |
1325 | llvm::Value *ResVal = Res.getScalarVal(); |
1326 | if (PostOpMinMax) { |
1327 | llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); |
1328 | ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), |
1329 | E->getValueType()->isSignedIntegerType(), |
1330 | ResVal, LoadVal1); |
1331 | } else if (PostOp) { |
1332 | llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); |
1333 | ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); |
1334 | } |
1335 | if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) |
1336 | ResVal = Builder.CreateNot(ResVal); |
1337 | |
1338 | Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); |
1339 | } |
1340 | |
1341 | if (RValTy->isVoidType()) |
1342 | return RValue::get(nullptr); |
1343 | |
1344 | return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), |
1345 | RValTy, E->getExprLoc()); |
1346 | } |
1347 | |
1348 | bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || |
1349 | E->getOp() == AtomicExpr::AO__opencl_atomic_store || |
1350 | E->getOp() == AtomicExpr::AO__hip_atomic_store || |
1351 | E->getOp() == AtomicExpr::AO__atomic_store || |
1352 | E->getOp() == AtomicExpr::AO__atomic_store_n; |
1353 | bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || |
1354 | E->getOp() == AtomicExpr::AO__opencl_atomic_load || |
1355 | E->getOp() == AtomicExpr::AO__hip_atomic_load || |
1356 | E->getOp() == AtomicExpr::AO__atomic_load || |
1357 | E->getOp() == AtomicExpr::AO__atomic_load_n; |
1358 | |
1359 | if (isa<llvm::ConstantInt>(Order)) { |
1360 | auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); |
1361 | // We should not ever get to a case where the ordering isn't a valid C ABI |
1362 | // value, but it's hard to enforce that in general. |
1363 | if (llvm::isValidAtomicOrderingCABI(ord)) |
1364 | switch ((llvm::AtomicOrderingCABI)ord) { |
1365 | case llvm::AtomicOrderingCABI::relaxed: |
1366 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1367 | llvm::AtomicOrdering::Monotonic, Scope); |
1368 | break; |
1369 | case llvm::AtomicOrderingCABI::consume: |
1370 | case llvm::AtomicOrderingCABI::acquire: |
1371 | if (IsStore) |
1372 | break; // Avoid crashing on code with undefined behavior |
1373 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1374 | llvm::AtomicOrdering::Acquire, Scope); |
1375 | break; |
1376 | case llvm::AtomicOrderingCABI::release: |
1377 | if (IsLoad) |
1378 | break; // Avoid crashing on code with undefined behavior |
1379 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1380 | llvm::AtomicOrdering::Release, Scope); |
1381 | break; |
1382 | case llvm::AtomicOrderingCABI::acq_rel: |
1383 | if (IsLoad || IsStore) |
1384 | break; // Avoid crashing on code with undefined behavior |
1385 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1386 | llvm::AtomicOrdering::AcquireRelease, Scope); |
1387 | break; |
1388 | case llvm::AtomicOrderingCABI::seq_cst: |
1389 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1390 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
1391 | break; |
1392 | } |
1393 | if (RValTy->isVoidType()) |
1394 | return RValue::get(nullptr); |
1395 | |
1396 | return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), |
1397 | RValTy, E->getExprLoc()); |
1398 | } |
1399 | |
1400 | // Long case, when Order isn't obviously constant. |
1401 | |
1402 | // Create all the relevant BB's |
1403 | llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, |
1404 | *ReleaseBB = nullptr, *AcqRelBB = nullptr, |
1405 | *SeqCstBB = nullptr; |
1406 | MonotonicBB = createBasicBlock("monotonic" , CurFn); |
1407 | if (!IsStore) |
1408 | AcquireBB = createBasicBlock("acquire" , CurFn); |
1409 | if (!IsLoad) |
1410 | ReleaseBB = createBasicBlock("release" , CurFn); |
1411 | if (!IsLoad && !IsStore) |
1412 | AcqRelBB = createBasicBlock("acqrel" , CurFn); |
1413 | SeqCstBB = createBasicBlock("seqcst" , CurFn); |
1414 | llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue" , CurFn); |
1415 | |
1416 | // Create the switch for the split |
1417 | // MonotonicBB is arbitrarily chosen as the default case; in practice, this |
1418 | // doesn't matter unless someone is crazy enough to use something that |
1419 | // doesn't fold to a constant for the ordering. |
1420 | Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); |
1421 | llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); |
1422 | |
1423 | // Emit all the different atomics |
1424 | Builder.SetInsertPoint(MonotonicBB); |
1425 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1426 | llvm::AtomicOrdering::Monotonic, Scope); |
1427 | Builder.CreateBr(ContBB); |
1428 | if (!IsStore) { |
1429 | Builder.SetInsertPoint(AcquireBB); |
1430 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1431 | llvm::AtomicOrdering::Acquire, Scope); |
1432 | Builder.CreateBr(ContBB); |
1433 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), |
1434 | AcquireBB); |
1435 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), |
1436 | AcquireBB); |
1437 | } |
1438 | if (!IsLoad) { |
1439 | Builder.SetInsertPoint(ReleaseBB); |
1440 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1441 | llvm::AtomicOrdering::Release, Scope); |
1442 | Builder.CreateBr(ContBB); |
1443 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), |
1444 | ReleaseBB); |
1445 | } |
1446 | if (!IsLoad && !IsStore) { |
1447 | Builder.SetInsertPoint(AcqRelBB); |
1448 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1449 | llvm::AtomicOrdering::AcquireRelease, Scope); |
1450 | Builder.CreateBr(ContBB); |
1451 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), |
1452 | AcqRelBB); |
1453 | } |
1454 | Builder.SetInsertPoint(SeqCstBB); |
1455 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1456 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
1457 | Builder.CreateBr(ContBB); |
1458 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), |
1459 | SeqCstBB); |
1460 | |
1461 | // Cleanup and return |
1462 | Builder.SetInsertPoint(ContBB); |
1463 | if (RValTy->isVoidType()) |
1464 | return RValue::get(nullptr); |
1465 | |
1466 | assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); |
1467 | return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), |
1468 | RValTy, E->getExprLoc()); |
1469 | } |
1470 | |
1471 | Address AtomicInfo::castToAtomicIntPointer(Address addr) const { |
1472 | llvm::IntegerType *ty = |
1473 | llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); |
1474 | return addr.withElementType(ty); |
1475 | } |
1476 | |
1477 | Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { |
1478 | llvm::Type *Ty = Addr.getElementType(); |
1479 | uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); |
1480 | if (SourceSizeInBits != AtomicSizeInBits) { |
1481 | Address Tmp = CreateTempAlloca(); |
1482 | CGF.Builder.CreateMemCpy(Tmp, Addr, |
1483 | std::min(AtomicSizeInBits, SourceSizeInBits) / 8); |
1484 | Addr = Tmp; |
1485 | } |
1486 | |
1487 | return castToAtomicIntPointer(Addr); |
1488 | } |
1489 | |
1490 | RValue AtomicInfo::convertAtomicTempToRValue(Address addr, |
1491 | AggValueSlot resultSlot, |
1492 | SourceLocation loc, |
1493 | bool asValue) const { |
1494 | if (LVal.isSimple()) { |
1495 | if (EvaluationKind == TEK_Aggregate) |
1496 | return resultSlot.asRValue(); |
1497 | |
1498 | // Drill into the padding structure if we have one. |
1499 | if (hasPadding()) |
1500 | addr = CGF.Builder.CreateStructGEP(addr, 0); |
1501 | |
1502 | // Otherwise, just convert the temporary to an r-value using the |
1503 | // normal conversion routine. |
1504 | return CGF.convertTempToRValue(addr, getValueType(), loc); |
1505 | } |
1506 | if (!asValue) |
1507 | // Get RValue from temp memory as atomic for non-simple lvalues |
1508 | return RValue::get(CGF.Builder.CreateLoad(addr)); |
1509 | if (LVal.isBitField()) |
1510 | return CGF.EmitLoadOfBitfieldLValue( |
1511 | LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), |
1512 | LVal.getBaseInfo(), TBAAAccessInfo()), loc); |
1513 | if (LVal.isVectorElt()) |
1514 | return CGF.EmitLoadOfLValue( |
1515 | LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), |
1516 | LVal.getBaseInfo(), TBAAAccessInfo()), loc); |
1517 | assert(LVal.isExtVectorElt()); |
1518 | return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( |
1519 | addr, LVal.getExtVectorElts(), LVal.getType(), |
1520 | LVal.getBaseInfo(), TBAAAccessInfo())); |
1521 | } |
1522 | |
1523 | RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, |
1524 | AggValueSlot ResultSlot, |
1525 | SourceLocation Loc, |
1526 | bool AsValue) const { |
1527 | // Try not to in some easy cases. |
1528 | assert(IntVal->getType()->isIntegerTy() && "Expected integer value" ); |
1529 | if (getEvaluationKind() == TEK_Scalar && |
1530 | (((!LVal.isBitField() || |
1531 | LVal.getBitFieldInfo().Size == ValueSizeInBits) && |
1532 | !hasPadding()) || |
1533 | !AsValue)) { |
1534 | auto *ValTy = AsValue |
1535 | ? CGF.ConvertTypeForMem(ValueTy) |
1536 | : getAtomicAddress().getElementType(); |
1537 | if (ValTy->isIntegerTy()) { |
1538 | assert(IntVal->getType() == ValTy && "Different integer types." ); |
1539 | return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); |
1540 | } else if (ValTy->isPointerTy()) |
1541 | return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); |
1542 | else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) |
1543 | return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); |
1544 | } |
1545 | |
1546 | // Create a temporary. This needs to be big enough to hold the |
1547 | // atomic integer. |
1548 | Address Temp = Address::invalid(); |
1549 | bool TempIsVolatile = false; |
1550 | if (AsValue && getEvaluationKind() == TEK_Aggregate) { |
1551 | assert(!ResultSlot.isIgnored()); |
1552 | Temp = ResultSlot.getAddress(); |
1553 | TempIsVolatile = ResultSlot.isVolatile(); |
1554 | } else { |
1555 | Temp = CreateTempAlloca(); |
1556 | } |
1557 | |
1558 | // Slam the integer into the temporary. |
1559 | Address CastTemp = castToAtomicIntPointer(Temp); |
1560 | CGF.Builder.CreateStore(IntVal, CastTemp) |
1561 | ->setVolatile(TempIsVolatile); |
1562 | |
1563 | return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); |
1564 | } |
1565 | |
1566 | void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, |
1567 | llvm::AtomicOrdering AO, bool) { |
1568 | // void __atomic_load(size_t size, void *mem, void *return, int order); |
1569 | CallArgList Args; |
1570 | Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); |
1571 | Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); |
1572 | Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy); |
1573 | Args.add( |
1574 | RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), |
1575 | CGF.getContext().IntTy); |
1576 | emitAtomicLibcall(CGF, "__atomic_load" , CGF.getContext().VoidTy, Args); |
1577 | } |
1578 | |
1579 | llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, |
1580 | bool IsVolatile) { |
1581 | // Okay, we're doing this natively. |
1582 | Address Addr = getAtomicAddressAsAtomicIntPointer(); |
1583 | llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load" ); |
1584 | Load->setAtomic(AO); |
1585 | |
1586 | // Other decoration. |
1587 | if (IsVolatile) |
1588 | Load->setVolatile(true); |
1589 | CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); |
1590 | return Load; |
1591 | } |
1592 | |
1593 | /// An LValue is a candidate for having its loads and stores be made atomic if |
1594 | /// we are operating under /volatile:ms *and* the LValue itself is volatile and |
1595 | /// performing such an operation can be performed without a libcall. |
1596 | bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { |
1597 | if (!CGM.getLangOpts().MSVolatile) return false; |
1598 | AtomicInfo AI(*this, LV); |
1599 | bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); |
1600 | // An atomic is inline if we don't need to use a libcall. |
1601 | bool AtomicIsInline = !AI.shouldUseLibcall(); |
1602 | // MSVC doesn't seem to do this for types wider than a pointer. |
1603 | if (getContext().getTypeSize(LV.getType()) > |
1604 | getContext().getTypeSize(getContext().getIntPtrType())) |
1605 | return false; |
1606 | return IsVolatile && AtomicIsInline; |
1607 | } |
1608 | |
1609 | RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, |
1610 | AggValueSlot Slot) { |
1611 | llvm::AtomicOrdering AO; |
1612 | bool IsVolatile = LV.isVolatileQualified(); |
1613 | if (LV.getType()->isAtomicType()) { |
1614 | AO = llvm::AtomicOrdering::SequentiallyConsistent; |
1615 | } else { |
1616 | AO = llvm::AtomicOrdering::Acquire; |
1617 | IsVolatile = true; |
1618 | } |
1619 | return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); |
1620 | } |
1621 | |
1622 | RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, |
1623 | bool AsValue, llvm::AtomicOrdering AO, |
1624 | bool IsVolatile) { |
1625 | // Check whether we should use a library call. |
1626 | if (shouldUseLibcall()) { |
1627 | Address TempAddr = Address::invalid(); |
1628 | if (LVal.isSimple() && !ResultSlot.isIgnored()) { |
1629 | assert(getEvaluationKind() == TEK_Aggregate); |
1630 | TempAddr = ResultSlot.getAddress(); |
1631 | } else |
1632 | TempAddr = CreateTempAlloca(); |
1633 | |
1634 | EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); |
1635 | |
1636 | // Okay, turn that back into the original value or whole atomic (for |
1637 | // non-simple lvalues) type. |
1638 | return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); |
1639 | } |
1640 | |
1641 | // Okay, we're doing this natively. |
1642 | auto *Load = EmitAtomicLoadOp(AO, IsVolatile); |
1643 | |
1644 | // If we're ignoring an aggregate return, don't do anything. |
1645 | if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) |
1646 | return RValue::getAggregate(Address::invalid(), false); |
1647 | |
1648 | // Okay, turn that back into the original value or atomic (for non-simple |
1649 | // lvalues) type. |
1650 | return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); |
1651 | } |
1652 | |
1653 | /// Emit a load from an l-value of atomic type. Note that the r-value |
1654 | /// we produce is an r-value of the atomic *value* type. |
1655 | RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, |
1656 | llvm::AtomicOrdering AO, bool IsVolatile, |
1657 | AggValueSlot resultSlot) { |
1658 | AtomicInfo Atomics(*this, src); |
1659 | return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, |
1660 | IsVolatile); |
1661 | } |
1662 | |
1663 | /// Copy an r-value into memory as part of storing to an atomic type. |
1664 | /// This needs to create a bit-pattern suitable for atomic operations. |
1665 | void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { |
1666 | assert(LVal.isSimple()); |
1667 | // If we have an r-value, the rvalue should be of the atomic type, |
1668 | // which means that the caller is responsible for having zeroed |
1669 | // any padding. Just do an aggregate copy of that type. |
1670 | if (rvalue.isAggregate()) { |
1671 | LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); |
1672 | LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), |
1673 | getAtomicType()); |
1674 | bool IsVolatile = rvalue.isVolatileQualified() || |
1675 | LVal.isVolatileQualified(); |
1676 | CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), |
1677 | AggValueSlot::DoesNotOverlap, IsVolatile); |
1678 | return; |
1679 | } |
1680 | |
1681 | // Okay, otherwise we're copying stuff. |
1682 | |
1683 | // Zero out the buffer if necessary. |
1684 | emitMemSetZeroIfNecessary(); |
1685 | |
1686 | // Drill past the padding if present. |
1687 | LValue TempLVal = projectValue(); |
1688 | |
1689 | // Okay, store the rvalue in. |
1690 | if (rvalue.isScalar()) { |
1691 | CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); |
1692 | } else { |
1693 | CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); |
1694 | } |
1695 | } |
1696 | |
1697 | |
1698 | /// Materialize an r-value into memory for the purposes of storing it |
1699 | /// to an atomic type. |
1700 | Address AtomicInfo::materializeRValue(RValue rvalue) const { |
1701 | // Aggregate r-values are already in memory, and EmitAtomicStore |
1702 | // requires them to be values of the atomic type. |
1703 | if (rvalue.isAggregate()) |
1704 | return rvalue.getAggregateAddress(); |
1705 | |
1706 | // Otherwise, make a temporary and materialize into it. |
1707 | LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); |
1708 | AtomicInfo Atomics(CGF, TempLV); |
1709 | Atomics.emitCopyIntoMemory(rvalue); |
1710 | return TempLV.getAddress(CGF); |
1711 | } |
1712 | |
1713 | llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { |
1714 | // If we've got a scalar value of the right size, try to avoid going |
1715 | // through memory. |
1716 | if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { |
1717 | llvm::Value *Value = RVal.getScalarVal(); |
1718 | if (isa<llvm::IntegerType>(Value->getType())) |
1719 | return CGF.EmitToMemory(Value, ValueTy); |
1720 | else { |
1721 | llvm::IntegerType *InputIntTy = llvm::IntegerType::get( |
1722 | CGF.getLLVMContext(), |
1723 | LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); |
1724 | if (isa<llvm::PointerType>(Value->getType())) |
1725 | return CGF.Builder.CreatePtrToInt(Value, InputIntTy); |
1726 | else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) |
1727 | return CGF.Builder.CreateBitCast(Value, InputIntTy); |
1728 | } |
1729 | } |
1730 | // Otherwise, we need to go through memory. |
1731 | // Put the r-value in memory. |
1732 | Address Addr = materializeRValue(RVal); |
1733 | |
1734 | // Cast the temporary to the atomic int type and pull a value out. |
1735 | Addr = castToAtomicIntPointer(Addr); |
1736 | return CGF.Builder.CreateLoad(Addr); |
1737 | } |
1738 | |
1739 | std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( |
1740 | llvm::Value *ExpectedVal, llvm::Value *DesiredVal, |
1741 | llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { |
1742 | // Do the atomic store. |
1743 | Address Addr = getAtomicAddressAsAtomicIntPointer(); |
1744 | auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), |
1745 | ExpectedVal, DesiredVal, |
1746 | Success, Failure); |
1747 | // Other decoration. |
1748 | Inst->setVolatile(LVal.isVolatileQualified()); |
1749 | Inst->setWeak(IsWeak); |
1750 | |
1751 | // Okay, turn that back into the original value type. |
1752 | auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); |
1753 | auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); |
1754 | return std::make_pair(PreviousVal, SuccessFailureVal); |
1755 | } |
1756 | |
1757 | llvm::Value * |
1758 | AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, |
1759 | llvm::Value *DesiredAddr, |
1760 | llvm::AtomicOrdering Success, |
1761 | llvm::AtomicOrdering Failure) { |
1762 | // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, |
1763 | // void *desired, int success, int failure); |
1764 | CallArgList Args; |
1765 | Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); |
1766 | Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); |
1767 | Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy); |
1768 | Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy); |
1769 | Args.add(RValue::get( |
1770 | llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), |
1771 | CGF.getContext().IntTy); |
1772 | Args.add(RValue::get( |
1773 | llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), |
1774 | CGF.getContext().IntTy); |
1775 | auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange" , |
1776 | CGF.getContext().BoolTy, Args); |
1777 | |
1778 | return SuccessFailureRVal.getScalarVal(); |
1779 | } |
1780 | |
1781 | std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( |
1782 | RValue Expected, RValue Desired, llvm::AtomicOrdering Success, |
1783 | llvm::AtomicOrdering Failure, bool IsWeak) { |
1784 | // Check whether we should use a library call. |
1785 | if (shouldUseLibcall()) { |
1786 | // Produce a source address. |
1787 | Address ExpectedAddr = materializeRValue(Expected); |
1788 | Address DesiredAddr = materializeRValue(Desired); |
1789 | auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1790 | DesiredAddr.getPointer(), |
1791 | Success, Failure); |
1792 | return std::make_pair( |
1793 | convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), |
1794 | SourceLocation(), /*AsValue=*/false), |
1795 | Res); |
1796 | } |
1797 | |
1798 | // If we've got a scalar value of the right size, try to avoid going |
1799 | // through memory. |
1800 | auto *ExpectedVal = convertRValueToInt(Expected); |
1801 | auto *DesiredVal = convertRValueToInt(Desired); |
1802 | auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, |
1803 | Failure, IsWeak); |
1804 | return std::make_pair( |
1805 | ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), |
1806 | SourceLocation(), /*AsValue=*/false), |
1807 | Res.second); |
1808 | } |
1809 | |
1810 | static void |
1811 | EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, |
1812 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1813 | Address DesiredAddr) { |
1814 | RValue UpRVal; |
1815 | LValue AtomicLVal = Atomics.getAtomicLValue(); |
1816 | LValue DesiredLVal; |
1817 | if (AtomicLVal.isSimple()) { |
1818 | UpRVal = OldRVal; |
1819 | DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); |
1820 | } else { |
1821 | // Build new lvalue for temp address. |
1822 | Address Ptr = Atomics.materializeRValue(OldRVal); |
1823 | LValue UpdateLVal; |
1824 | if (AtomicLVal.isBitField()) { |
1825 | UpdateLVal = |
1826 | LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), |
1827 | AtomicLVal.getType(), |
1828 | AtomicLVal.getBaseInfo(), |
1829 | AtomicLVal.getTBAAInfo()); |
1830 | DesiredLVal = |
1831 | LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), |
1832 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1833 | AtomicLVal.getTBAAInfo()); |
1834 | } else if (AtomicLVal.isVectorElt()) { |
1835 | UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), |
1836 | AtomicLVal.getType(), |
1837 | AtomicLVal.getBaseInfo(), |
1838 | AtomicLVal.getTBAAInfo()); |
1839 | DesiredLVal = LValue::MakeVectorElt( |
1840 | DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), |
1841 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1842 | } else { |
1843 | assert(AtomicLVal.isExtVectorElt()); |
1844 | UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), |
1845 | AtomicLVal.getType(), |
1846 | AtomicLVal.getBaseInfo(), |
1847 | AtomicLVal.getTBAAInfo()); |
1848 | DesiredLVal = LValue::MakeExtVectorElt( |
1849 | DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), |
1850 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1851 | } |
1852 | UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); |
1853 | } |
1854 | // Store new value in the corresponding memory area. |
1855 | RValue NewRVal = UpdateOp(UpRVal); |
1856 | if (NewRVal.isScalar()) { |
1857 | CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); |
1858 | } else { |
1859 | assert(NewRVal.isComplex()); |
1860 | CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, |
1861 | /*isInit=*/false); |
1862 | } |
1863 | } |
1864 | |
1865 | void AtomicInfo::EmitAtomicUpdateLibcall( |
1866 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1867 | bool IsVolatile) { |
1868 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1869 | |
1870 | Address ExpectedAddr = CreateTempAlloca(); |
1871 | |
1872 | EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); |
1873 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1874 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1875 | CGF.EmitBlock(ContBB); |
1876 | Address DesiredAddr = CreateTempAlloca(); |
1877 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1878 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1879 | auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); |
1880 | CGF.Builder.CreateStore(OldVal, DesiredAddr); |
1881 | } |
1882 | auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, |
1883 | AggValueSlot::ignored(), |
1884 | SourceLocation(), /*AsValue=*/false); |
1885 | EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); |
1886 | auto *Res = |
1887 | EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1888 | DesiredAddr.getPointer(), |
1889 | AO, Failure); |
1890 | CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); |
1891 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1892 | } |
1893 | |
1894 | void AtomicInfo::EmitAtomicUpdateOp( |
1895 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1896 | bool IsVolatile) { |
1897 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1898 | |
1899 | // Do the atomic load. |
1900 | auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); |
1901 | // For non-simple lvalues perform compare-and-swap procedure. |
1902 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1903 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1904 | auto *CurBB = CGF.Builder.GetInsertBlock(); |
1905 | CGF.EmitBlock(ContBB); |
1906 | llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), |
1907 | /*NumReservedValues=*/2); |
1908 | PHI->addIncoming(OldVal, CurBB); |
1909 | Address NewAtomicAddr = CreateTempAlloca(); |
1910 | Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); |
1911 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1912 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1913 | CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); |
1914 | } |
1915 | auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), |
1916 | SourceLocation(), /*AsValue=*/false); |
1917 | EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); |
1918 | auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); |
1919 | // Try to write new value using cmpxchg operation. |
1920 | auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); |
1921 | PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); |
1922 | CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); |
1923 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1924 | } |
1925 | |
1926 | static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, |
1927 | RValue UpdateRVal, Address DesiredAddr) { |
1928 | LValue AtomicLVal = Atomics.getAtomicLValue(); |
1929 | LValue DesiredLVal; |
1930 | // Build new lvalue for temp address. |
1931 | if (AtomicLVal.isBitField()) { |
1932 | DesiredLVal = |
1933 | LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), |
1934 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1935 | AtomicLVal.getTBAAInfo()); |
1936 | } else if (AtomicLVal.isVectorElt()) { |
1937 | DesiredLVal = |
1938 | LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), |
1939 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1940 | AtomicLVal.getTBAAInfo()); |
1941 | } else { |
1942 | assert(AtomicLVal.isExtVectorElt()); |
1943 | DesiredLVal = LValue::MakeExtVectorElt( |
1944 | DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), |
1945 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1946 | } |
1947 | // Store new value in the corresponding memory area. |
1948 | assert(UpdateRVal.isScalar()); |
1949 | CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); |
1950 | } |
1951 | |
1952 | void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, |
1953 | RValue UpdateRVal, bool IsVolatile) { |
1954 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1955 | |
1956 | Address ExpectedAddr = CreateTempAlloca(); |
1957 | |
1958 | EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); |
1959 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1960 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1961 | CGF.EmitBlock(ContBB); |
1962 | Address DesiredAddr = CreateTempAlloca(); |
1963 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1964 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1965 | auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); |
1966 | CGF.Builder.CreateStore(OldVal, DesiredAddr); |
1967 | } |
1968 | EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); |
1969 | auto *Res = |
1970 | EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1971 | DesiredAddr.getPointer(), |
1972 | AO, Failure); |
1973 | CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); |
1974 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1975 | } |
1976 | |
1977 | void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, |
1978 | bool IsVolatile) { |
1979 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1980 | |
1981 | // Do the atomic load. |
1982 | auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); |
1983 | // For non-simple lvalues perform compare-and-swap procedure. |
1984 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1985 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1986 | auto *CurBB = CGF.Builder.GetInsertBlock(); |
1987 | CGF.EmitBlock(ContBB); |
1988 | llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), |
1989 | /*NumReservedValues=*/2); |
1990 | PHI->addIncoming(OldVal, CurBB); |
1991 | Address NewAtomicAddr = CreateTempAlloca(); |
1992 | Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); |
1993 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1994 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1995 | CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); |
1996 | } |
1997 | EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); |
1998 | auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); |
1999 | // Try to write new value using cmpxchg operation. |
2000 | auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); |
2001 | PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); |
2002 | CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); |
2003 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
2004 | } |
2005 | |
2006 | void AtomicInfo::EmitAtomicUpdate( |
2007 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
2008 | bool IsVolatile) { |
2009 | if (shouldUseLibcall()) { |
2010 | EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); |
2011 | } else { |
2012 | EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); |
2013 | } |
2014 | } |
2015 | |
2016 | void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, |
2017 | bool IsVolatile) { |
2018 | if (shouldUseLibcall()) { |
2019 | EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); |
2020 | } else { |
2021 | EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); |
2022 | } |
2023 | } |
2024 | |
2025 | void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, |
2026 | bool isInit) { |
2027 | bool IsVolatile = lvalue.isVolatileQualified(); |
2028 | llvm::AtomicOrdering AO; |
2029 | if (lvalue.getType()->isAtomicType()) { |
2030 | AO = llvm::AtomicOrdering::SequentiallyConsistent; |
2031 | } else { |
2032 | AO = llvm::AtomicOrdering::Release; |
2033 | IsVolatile = true; |
2034 | } |
2035 | return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); |
2036 | } |
2037 | |
2038 | /// Emit a store to an l-value of atomic type. |
2039 | /// |
2040 | /// Note that the r-value is expected to be an r-value *of the atomic |
2041 | /// type*; this means that for aggregate r-values, it should include |
2042 | /// storage for any padding that was necessary. |
2043 | void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, |
2044 | llvm::AtomicOrdering AO, bool IsVolatile, |
2045 | bool isInit) { |
2046 | // If this is an aggregate r-value, it should agree in type except |
2047 | // maybe for address-space qualification. |
2048 | assert(!rvalue.isAggregate() || |
2049 | rvalue.getAggregateAddress().getElementType() == |
2050 | dest.getAddress(*this).getElementType()); |
2051 | |
2052 | AtomicInfo atomics(*this, dest); |
2053 | LValue LVal = atomics.getAtomicLValue(); |
2054 | |
2055 | // If this is an initialization, just put the value there normally. |
2056 | if (LVal.isSimple()) { |
2057 | if (isInit) { |
2058 | atomics.emitCopyIntoMemory(rvalue); |
2059 | return; |
2060 | } |
2061 | |
2062 | // Check whether we should use a library call. |
2063 | if (atomics.shouldUseLibcall()) { |
2064 | // Produce a source address. |
2065 | Address srcAddr = atomics.materializeRValue(rvalue); |
2066 | |
2067 | // void __atomic_store(size_t size, void *mem, void *val, int order) |
2068 | CallArgList args; |
2069 | args.add(RValue::get(atomics.getAtomicSizeValue()), |
2070 | getContext().getSizeType()); |
2071 | args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); |
2072 | args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); |
2073 | args.add( |
2074 | RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), |
2075 | getContext().IntTy); |
2076 | emitAtomicLibcall(*this, "__atomic_store" , getContext().VoidTy, args); |
2077 | return; |
2078 | } |
2079 | |
2080 | // Okay, we're doing this natively. |
2081 | llvm::Value *intValue = atomics.convertRValueToInt(rvalue); |
2082 | |
2083 | // Do the atomic store. |
2084 | Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress()); |
2085 | intValue = Builder.CreateIntCast( |
2086 | intValue, addr.getElementType(), /*isSigned=*/false); |
2087 | llvm::StoreInst *store = Builder.CreateStore(intValue, addr); |
2088 | |
2089 | if (AO == llvm::AtomicOrdering::Acquire) |
2090 | AO = llvm::AtomicOrdering::Monotonic; |
2091 | else if (AO == llvm::AtomicOrdering::AcquireRelease) |
2092 | AO = llvm::AtomicOrdering::Release; |
2093 | // Initializations don't need to be atomic. |
2094 | if (!isInit) |
2095 | store->setAtomic(AO); |
2096 | |
2097 | // Other decoration. |
2098 | if (IsVolatile) |
2099 | store->setVolatile(true); |
2100 | CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); |
2101 | return; |
2102 | } |
2103 | |
2104 | // Emit simple atomic update operation. |
2105 | atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); |
2106 | } |
2107 | |
2108 | /// Emit a compare-and-exchange op for atomic type. |
2109 | /// |
2110 | std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( |
2111 | LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, |
2112 | llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, |
2113 | AggValueSlot Slot) { |
2114 | // If this is an aggregate r-value, it should agree in type except |
2115 | // maybe for address-space qualification. |
2116 | assert(!Expected.isAggregate() || |
2117 | Expected.getAggregateAddress().getElementType() == |
2118 | Obj.getAddress(*this).getElementType()); |
2119 | assert(!Desired.isAggregate() || |
2120 | Desired.getAggregateAddress().getElementType() == |
2121 | Obj.getAddress(*this).getElementType()); |
2122 | AtomicInfo Atomics(*this, Obj); |
2123 | |
2124 | return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, |
2125 | IsWeak); |
2126 | } |
2127 | |
2128 | void CodeGenFunction::EmitAtomicUpdate( |
2129 | LValue LVal, llvm::AtomicOrdering AO, |
2130 | const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { |
2131 | AtomicInfo Atomics(*this, LVal); |
2132 | Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); |
2133 | } |
2134 | |
2135 | void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { |
2136 | AtomicInfo atomics(*this, dest); |
2137 | |
2138 | switch (atomics.getEvaluationKind()) { |
2139 | case TEK_Scalar: { |
2140 | llvm::Value *value = EmitScalarExpr(init); |
2141 | atomics.emitCopyIntoMemory(RValue::get(value)); |
2142 | return; |
2143 | } |
2144 | |
2145 | case TEK_Complex: { |
2146 | ComplexPairTy value = EmitComplexExpr(init); |
2147 | atomics.emitCopyIntoMemory(RValue::getComplex(value)); |
2148 | return; |
2149 | } |
2150 | |
2151 | case TEK_Aggregate: { |
2152 | // Fix up the destination if the initializer isn't an expression |
2153 | // of atomic type. |
2154 | bool Zeroed = false; |
2155 | if (!init->getType()->isAtomicType()) { |
2156 | Zeroed = atomics.emitMemSetZeroIfNecessary(); |
2157 | dest = atomics.projectValue(); |
2158 | } |
2159 | |
2160 | // Evaluate the expression directly into the destination. |
2161 | AggValueSlot slot = AggValueSlot::forLValue( |
2162 | dest, *this, AggValueSlot::IsNotDestructed, |
2163 | AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, |
2164 | AggValueSlot::DoesNotOverlap, |
2165 | Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); |
2166 | |
2167 | EmitAggExpr(init, slot); |
2168 | return; |
2169 | } |
2170 | } |
2171 | llvm_unreachable("bad evaluation kind" ); |
2172 | } |
2173 | |