1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/CodeGen/CGFunctionInfo.h"
20#include "clang/Frontend/FrontendDiagnostic.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
25
26using namespace clang;
27using namespace CodeGen;
28
29namespace {
30 class AtomicInfo {
31 CodeGenFunction &CGF;
32 QualType AtomicTy;
33 QualType ValueTy;
34 uint64_t AtomicSizeInBits;
35 uint64_t ValueSizeInBits;
36 CharUnits AtomicAlign;
37 CharUnits ValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
41 CGBitFieldInfo BFI;
42 public:
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .alignTo(lvalue.getAlignment()));
83 llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
88 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
89 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
90 StoragePtr, llvm::PointerType::getUnqual(CGF.getLLVMContext()),
91 "atomic_bitfield_base");
92 BFI = OrigBFI;
93 BFI.Offset = Offset;
94 BFI.StorageSize = AtomicSizeInBits;
95 BFI.StorageOffset += OffsetInChars;
96 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
97 LVal = LValue::MakeBitfield(
98 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
99 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101 if (AtomicTy.isNull()) {
102 llvm::APInt Size(
103 /*numBits=*/32,
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy =
106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107 /*IndexTypeQuals=*/0);
108 }
109 AtomicAlign = ValueAlign = lvalue.getAlignment();
110 } else if (lvalue.isVectorElt()) {
111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112 ValueSizeInBits = C.getTypeSize(ValueTy);
113 AtomicTy = lvalue.getType();
114 AtomicSizeInBits = C.getTypeSize(AtomicTy);
115 AtomicAlign = ValueAlign = lvalue.getAlignment();
116 LVal = lvalue;
117 } else {
118 assert(lvalue.isExtVectorElt());
119 ValueTy = lvalue.getType();
120 ValueSizeInBits = C.getTypeSize(ValueTy);
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122 lvalue.getType(), cast<llvm::FixedVectorType>(
123 lvalue.getExtVectorAddress().getElementType())
124 ->getNumElements());
125 AtomicSizeInBits = C.getTypeSize(AtomicTy);
126 AtomicAlign = ValueAlign = lvalue.getAlignment();
127 LVal = lvalue;
128 }
129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131 }
132
133 QualType getAtomicType() const { return AtomicTy; }
134 QualType getValueType() const { return ValueTy; }
135 CharUnits getAtomicAlignment() const { return AtomicAlign; }
136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139 bool shouldUseLibcall() const { return UseLibcall; }
140 const LValue &getAtomicLValue() const { return LVal; }
141 llvm::Value *getAtomicPointer() const {
142 if (LVal.isSimple())
143 return LVal.getPointer(CGF);
144 else if (LVal.isBitField())
145 return LVal.getBitFieldPointer();
146 else if (LVal.isVectorElt())
147 return LVal.getVectorPointer();
148 assert(LVal.isExtVectorElt());
149 return LVal.getExtVectorPointer();
150 }
151 Address getAtomicAddress() const {
152 llvm::Type *ElTy;
153 if (LVal.isSimple())
154 ElTy = LVal.getAddress(CGF).getElementType();
155 else if (LVal.isBitField())
156 ElTy = LVal.getBitFieldAddress().getElementType();
157 else if (LVal.isVectorElt())
158 ElTy = LVal.getVectorAddress().getElementType();
159 else
160 ElTy = LVal.getExtVectorAddress().getElementType();
161 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
162 }
163
164 Address getAtomicAddressAsAtomicIntPointer() const {
165 return castToAtomicIntPointer(getAtomicAddress());
166 }
167
168 /// Is the atomic size larger than the underlying value type?
169 ///
170 /// Note that the absence of padding does not mean that atomic
171 /// objects are completely interchangeable with non-atomic
172 /// objects: we might have promoted the alignment of a type
173 /// without making it bigger.
174 bool hasPadding() const {
175 return (ValueSizeInBits != AtomicSizeInBits);
176 }
177
178 bool emitMemSetZeroIfNecessary() const;
179
180 llvm::Value *getAtomicSizeValue() const {
181 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
182 return CGF.CGM.getSize(size);
183 }
184
185 /// Cast the given pointer to an integer pointer suitable for atomic
186 /// operations if the source.
187 Address castToAtomicIntPointer(Address Addr) const;
188
189 /// If Addr is compatible with the iN that will be used for an atomic
190 /// operation, bitcast it. Otherwise, create a temporary that is suitable
191 /// and copy the value across.
192 Address convertToAtomicIntPointer(Address Addr) const;
193
194 /// Turn an atomic-layout object into an r-value.
195 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
196 SourceLocation loc, bool AsValue) const;
197
198 /// Converts a rvalue to integer value.
199 llvm::Value *convertRValueToInt(RValue RVal) const;
200
201 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
202 AggValueSlot ResultSlot,
203 SourceLocation Loc, bool AsValue) const;
204
205 /// Copy an atomic r-value into atomic-layout memory.
206 void emitCopyIntoMemory(RValue rvalue) const;
207
208 /// Project an l-value down to the value field.
209 LValue projectValue() const {
210 assert(LVal.isSimple());
211 Address addr = getAtomicAddress();
212 if (hasPadding())
213 addr = CGF.Builder.CreateStructGEP(addr, 0);
214
215 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
216 LVal.getBaseInfo(), LVal.getTBAAInfo());
217 }
218
219 /// Emits atomic load.
220 /// \returns Loaded value.
221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222 bool AsValue, llvm::AtomicOrdering AO,
223 bool IsVolatile);
224
225 /// Emits atomic compare-and-exchange sequence.
226 /// \param Expected Expected value.
227 /// \param Desired Desired value.
228 /// \param Success Atomic ordering for success operation.
229 /// \param Failure Atomic ordering for failed operation.
230 /// \param IsWeak true if atomic operation is weak, false otherwise.
231 /// \returns Pair of values: previous value from storage (value type) and
232 /// boolean flag (i1 type) with true if success and false otherwise.
233 std::pair<RValue, llvm::Value *>
234 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235 llvm::AtomicOrdering Success =
236 llvm::AtomicOrdering::SequentiallyConsistent,
237 llvm::AtomicOrdering Failure =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 bool IsWeak = false);
240
241 /// Emits atomic update.
242 /// \param AO Atomic ordering.
243 /// \param UpdateOp Update operation for the current lvalue.
244 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245 const llvm::function_ref<RValue(RValue)> &UpdateOp,
246 bool IsVolatile);
247 /// Emits atomic update.
248 /// \param AO Atomic ordering.
249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250 bool IsVolatile);
251
252 /// Materialize an atomic r-value in atomic-layout memory.
253 Address materializeRValue(RValue rvalue) const;
254
255 /// Creates temp alloca for intermediate operations on atomic value.
256 Address CreateTempAlloca() const;
257 private:
258 bool requiresMemSetZero(llvm::Type *type) const;
259
260
261 /// Emits atomic load as a libcall.
262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263 llvm::AtomicOrdering AO, bool IsVolatile);
264 /// Emits atomic load as LLVM instruction.
265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
266 /// Emits atomic compare-and-exchange op as a libcall.
267 llvm::Value *EmitAtomicCompareExchangeLibcall(
268 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
269 llvm::AtomicOrdering Success =
270 llvm::AtomicOrdering::SequentiallyConsistent,
271 llvm::AtomicOrdering Failure =
272 llvm::AtomicOrdering::SequentiallyConsistent);
273 /// Emits atomic compare-and-exchange op as LLVM instruction.
274 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
275 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
276 llvm::AtomicOrdering Success =
277 llvm::AtomicOrdering::SequentiallyConsistent,
278 llvm::AtomicOrdering Failure =
279 llvm::AtomicOrdering::SequentiallyConsistent,
280 bool IsWeak = false);
281 /// Emit atomic update as libcalls.
282 void
283 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
284 const llvm::function_ref<RValue(RValue)> &UpdateOp,
285 bool IsVolatile);
286 /// Emit atomic update as LLVM instructions.
287 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
288 const llvm::function_ref<RValue(RValue)> &UpdateOp,
289 bool IsVolatile);
290 /// Emit atomic update as libcalls.
291 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
292 bool IsVolatile);
293 /// Emit atomic update as LLVM instructions.
294 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
295 bool IsVolatile);
296 };
297}
298
299Address AtomicInfo::CreateTempAlloca() const {
300 Address TempAlloca = CGF.CreateMemTemp(
301 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
302 : AtomicTy,
303 getAtomicAlignment(),
304 "atomic-temp");
305 // Cast to pointer to value type for bitfields.
306 if (LVal.isBitField())
307 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
308 TempAlloca, getAtomicAddress().getType(),
309 getAtomicAddress().getElementType());
310 return TempAlloca;
311}
312
313static RValue emitAtomicLibcall(CodeGenFunction &CGF,
314 StringRef fnName,
315 QualType resultType,
316 CallArgList &args) {
317 const CGFunctionInfo &fnInfo =
318 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
319 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
320 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
321 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
322 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
323 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
324 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
325
326 llvm::FunctionCallee fn =
327 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
328 auto callee = CGCallee::forDirect(fn);
329 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
330}
331
332/// Does a store of the given IR type modify the full expected width?
333static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
334 uint64_t expectedSize) {
335 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
336}
337
338/// Does the atomic type require memsetting to zero before initialization?
339///
340/// The IR type is provided as a way of making certain queries faster.
341bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
342 // If the atomic type has size padding, we definitely need a memset.
343 if (hasPadding()) return true;
344
345 // Otherwise, do some simple heuristics to try to avoid it:
346 switch (getEvaluationKind()) {
347 // For scalars and complexes, check whether the store size of the
348 // type uses the full size.
349 case TEK_Scalar:
350 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
351 case TEK_Complex:
352 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
353 AtomicSizeInBits / 2);
354
355 // Padding in structs has an undefined bit pattern. User beware.
356 case TEK_Aggregate:
357 return false;
358 }
359 llvm_unreachable("bad evaluation kind");
360}
361
362bool AtomicInfo::emitMemSetZeroIfNecessary() const {
363 assert(LVal.isSimple());
364 Address addr = LVal.getAddress(CGF);
365 if (!requiresMemSetZero(addr.getElementType()))
366 return false;
367
368 CGF.Builder.CreateMemSet(
369 addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
370 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
371 LVal.getAlignment().getAsAlign());
372 return true;
373}
374
375static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
376 Address Dest, Address Ptr,
377 Address Val1, Address Val2,
378 uint64_t Size,
379 llvm::AtomicOrdering SuccessOrder,
380 llvm::AtomicOrdering FailureOrder,
381 llvm::SyncScope::ID Scope) {
382 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
383 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
384 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
385
386 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
387 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
388 Scope);
389 Pair->setVolatile(E->isVolatile());
390 Pair->setWeak(IsWeak);
391
392 // Cmp holds the result of the compare-exchange operation: true on success,
393 // false on failure.
394 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
395 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
396
397 // This basic block is used to hold the store instruction if the operation
398 // failed.
399 llvm::BasicBlock *StoreExpectedBB =
400 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
401
402 // This basic block is the exit point of the operation, we should end up
403 // here regardless of whether or not the operation succeeded.
404 llvm::BasicBlock *ContinueBB =
405 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
406
407 // Update Expected if Expected isn't equal to Old, otherwise branch to the
408 // exit point.
409 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
410
411 CGF.Builder.SetInsertPoint(StoreExpectedBB);
412 // Update the memory at Expected with Old's value.
413 CGF.Builder.CreateStore(Old, Val1);
414 // Finally, branch to the exit point.
415 CGF.Builder.CreateBr(ContinueBB);
416
417 CGF.Builder.SetInsertPoint(ContinueBB);
418 // Update the memory at Dest with Cmp's value.
419 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
420}
421
422/// Given an ordering required on success, emit all possible cmpxchg
423/// instructions to cope with the provided (but possibly only dynamically known)
424/// FailureOrder.
425static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
426 bool IsWeak, Address Dest, Address Ptr,
427 Address Val1, Address Val2,
428 llvm::Value *FailureOrderVal,
429 uint64_t Size,
430 llvm::AtomicOrdering SuccessOrder,
431 llvm::SyncScope::ID Scope) {
432 llvm::AtomicOrdering FailureOrder;
433 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
434 auto FOS = FO->getSExtValue();
435 if (!llvm::isValidAtomicOrderingCABI(FOS))
436 FailureOrder = llvm::AtomicOrdering::Monotonic;
437 else
438 switch ((llvm::AtomicOrderingCABI)FOS) {
439 case llvm::AtomicOrderingCABI::relaxed:
440 // 31.7.2.18: "The failure argument shall not be memory_order_release
441 // nor memory_order_acq_rel". Fallback to monotonic.
442 case llvm::AtomicOrderingCABI::release:
443 case llvm::AtomicOrderingCABI::acq_rel:
444 FailureOrder = llvm::AtomicOrdering::Monotonic;
445 break;
446 case llvm::AtomicOrderingCABI::consume:
447 case llvm::AtomicOrderingCABI::acquire:
448 FailureOrder = llvm::AtomicOrdering::Acquire;
449 break;
450 case llvm::AtomicOrderingCABI::seq_cst:
451 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
452 break;
453 }
454 // Prior to c++17, "the failure argument shall be no stronger than the
455 // success argument". This condition has been lifted and the only
456 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
457 // language version checks.
458 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
459 FailureOrder, Scope);
460 return;
461 }
462
463 // Create all the relevant BB's
464 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
465 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
466 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
467 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
468
469 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
470 // doesn't matter unless someone is crazy enough to use something that
471 // doesn't fold to a constant for the ordering.
472 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
473 // Implemented as acquire, since it's the closest in LLVM.
474 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
475 AcquireBB);
476 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
477 AcquireBB);
478 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
479 SeqCstBB);
480
481 // Emit all the different atomics
482 CGF.Builder.SetInsertPoint(MonotonicBB);
483 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
484 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
485 CGF.Builder.CreateBr(ContBB);
486
487 CGF.Builder.SetInsertPoint(AcquireBB);
488 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
489 llvm::AtomicOrdering::Acquire, Scope);
490 CGF.Builder.CreateBr(ContBB);
491
492 CGF.Builder.SetInsertPoint(SeqCstBB);
493 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
494 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
495 CGF.Builder.CreateBr(ContBB);
496
497 CGF.Builder.SetInsertPoint(ContBB);
498}
499
500/// Duplicate the atomic min/max operation in conventional IR for the builtin
501/// variants that return the new rather than the original value.
502static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
503 AtomicExpr::AtomicOp Op,
504 bool IsSigned,
505 llvm::Value *OldVal,
506 llvm::Value *RHS) {
507 llvm::CmpInst::Predicate Pred;
508 switch (Op) {
509 default:
510 llvm_unreachable("Unexpected min/max operation");
511 case AtomicExpr::AO__atomic_max_fetch:
512 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
513 break;
514 case AtomicExpr::AO__atomic_min_fetch:
515 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
516 break;
517 }
518 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
519 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
520}
521
522static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
523 Address Ptr, Address Val1, Address Val2,
524 llvm::Value *IsWeak, llvm::Value *FailureOrder,
525 uint64_t Size, llvm::AtomicOrdering Order,
526 llvm::SyncScope::ID Scope) {
527 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
528 bool PostOpMinMax = false;
529 unsigned PostOp = 0;
530
531 switch (E->getOp()) {
532 case AtomicExpr::AO__c11_atomic_init:
533 case AtomicExpr::AO__opencl_atomic_init:
534 llvm_unreachable("Already handled!");
535
536 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
537 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
538 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
539 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
540 FailureOrder, Size, Order, Scope);
541 return;
542 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
543 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
544 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
545 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
546 FailureOrder, Size, Order, Scope);
547 return;
548 case AtomicExpr::AO__atomic_compare_exchange:
549 case AtomicExpr::AO__atomic_compare_exchange_n: {
550 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
551 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
552 Val1, Val2, FailureOrder, Size, Order, Scope);
553 } else {
554 // Create all the relevant BB's
555 llvm::BasicBlock *StrongBB =
556 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
557 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
558 llvm::BasicBlock *ContBB =
559 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
560
561 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
562 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
563
564 CGF.Builder.SetInsertPoint(StrongBB);
565 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
566 FailureOrder, Size, Order, Scope);
567 CGF.Builder.CreateBr(ContBB);
568
569 CGF.Builder.SetInsertPoint(WeakBB);
570 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
571 FailureOrder, Size, Order, Scope);
572 CGF.Builder.CreateBr(ContBB);
573
574 CGF.Builder.SetInsertPoint(ContBB);
575 }
576 return;
577 }
578 case AtomicExpr::AO__c11_atomic_load:
579 case AtomicExpr::AO__opencl_atomic_load:
580 case AtomicExpr::AO__hip_atomic_load:
581 case AtomicExpr::AO__atomic_load_n:
582 case AtomicExpr::AO__atomic_load: {
583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
584 Load->setAtomic(Order, Scope);
585 Load->setVolatile(E->isVolatile());
586 CGF.Builder.CreateStore(Load, Dest);
587 return;
588 }
589
590 case AtomicExpr::AO__c11_atomic_store:
591 case AtomicExpr::AO__opencl_atomic_store:
592 case AtomicExpr::AO__hip_atomic_store:
593 case AtomicExpr::AO__atomic_store:
594 case AtomicExpr::AO__atomic_store_n: {
595 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
596 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
597 Store->setAtomic(Order, Scope);
598 Store->setVolatile(E->isVolatile());
599 return;
600 }
601
602 case AtomicExpr::AO__c11_atomic_exchange:
603 case AtomicExpr::AO__hip_atomic_exchange:
604 case AtomicExpr::AO__opencl_atomic_exchange:
605 case AtomicExpr::AO__atomic_exchange_n:
606 case AtomicExpr::AO__atomic_exchange:
607 Op = llvm::AtomicRMWInst::Xchg;
608 break;
609
610 case AtomicExpr::AO__atomic_add_fetch:
611 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
612 : llvm::Instruction::Add;
613 [[fallthrough]];
614 case AtomicExpr::AO__c11_atomic_fetch_add:
615 case AtomicExpr::AO__hip_atomic_fetch_add:
616 case AtomicExpr::AO__opencl_atomic_fetch_add:
617 case AtomicExpr::AO__atomic_fetch_add:
618 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
619 : llvm::AtomicRMWInst::Add;
620 break;
621
622 case AtomicExpr::AO__atomic_sub_fetch:
623 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
624 : llvm::Instruction::Sub;
625 [[fallthrough]];
626 case AtomicExpr::AO__c11_atomic_fetch_sub:
627 case AtomicExpr::AO__hip_atomic_fetch_sub:
628 case AtomicExpr::AO__opencl_atomic_fetch_sub:
629 case AtomicExpr::AO__atomic_fetch_sub:
630 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
631 : llvm::AtomicRMWInst::Sub;
632 break;
633
634 case AtomicExpr::AO__atomic_min_fetch:
635 PostOpMinMax = true;
636 [[fallthrough]];
637 case AtomicExpr::AO__c11_atomic_fetch_min:
638 case AtomicExpr::AO__hip_atomic_fetch_min:
639 case AtomicExpr::AO__opencl_atomic_fetch_min:
640 case AtomicExpr::AO__atomic_fetch_min:
641 Op = E->getValueType()->isFloatingType()
642 ? llvm::AtomicRMWInst::FMin
643 : (E->getValueType()->isSignedIntegerType()
644 ? llvm::AtomicRMWInst::Min
645 : llvm::AtomicRMWInst::UMin);
646 break;
647
648 case AtomicExpr::AO__atomic_max_fetch:
649 PostOpMinMax = true;
650 [[fallthrough]];
651 case AtomicExpr::AO__c11_atomic_fetch_max:
652 case AtomicExpr::AO__hip_atomic_fetch_max:
653 case AtomicExpr::AO__opencl_atomic_fetch_max:
654 case AtomicExpr::AO__atomic_fetch_max:
655 Op = E->getValueType()->isFloatingType()
656 ? llvm::AtomicRMWInst::FMax
657 : (E->getValueType()->isSignedIntegerType()
658 ? llvm::AtomicRMWInst::Max
659 : llvm::AtomicRMWInst::UMax);
660 break;
661
662 case AtomicExpr::AO__atomic_and_fetch:
663 PostOp = llvm::Instruction::And;
664 [[fallthrough]];
665 case AtomicExpr::AO__c11_atomic_fetch_and:
666 case AtomicExpr::AO__hip_atomic_fetch_and:
667 case AtomicExpr::AO__opencl_atomic_fetch_and:
668 case AtomicExpr::AO__atomic_fetch_and:
669 Op = llvm::AtomicRMWInst::And;
670 break;
671
672 case AtomicExpr::AO__atomic_or_fetch:
673 PostOp = llvm::Instruction::Or;
674 [[fallthrough]];
675 case AtomicExpr::AO__c11_atomic_fetch_or:
676 case AtomicExpr::AO__hip_atomic_fetch_or:
677 case AtomicExpr::AO__opencl_atomic_fetch_or:
678 case AtomicExpr::AO__atomic_fetch_or:
679 Op = llvm::AtomicRMWInst::Or;
680 break;
681
682 case AtomicExpr::AO__atomic_xor_fetch:
683 PostOp = llvm::Instruction::Xor;
684 [[fallthrough]];
685 case AtomicExpr::AO__c11_atomic_fetch_xor:
686 case AtomicExpr::AO__hip_atomic_fetch_xor:
687 case AtomicExpr::AO__opencl_atomic_fetch_xor:
688 case AtomicExpr::AO__atomic_fetch_xor:
689 Op = llvm::AtomicRMWInst::Xor;
690 break;
691
692 case AtomicExpr::AO__atomic_nand_fetch:
693 PostOp = llvm::Instruction::And; // the NOT is special cased below
694 [[fallthrough]];
695 case AtomicExpr::AO__c11_atomic_fetch_nand:
696 case AtomicExpr::AO__atomic_fetch_nand:
697 Op = llvm::AtomicRMWInst::Nand;
698 break;
699 }
700
701 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
702 llvm::AtomicRMWInst *RMWI =
703 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
704 RMWI->setVolatile(E->isVolatile());
705
706 // For __atomic_*_fetch operations, perform the operation again to
707 // determine the value which was written.
708 llvm::Value *Result = RMWI;
709 if (PostOpMinMax)
710 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
711 E->getValueType()->isSignedIntegerType(),
712 RMWI, LoadVal1);
713 else if (PostOp)
714 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
715 LoadVal1);
716 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
717 Result = CGF.Builder.CreateNot(Result);
718 CGF.Builder.CreateStore(Result, Dest);
719}
720
721// This function emits any expression (scalar, complex, or aggregate)
722// into a temporary alloca.
723static Address
724EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
725 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
726 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
727 /*Init*/ true);
728 return DeclPtr;
729}
730
731static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
732 Address Ptr, Address Val1, Address Val2,
733 llvm::Value *IsWeak, llvm::Value *FailureOrder,
734 uint64_t Size, llvm::AtomicOrdering Order,
735 llvm::Value *Scope) {
736 auto ScopeModel = Expr->getScopeModel();
737
738 // LLVM atomic instructions always have synch scope. If clang atomic
739 // expression has no scope operand, use default LLVM synch scope.
740 if (!ScopeModel) {
741 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
742 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
743 return;
744 }
745
746 // Handle constant scope.
747 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
748 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
749 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
750 Order, CGF.CGM.getLLVMContext());
751 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
752 Order, SCID);
753 return;
754 }
755
756 // Handle non-constant scope.
757 auto &Builder = CGF.Builder;
758 auto Scopes = ScopeModel->getRuntimeValues();
759 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
760 for (auto S : Scopes)
761 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
762
763 llvm::BasicBlock *ContBB =
764 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
765
766 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
767 // If unsupported synch scope is encountered at run time, assume a fallback
768 // synch scope value.
769 auto FallBack = ScopeModel->getFallBackValue();
770 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
771 for (auto S : Scopes) {
772 auto *B = BB[S];
773 if (S != FallBack)
774 SI->addCase(Builder.getInt32(S), B);
775
776 Builder.SetInsertPoint(B);
777 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
778 Order,
779 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
780 ScopeModel->map(S),
781 Order,
782 CGF.getLLVMContext()));
783 Builder.CreateBr(ContBB);
784 }
785
786 Builder.SetInsertPoint(ContBB);
787}
788
789static void
790AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
791 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
792 SourceLocation Loc, CharUnits SizeInChars) {
793 if (UseOptimizedLibcall) {
794 // Load value and pass it to the function directly.
795 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
796 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
797 ValTy =
798 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
799 llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
800 Address Ptr = Address(Val, ITy, Align);
801 Val = CGF.EmitLoadOfScalar(Ptr, false,
802 CGF.getContext().getPointerType(ValTy),
803 Loc);
804 // Coerce the value into an appropriately sized integer type.
805 Args.add(RValue::get(Val), ValTy);
806 } else {
807 // Non-optimized functions always take a reference.
808 Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy);
809 }
810}
811
812RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
813 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
814 QualType MemTy = AtomicTy;
815 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
816 MemTy = AT->getValueType();
817 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
818
819 Address Val1 = Address::invalid();
820 Address Val2 = Address::invalid();
821 Address Dest = Address::invalid();
822 Address Ptr = EmitPointerWithAlignment(E->getPtr());
823
824 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
825 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
826 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
827 EmitAtomicInit(E->getVal1(), lvalue);
828 return RValue::get(nullptr);
829 }
830
831 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
832 uint64_t Size = TInfo.Width.getQuantity();
833 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
834
835 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
836 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
837 bool UseLibcall = Misaligned | Oversized;
838 bool ShouldCastToIntPtrTy = true;
839
840 CharUnits MaxInlineWidth =
841 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
842
843 DiagnosticsEngine &Diags = CGM.getDiags();
844
845 if (Misaligned) {
846 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
847 << (int)TInfo.Width.getQuantity()
848 << (int)Ptr.getAlignment().getQuantity();
849 }
850
851 if (Oversized) {
852 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
853 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
854 }
855
856 llvm::Value *Order = EmitScalarExpr(E->getOrder());
857 llvm::Value *Scope =
858 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
859
860 switch (E->getOp()) {
861 case AtomicExpr::AO__c11_atomic_init:
862 case AtomicExpr::AO__opencl_atomic_init:
863 llvm_unreachable("Already handled above with EmitAtomicInit!");
864
865 case AtomicExpr::AO__c11_atomic_load:
866 case AtomicExpr::AO__opencl_atomic_load:
867 case AtomicExpr::AO__hip_atomic_load:
868 case AtomicExpr::AO__atomic_load_n:
869 break;
870
871 case AtomicExpr::AO__atomic_load:
872 Dest = EmitPointerWithAlignment(E->getVal1());
873 break;
874
875 case AtomicExpr::AO__atomic_store:
876 Val1 = EmitPointerWithAlignment(E->getVal1());
877 break;
878
879 case AtomicExpr::AO__atomic_exchange:
880 Val1 = EmitPointerWithAlignment(E->getVal1());
881 Dest = EmitPointerWithAlignment(E->getVal2());
882 break;
883
884 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
885 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
886 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
887 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
888 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
889 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
890 case AtomicExpr::AO__atomic_compare_exchange_n:
891 case AtomicExpr::AO__atomic_compare_exchange:
892 Val1 = EmitPointerWithAlignment(E->getVal1());
893 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
894 Val2 = EmitPointerWithAlignment(E->getVal2());
895 else
896 Val2 = EmitValToTemp(*this, E->getVal2());
897 OrderFail = EmitScalarExpr(E->getOrderFail());
898 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
899 E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
900 IsWeak = EmitScalarExpr(E->getWeak());
901 break;
902
903 case AtomicExpr::AO__c11_atomic_fetch_add:
904 case AtomicExpr::AO__c11_atomic_fetch_sub:
905 case AtomicExpr::AO__hip_atomic_fetch_add:
906 case AtomicExpr::AO__hip_atomic_fetch_sub:
907 case AtomicExpr::AO__opencl_atomic_fetch_add:
908 case AtomicExpr::AO__opencl_atomic_fetch_sub:
909 if (MemTy->isPointerType()) {
910 // For pointer arithmetic, we're required to do a bit of math:
911 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
912 // ... but only for the C11 builtins. The GNU builtins expect the
913 // user to multiply by sizeof(T).
914 QualType Val1Ty = E->getVal1()->getType();
915 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
916 CharUnits PointeeIncAmt =
917 getContext().getTypeSizeInChars(MemTy->getPointeeType());
918 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
919 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
920 Val1 = Temp;
921 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
922 break;
923 }
924 [[fallthrough]];
925 case AtomicExpr::AO__atomic_fetch_add:
926 case AtomicExpr::AO__atomic_fetch_max:
927 case AtomicExpr::AO__atomic_fetch_min:
928 case AtomicExpr::AO__atomic_fetch_sub:
929 case AtomicExpr::AO__atomic_add_fetch:
930 case AtomicExpr::AO__atomic_max_fetch:
931 case AtomicExpr::AO__atomic_min_fetch:
932 case AtomicExpr::AO__atomic_sub_fetch:
933 case AtomicExpr::AO__c11_atomic_fetch_max:
934 case AtomicExpr::AO__c11_atomic_fetch_min:
935 case AtomicExpr::AO__opencl_atomic_fetch_max:
936 case AtomicExpr::AO__opencl_atomic_fetch_min:
937 case AtomicExpr::AO__hip_atomic_fetch_max:
938 case AtomicExpr::AO__hip_atomic_fetch_min:
939 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
940 [[fallthrough]];
941
942 case AtomicExpr::AO__c11_atomic_store:
943 case AtomicExpr::AO__c11_atomic_exchange:
944 case AtomicExpr::AO__opencl_atomic_store:
945 case AtomicExpr::AO__hip_atomic_store:
946 case AtomicExpr::AO__opencl_atomic_exchange:
947 case AtomicExpr::AO__hip_atomic_exchange:
948 case AtomicExpr::AO__atomic_store_n:
949 case AtomicExpr::AO__atomic_exchange_n:
950 case AtomicExpr::AO__c11_atomic_fetch_and:
951 case AtomicExpr::AO__c11_atomic_fetch_or:
952 case AtomicExpr::AO__c11_atomic_fetch_xor:
953 case AtomicExpr::AO__c11_atomic_fetch_nand:
954 case AtomicExpr::AO__opencl_atomic_fetch_and:
955 case AtomicExpr::AO__opencl_atomic_fetch_or:
956 case AtomicExpr::AO__opencl_atomic_fetch_xor:
957 case AtomicExpr::AO__atomic_fetch_and:
958 case AtomicExpr::AO__hip_atomic_fetch_and:
959 case AtomicExpr::AO__atomic_fetch_or:
960 case AtomicExpr::AO__hip_atomic_fetch_or:
961 case AtomicExpr::AO__atomic_fetch_xor:
962 case AtomicExpr::AO__hip_atomic_fetch_xor:
963 case AtomicExpr::AO__atomic_fetch_nand:
964 case AtomicExpr::AO__atomic_and_fetch:
965 case AtomicExpr::AO__atomic_or_fetch:
966 case AtomicExpr::AO__atomic_xor_fetch:
967 case AtomicExpr::AO__atomic_nand_fetch:
968 Val1 = EmitValToTemp(*this, E->getVal1());
969 break;
970 }
971
972 QualType RValTy = E->getType().getUnqualifiedType();
973
974 // The inlined atomics only function on iN types, where N is a power of 2. We
975 // need to make sure (via temporaries if necessary) that all incoming values
976 // are compatible.
977 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
978 AtomicInfo Atomics(*this, AtomicVal);
979
980 if (ShouldCastToIntPtrTy) {
981 Ptr = Atomics.castToAtomicIntPointer(Ptr);
982 if (Val1.isValid())
983 Val1 = Atomics.convertToAtomicIntPointer(Val1);
984 if (Val2.isValid())
985 Val2 = Atomics.convertToAtomicIntPointer(Val2);
986 }
987 if (Dest.isValid()) {
988 if (ShouldCastToIntPtrTy)
989 Dest = Atomics.castToAtomicIntPointer(Dest);
990 } else if (E->isCmpXChg())
991 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
992 else if (!RValTy->isVoidType()) {
993 Dest = Atomics.CreateTempAlloca();
994 if (ShouldCastToIntPtrTy)
995 Dest = Atomics.castToAtomicIntPointer(Dest);
996 }
997
998 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
999 if (UseLibcall) {
1000 bool UseOptimizedLibcall = false;
1001 switch (E->getOp()) {
1002 case AtomicExpr::AO__c11_atomic_init:
1003 case AtomicExpr::AO__opencl_atomic_init:
1004 llvm_unreachable("Already handled above with EmitAtomicInit!");
1005
1006 case AtomicExpr::AO__c11_atomic_fetch_add:
1007 case AtomicExpr::AO__opencl_atomic_fetch_add:
1008 case AtomicExpr::AO__atomic_fetch_add:
1009 case AtomicExpr::AO__hip_atomic_fetch_add:
1010 case AtomicExpr::AO__c11_atomic_fetch_and:
1011 case AtomicExpr::AO__opencl_atomic_fetch_and:
1012 case AtomicExpr::AO__hip_atomic_fetch_and:
1013 case AtomicExpr::AO__atomic_fetch_and:
1014 case AtomicExpr::AO__c11_atomic_fetch_or:
1015 case AtomicExpr::AO__opencl_atomic_fetch_or:
1016 case AtomicExpr::AO__hip_atomic_fetch_or:
1017 case AtomicExpr::AO__atomic_fetch_or:
1018 case AtomicExpr::AO__c11_atomic_fetch_nand:
1019 case AtomicExpr::AO__atomic_fetch_nand:
1020 case AtomicExpr::AO__c11_atomic_fetch_sub:
1021 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1022 case AtomicExpr::AO__atomic_fetch_sub:
1023 case AtomicExpr::AO__hip_atomic_fetch_sub:
1024 case AtomicExpr::AO__c11_atomic_fetch_xor:
1025 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1026 case AtomicExpr::AO__opencl_atomic_fetch_min:
1027 case AtomicExpr::AO__opencl_atomic_fetch_max:
1028 case AtomicExpr::AO__atomic_fetch_xor:
1029 case AtomicExpr::AO__hip_atomic_fetch_xor:
1030 case AtomicExpr::AO__c11_atomic_fetch_max:
1031 case AtomicExpr::AO__c11_atomic_fetch_min:
1032 case AtomicExpr::AO__atomic_add_fetch:
1033 case AtomicExpr::AO__atomic_and_fetch:
1034 case AtomicExpr::AO__atomic_nand_fetch:
1035 case AtomicExpr::AO__atomic_or_fetch:
1036 case AtomicExpr::AO__atomic_sub_fetch:
1037 case AtomicExpr::AO__atomic_xor_fetch:
1038 case AtomicExpr::AO__atomic_fetch_max:
1039 case AtomicExpr::AO__hip_atomic_fetch_max:
1040 case AtomicExpr::AO__atomic_fetch_min:
1041 case AtomicExpr::AO__hip_atomic_fetch_min:
1042 case AtomicExpr::AO__atomic_max_fetch:
1043 case AtomicExpr::AO__atomic_min_fetch:
1044 // For these, only library calls for certain sizes exist.
1045 UseOptimizedLibcall = true;
1046 break;
1047
1048 case AtomicExpr::AO__atomic_load:
1049 case AtomicExpr::AO__atomic_store:
1050 case AtomicExpr::AO__atomic_exchange:
1051 case AtomicExpr::AO__atomic_compare_exchange:
1052 // Use the generic version if we don't know that the operand will be
1053 // suitably aligned for the optimized version.
1054 if (Misaligned)
1055 break;
1056 [[fallthrough]];
1057 case AtomicExpr::AO__c11_atomic_load:
1058 case AtomicExpr::AO__c11_atomic_store:
1059 case AtomicExpr::AO__c11_atomic_exchange:
1060 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1061 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1062 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1063 case AtomicExpr::AO__opencl_atomic_load:
1064 case AtomicExpr::AO__hip_atomic_load:
1065 case AtomicExpr::AO__opencl_atomic_store:
1066 case AtomicExpr::AO__hip_atomic_store:
1067 case AtomicExpr::AO__opencl_atomic_exchange:
1068 case AtomicExpr::AO__hip_atomic_exchange:
1069 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1070 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1071 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1072 case AtomicExpr::AO__atomic_load_n:
1073 case AtomicExpr::AO__atomic_store_n:
1074 case AtomicExpr::AO__atomic_exchange_n:
1075 case AtomicExpr::AO__atomic_compare_exchange_n:
1076 // Only use optimized library calls for sizes for which they exist.
1077 // FIXME: Size == 16 optimized library functions exist too.
1078 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1079 UseOptimizedLibcall = true;
1080 break;
1081 }
1082
1083 CallArgList Args;
1084 if (!UseOptimizedLibcall) {
1085 // For non-optimized library calls, the size is the first parameter
1086 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1087 getContext().getSizeType());
1088 }
1089 // Atomic address is the first or second parameter
1090 // The OpenCL atomic library functions only accept pointer arguments to
1091 // generic address space.
1092 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1093 if (!E->isOpenCL())
1094 return V;
1095 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1096 if (AS == LangAS::opencl_generic)
1097 return V;
1098 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1099 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);
1100
1101 return getTargetHooks().performAddrSpaceCast(
1102 *this, V, AS, LangAS::opencl_generic, DestType, false);
1103 };
1104
1105 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(),
1106 E->getPtr()->getType())),
1107 getContext().VoidPtrTy);
1108
1109 std::string LibCallName;
1110 QualType LoweredMemTy =
1111 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1112 QualType RetTy;
1113 bool HaveRetTy = false;
1114 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1115 bool PostOpMinMax = false;
1116 switch (E->getOp()) {
1117 case AtomicExpr::AO__c11_atomic_init:
1118 case AtomicExpr::AO__opencl_atomic_init:
1119 llvm_unreachable("Already handled!");
1120
1121 // There is only one libcall for compare an exchange, because there is no
1122 // optimisation benefit possible from a libcall version of a weak compare
1123 // and exchange.
1124 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1125 // void *desired, int success, int failure)
1126 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1127 // int success, int failure)
1128 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1129 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1130 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1131 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1132 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1133 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1134 case AtomicExpr::AO__atomic_compare_exchange:
1135 case AtomicExpr::AO__atomic_compare_exchange_n:
1136 LibCallName = "__atomic_compare_exchange";
1137 RetTy = getContext().BoolTy;
1138 HaveRetTy = true;
1139 Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
1140 E->getVal1()->getType())),
1141 getContext().VoidPtrTy);
1142 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1143 MemTy, E->getExprLoc(), TInfo.Width);
1144 Args.add(RValue::get(Order), getContext().IntTy);
1145 Order = OrderFail;
1146 break;
1147 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1148 // int order)
1149 // T __atomic_exchange_N(T *mem, T val, int order)
1150 case AtomicExpr::AO__c11_atomic_exchange:
1151 case AtomicExpr::AO__opencl_atomic_exchange:
1152 case AtomicExpr::AO__atomic_exchange_n:
1153 case AtomicExpr::AO__atomic_exchange:
1154 case AtomicExpr::AO__hip_atomic_exchange:
1155 LibCallName = "__atomic_exchange";
1156 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1157 MemTy, E->getExprLoc(), TInfo.Width);
1158 break;
1159 // void __atomic_store(size_t size, void *mem, void *val, int order)
1160 // void __atomic_store_N(T *mem, T val, int order)
1161 case AtomicExpr::AO__c11_atomic_store:
1162 case AtomicExpr::AO__opencl_atomic_store:
1163 case AtomicExpr::AO__hip_atomic_store:
1164 case AtomicExpr::AO__atomic_store:
1165 case AtomicExpr::AO__atomic_store_n:
1166 LibCallName = "__atomic_store";
1167 RetTy = getContext().VoidTy;
1168 HaveRetTy = true;
1169 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1170 MemTy, E->getExprLoc(), TInfo.Width);
1171 break;
1172 // void __atomic_load(size_t size, void *mem, void *return, int order)
1173 // T __atomic_load_N(T *mem, int order)
1174 case AtomicExpr::AO__c11_atomic_load:
1175 case AtomicExpr::AO__opencl_atomic_load:
1176 case AtomicExpr::AO__hip_atomic_load:
1177 case AtomicExpr::AO__atomic_load:
1178 case AtomicExpr::AO__atomic_load_n:
1179 LibCallName = "__atomic_load";
1180 break;
1181 // T __atomic_add_fetch_N(T *mem, T val, int order)
1182 // T __atomic_fetch_add_N(T *mem, T val, int order)
1183 case AtomicExpr::AO__atomic_add_fetch:
1184 PostOp = llvm::Instruction::Add;
1185 [[fallthrough]];
1186 case AtomicExpr::AO__c11_atomic_fetch_add:
1187 case AtomicExpr::AO__opencl_atomic_fetch_add:
1188 case AtomicExpr::AO__atomic_fetch_add:
1189 case AtomicExpr::AO__hip_atomic_fetch_add:
1190 LibCallName = "__atomic_fetch_add";
1191 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1192 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1193 break;
1194 // T __atomic_and_fetch_N(T *mem, T val, int order)
1195 // T __atomic_fetch_and_N(T *mem, T val, int order)
1196 case AtomicExpr::AO__atomic_and_fetch:
1197 PostOp = llvm::Instruction::And;
1198 [[fallthrough]];
1199 case AtomicExpr::AO__c11_atomic_fetch_and:
1200 case AtomicExpr::AO__opencl_atomic_fetch_and:
1201 case AtomicExpr::AO__hip_atomic_fetch_and:
1202 case AtomicExpr::AO__atomic_fetch_and:
1203 LibCallName = "__atomic_fetch_and";
1204 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1205 MemTy, E->getExprLoc(), TInfo.Width);
1206 break;
1207 // T __atomic_or_fetch_N(T *mem, T val, int order)
1208 // T __atomic_fetch_or_N(T *mem, T val, int order)
1209 case AtomicExpr::AO__atomic_or_fetch:
1210 PostOp = llvm::Instruction::Or;
1211 [[fallthrough]];
1212 case AtomicExpr::AO__c11_atomic_fetch_or:
1213 case AtomicExpr::AO__opencl_atomic_fetch_or:
1214 case AtomicExpr::AO__hip_atomic_fetch_or:
1215 case AtomicExpr::AO__atomic_fetch_or:
1216 LibCallName = "__atomic_fetch_or";
1217 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1218 MemTy, E->getExprLoc(), TInfo.Width);
1219 break;
1220 // T __atomic_sub_fetch_N(T *mem, T val, int order)
1221 // T __atomic_fetch_sub_N(T *mem, T val, int order)
1222 case AtomicExpr::AO__atomic_sub_fetch:
1223 PostOp = llvm::Instruction::Sub;
1224 [[fallthrough]];
1225 case AtomicExpr::AO__c11_atomic_fetch_sub:
1226 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1227 case AtomicExpr::AO__hip_atomic_fetch_sub:
1228 case AtomicExpr::AO__atomic_fetch_sub:
1229 LibCallName = "__atomic_fetch_sub";
1230 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1231 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1232 break;
1233 // T __atomic_xor_fetch_N(T *mem, T val, int order)
1234 // T __atomic_fetch_xor_N(T *mem, T val, int order)
1235 case AtomicExpr::AO__atomic_xor_fetch:
1236 PostOp = llvm::Instruction::Xor;
1237 [[fallthrough]];
1238 case AtomicExpr::AO__c11_atomic_fetch_xor:
1239 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1240 case AtomicExpr::AO__hip_atomic_fetch_xor:
1241 case AtomicExpr::AO__atomic_fetch_xor:
1242 LibCallName = "__atomic_fetch_xor";
1243 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1244 MemTy, E->getExprLoc(), TInfo.Width);
1245 break;
1246 case AtomicExpr::AO__atomic_min_fetch:
1247 PostOpMinMax = true;
1248 [[fallthrough]];
1249 case AtomicExpr::AO__c11_atomic_fetch_min:
1250 case AtomicExpr::AO__atomic_fetch_min:
1251 case AtomicExpr::AO__hip_atomic_fetch_min:
1252 case AtomicExpr::AO__opencl_atomic_fetch_min:
1253 LibCallName = E->getValueType()->isSignedIntegerType()
1254 ? "__atomic_fetch_min"
1255 : "__atomic_fetch_umin";
1256 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1257 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1258 break;
1259 case AtomicExpr::AO__atomic_max_fetch:
1260 PostOpMinMax = true;
1261 [[fallthrough]];
1262 case AtomicExpr::AO__c11_atomic_fetch_max:
1263 case AtomicExpr::AO__atomic_fetch_max:
1264 case AtomicExpr::AO__hip_atomic_fetch_max:
1265 case AtomicExpr::AO__opencl_atomic_fetch_max:
1266 LibCallName = E->getValueType()->isSignedIntegerType()
1267 ? "__atomic_fetch_max"
1268 : "__atomic_fetch_umax";
1269 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1270 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1271 break;
1272 // T __atomic_nand_fetch_N(T *mem, T val, int order)
1273 // T __atomic_fetch_nand_N(T *mem, T val, int order)
1274 case AtomicExpr::AO__atomic_nand_fetch:
1275 PostOp = llvm::Instruction::And; // the NOT is special cased below
1276 [[fallthrough]];
1277 case AtomicExpr::AO__c11_atomic_fetch_nand:
1278 case AtomicExpr::AO__atomic_fetch_nand:
1279 LibCallName = "__atomic_fetch_nand";
1280 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1281 MemTy, E->getExprLoc(), TInfo.Width);
1282 break;
1283 }
1284
1285 if (E->isOpenCL()) {
1286 LibCallName = std::string("__opencl") +
1287 StringRef(LibCallName).drop_front(1).str();
1288
1289 }
1290 // Optimized functions have the size in their name.
1291 if (UseOptimizedLibcall)
1292 LibCallName += "_" + llvm::utostr(Size);
1293 // By default, assume we return a value of the atomic type.
1294 if (!HaveRetTy) {
1295 if (UseOptimizedLibcall) {
1296 // Value is returned directly.
1297 // The function returns an appropriately sized integer type.
1298 RetTy = getContext().getIntTypeForBitwidth(
1299 getContext().toBits(TInfo.Width), /*Signed=*/false);
1300 } else {
1301 // Value is returned through parameter before the order.
1302 RetTy = getContext().VoidTy;
1303 Args.add(RValue::get(Dest.getPointer()), getContext().VoidPtrTy);
1304 }
1305 }
1306 // order is always the last parameter
1307 Args.add(RValue::get(Order),
1308 getContext().IntTy);
1309 if (E->isOpenCL())
1310 Args.add(RValue::get(Scope), getContext().IntTy);
1311
1312 // PostOp is only needed for the atomic_*_fetch operations, and
1313 // thus is only needed for and implemented in the
1314 // UseOptimizedLibcall codepath.
1315 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1316
1317 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1318 // The value is returned directly from the libcall.
1319 if (E->isCmpXChg())
1320 return Res;
1321
1322 // The value is returned directly for optimized libcalls but the expr
1323 // provided an out-param.
1324 if (UseOptimizedLibcall && Res.getScalarVal()) {
1325 llvm::Value *ResVal = Res.getScalarVal();
1326 if (PostOpMinMax) {
1327 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1328 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1329 E->getValueType()->isSignedIntegerType(),
1330 ResVal, LoadVal1);
1331 } else if (PostOp) {
1332 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1333 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1334 }
1335 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1336 ResVal = Builder.CreateNot(ResVal);
1337
1338 Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType()));
1339 }
1340
1341 if (RValTy->isVoidType())
1342 return RValue::get(nullptr);
1343
1344 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1345 RValTy, E->getExprLoc());
1346 }
1347
1348 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1349 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1350 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1351 E->getOp() == AtomicExpr::AO__atomic_store ||
1352 E->getOp() == AtomicExpr::AO__atomic_store_n;
1353 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1354 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1355 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1356 E->getOp() == AtomicExpr::AO__atomic_load ||
1357 E->getOp() == AtomicExpr::AO__atomic_load_n;
1358
1359 if (isa<llvm::ConstantInt>(Order)) {
1360 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1361 // We should not ever get to a case where the ordering isn't a valid C ABI
1362 // value, but it's hard to enforce that in general.
1363 if (llvm::isValidAtomicOrderingCABI(ord))
1364 switch ((llvm::AtomicOrderingCABI)ord) {
1365 case llvm::AtomicOrderingCABI::relaxed:
1366 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1367 llvm::AtomicOrdering::Monotonic, Scope);
1368 break;
1369 case llvm::AtomicOrderingCABI::consume:
1370 case llvm::AtomicOrderingCABI::acquire:
1371 if (IsStore)
1372 break; // Avoid crashing on code with undefined behavior
1373 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1374 llvm::AtomicOrdering::Acquire, Scope);
1375 break;
1376 case llvm::AtomicOrderingCABI::release:
1377 if (IsLoad)
1378 break; // Avoid crashing on code with undefined behavior
1379 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1380 llvm::AtomicOrdering::Release, Scope);
1381 break;
1382 case llvm::AtomicOrderingCABI::acq_rel:
1383 if (IsLoad || IsStore)
1384 break; // Avoid crashing on code with undefined behavior
1385 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386 llvm::AtomicOrdering::AcquireRelease, Scope);
1387 break;
1388 case llvm::AtomicOrderingCABI::seq_cst:
1389 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1390 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1391 break;
1392 }
1393 if (RValTy->isVoidType())
1394 return RValue::get(nullptr);
1395
1396 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1397 RValTy, E->getExprLoc());
1398 }
1399
1400 // Long case, when Order isn't obviously constant.
1401
1402 // Create all the relevant BB's
1403 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1404 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1405 *SeqCstBB = nullptr;
1406 MonotonicBB = createBasicBlock("monotonic", CurFn);
1407 if (!IsStore)
1408 AcquireBB = createBasicBlock("acquire", CurFn);
1409 if (!IsLoad)
1410 ReleaseBB = createBasicBlock("release", CurFn);
1411 if (!IsLoad && !IsStore)
1412 AcqRelBB = createBasicBlock("acqrel", CurFn);
1413 SeqCstBB = createBasicBlock("seqcst", CurFn);
1414 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1415
1416 // Create the switch for the split
1417 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1418 // doesn't matter unless someone is crazy enough to use something that
1419 // doesn't fold to a constant for the ordering.
1420 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1421 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1422
1423 // Emit all the different atomics
1424 Builder.SetInsertPoint(MonotonicBB);
1425 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1426 llvm::AtomicOrdering::Monotonic, Scope);
1427 Builder.CreateBr(ContBB);
1428 if (!IsStore) {
1429 Builder.SetInsertPoint(AcquireBB);
1430 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1431 llvm::AtomicOrdering::Acquire, Scope);
1432 Builder.CreateBr(ContBB);
1433 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1434 AcquireBB);
1435 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1436 AcquireBB);
1437 }
1438 if (!IsLoad) {
1439 Builder.SetInsertPoint(ReleaseBB);
1440 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1441 llvm::AtomicOrdering::Release, Scope);
1442 Builder.CreateBr(ContBB);
1443 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1444 ReleaseBB);
1445 }
1446 if (!IsLoad && !IsStore) {
1447 Builder.SetInsertPoint(AcqRelBB);
1448 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1449 llvm::AtomicOrdering::AcquireRelease, Scope);
1450 Builder.CreateBr(ContBB);
1451 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1452 AcqRelBB);
1453 }
1454 Builder.SetInsertPoint(SeqCstBB);
1455 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1456 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1457 Builder.CreateBr(ContBB);
1458 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1459 SeqCstBB);
1460
1461 // Cleanup and return
1462 Builder.SetInsertPoint(ContBB);
1463 if (RValTy->isVoidType())
1464 return RValue::get(nullptr);
1465
1466 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1467 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1468 RValTy, E->getExprLoc());
1469}
1470
1471Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1472 llvm::IntegerType *ty =
1473 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1474 return addr.withElementType(ty);
1475}
1476
1477Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1478 llvm::Type *Ty = Addr.getElementType();
1479 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1480 if (SourceSizeInBits != AtomicSizeInBits) {
1481 Address Tmp = CreateTempAlloca();
1482 CGF.Builder.CreateMemCpy(Tmp, Addr,
1483 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1484 Addr = Tmp;
1485 }
1486
1487 return castToAtomicIntPointer(Addr);
1488}
1489
1490RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1491 AggValueSlot resultSlot,
1492 SourceLocation loc,
1493 bool asValue) const {
1494 if (LVal.isSimple()) {
1495 if (EvaluationKind == TEK_Aggregate)
1496 return resultSlot.asRValue();
1497
1498 // Drill into the padding structure if we have one.
1499 if (hasPadding())
1500 addr = CGF.Builder.CreateStructGEP(addr, 0);
1501
1502 // Otherwise, just convert the temporary to an r-value using the
1503 // normal conversion routine.
1504 return CGF.convertTempToRValue(addr, getValueType(), loc);
1505 }
1506 if (!asValue)
1507 // Get RValue from temp memory as atomic for non-simple lvalues
1508 return RValue::get(CGF.Builder.CreateLoad(addr));
1509 if (LVal.isBitField())
1510 return CGF.EmitLoadOfBitfieldLValue(
1511 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1512 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1513 if (LVal.isVectorElt())
1514 return CGF.EmitLoadOfLValue(
1515 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1516 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1517 assert(LVal.isExtVectorElt());
1518 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1519 addr, LVal.getExtVectorElts(), LVal.getType(),
1520 LVal.getBaseInfo(), TBAAAccessInfo()));
1521}
1522
1523RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1524 AggValueSlot ResultSlot,
1525 SourceLocation Loc,
1526 bool AsValue) const {
1527 // Try not to in some easy cases.
1528 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1529 if (getEvaluationKind() == TEK_Scalar &&
1530 (((!LVal.isBitField() ||
1531 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1532 !hasPadding()) ||
1533 !AsValue)) {
1534 auto *ValTy = AsValue
1535 ? CGF.ConvertTypeForMem(ValueTy)
1536 : getAtomicAddress().getElementType();
1537 if (ValTy->isIntegerTy()) {
1538 assert(IntVal->getType() == ValTy && "Different integer types.");
1539 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1540 } else if (ValTy->isPointerTy())
1541 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1542 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1543 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1544 }
1545
1546 // Create a temporary. This needs to be big enough to hold the
1547 // atomic integer.
1548 Address Temp = Address::invalid();
1549 bool TempIsVolatile = false;
1550 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1551 assert(!ResultSlot.isIgnored());
1552 Temp = ResultSlot.getAddress();
1553 TempIsVolatile = ResultSlot.isVolatile();
1554 } else {
1555 Temp = CreateTempAlloca();
1556 }
1557
1558 // Slam the integer into the temporary.
1559 Address CastTemp = castToAtomicIntPointer(Temp);
1560 CGF.Builder.CreateStore(IntVal, CastTemp)
1561 ->setVolatile(TempIsVolatile);
1562
1563 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1564}
1565
1566void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1567 llvm::AtomicOrdering AO, bool) {
1568 // void __atomic_load(size_t size, void *mem, void *return, int order);
1569 CallArgList Args;
1570 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1571 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1572 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);
1573 Args.add(
1574 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575 CGF.getContext().IntTy);
1576 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577}
1578
1579llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580 bool IsVolatile) {
1581 // Okay, we're doing this natively.
1582 Address Addr = getAtomicAddressAsAtomicIntPointer();
1583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584 Load->setAtomic(AO);
1585
1586 // Other decoration.
1587 if (IsVolatile)
1588 Load->setVolatile(true);
1589 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1590 return Load;
1591}
1592
1593/// An LValue is a candidate for having its loads and stores be made atomic if
1594/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595/// performing such an operation can be performed without a libcall.
1596bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1597 if (!CGM.getLangOpts().MSVolatile) return false;
1598 AtomicInfo AI(*this, LV);
1599 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1600 // An atomic is inline if we don't need to use a libcall.
1601 bool AtomicIsInline = !AI.shouldUseLibcall();
1602 // MSVC doesn't seem to do this for types wider than a pointer.
1603 if (getContext().getTypeSize(LV.getType()) >
1604 getContext().getTypeSize(getContext().getIntPtrType()))
1605 return false;
1606 return IsVolatile && AtomicIsInline;
1607}
1608
1609RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1610 AggValueSlot Slot) {
1611 llvm::AtomicOrdering AO;
1612 bool IsVolatile = LV.isVolatileQualified();
1613 if (LV.getType()->isAtomicType()) {
1614 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615 } else {
1616 AO = llvm::AtomicOrdering::Acquire;
1617 IsVolatile = true;
1618 }
1619 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620}
1621
1622RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623 bool AsValue, llvm::AtomicOrdering AO,
1624 bool IsVolatile) {
1625 // Check whether we should use a library call.
1626 if (shouldUseLibcall()) {
1627 Address TempAddr = Address::invalid();
1628 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1629 assert(getEvaluationKind() == TEK_Aggregate);
1630 TempAddr = ResultSlot.getAddress();
1631 } else
1632 TempAddr = CreateTempAlloca();
1633
1634 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635
1636 // Okay, turn that back into the original value or whole atomic (for
1637 // non-simple lvalues) type.
1638 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639 }
1640
1641 // Okay, we're doing this natively.
1642 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643
1644 // If we're ignoring an aggregate return, don't do anything.
1645 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1646 return RValue::getAggregate(Address::invalid(), false);
1647
1648 // Okay, turn that back into the original value or atomic (for non-simple
1649 // lvalues) type.
1650 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651}
1652
1653/// Emit a load from an l-value of atomic type. Note that the r-value
1654/// we produce is an r-value of the atomic *value* type.
1655RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1656 llvm::AtomicOrdering AO, bool IsVolatile,
1657 AggValueSlot resultSlot) {
1658 AtomicInfo Atomics(*this, src);
1659 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660 IsVolatile);
1661}
1662
1663/// Copy an r-value into memory as part of storing to an atomic type.
1664/// This needs to create a bit-pattern suitable for atomic operations.
1665void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666 assert(LVal.isSimple());
1667 // If we have an r-value, the rvalue should be of the atomic type,
1668 // which means that the caller is responsible for having zeroed
1669 // any padding. Just do an aggregate copy of that type.
1670 if (rvalue.isAggregate()) {
1671 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673 getAtomicType());
1674 bool IsVolatile = rvalue.isVolatileQualified() ||
1675 LVal.isVolatileQualified();
1676 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677 AggValueSlot::DoesNotOverlap, IsVolatile);
1678 return;
1679 }
1680
1681 // Okay, otherwise we're copying stuff.
1682
1683 // Zero out the buffer if necessary.
1684 emitMemSetZeroIfNecessary();
1685
1686 // Drill past the padding if present.
1687 LValue TempLVal = projectValue();
1688
1689 // Okay, store the rvalue in.
1690 if (rvalue.isScalar()) {
1691 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692 } else {
1693 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694 }
1695}
1696
1697
1698/// Materialize an r-value into memory for the purposes of storing it
1699/// to an atomic type.
1700Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701 // Aggregate r-values are already in memory, and EmitAtomicStore
1702 // requires them to be values of the atomic type.
1703 if (rvalue.isAggregate())
1704 return rvalue.getAggregateAddress();
1705
1706 // Otherwise, make a temporary and materialize into it.
1707 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708 AtomicInfo Atomics(CGF, TempLV);
1709 Atomics.emitCopyIntoMemory(rvalue);
1710 return TempLV.getAddress(CGF);
1711}
1712
1713llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714 // If we've got a scalar value of the right size, try to avoid going
1715 // through memory.
1716 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1717 llvm::Value *Value = RVal.getScalarVal();
1718 if (isa<llvm::IntegerType>(Value->getType()))
1719 return CGF.EmitToMemory(Value, ValueTy);
1720 else {
1721 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722 CGF.getLLVMContext(),
1723 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1724 if (isa<llvm::PointerType>(Value->getType()))
1725 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728 }
1729 }
1730 // Otherwise, we need to go through memory.
1731 // Put the r-value in memory.
1732 Address Addr = materializeRValue(RVal);
1733
1734 // Cast the temporary to the atomic int type and pull a value out.
1735 Addr = castToAtomicIntPointer(Addr);
1736 return CGF.Builder.CreateLoad(Addr);
1737}
1738
1739std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742 // Do the atomic store.
1743 Address Addr = getAtomicAddressAsAtomicIntPointer();
1744 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745 ExpectedVal, DesiredVal,
1746 Success, Failure);
1747 // Other decoration.
1748 Inst->setVolatile(LVal.isVolatileQualified());
1749 Inst->setWeak(IsWeak);
1750
1751 // Okay, turn that back into the original value type.
1752 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754 return std::make_pair(PreviousVal, SuccessFailureVal);
1755}
1756
1757llvm::Value *
1758AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759 llvm::Value *DesiredAddr,
1760 llvm::AtomicOrdering Success,
1761 llvm::AtomicOrdering Failure) {
1762 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763 // void *desired, int success, int failure);
1764 CallArgList Args;
1765 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1767 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
1768 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
1769 Args.add(RValue::get(
1770 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1771 CGF.getContext().IntTy);
1772 Args.add(RValue::get(
1773 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1774 CGF.getContext().IntTy);
1775 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1776 CGF.getContext().BoolTy, Args);
1777
1778 return SuccessFailureRVal.getScalarVal();
1779}
1780
1781std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1782 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1783 llvm::AtomicOrdering Failure, bool IsWeak) {
1784 // Check whether we should use a library call.
1785 if (shouldUseLibcall()) {
1786 // Produce a source address.
1787 Address ExpectedAddr = materializeRValue(Expected);
1788 Address DesiredAddr = materializeRValue(Desired);
1789 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1790 DesiredAddr.getPointer(),
1791 Success, Failure);
1792 return std::make_pair(
1793 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1794 SourceLocation(), /*AsValue=*/false),
1795 Res);
1796 }
1797
1798 // If we've got a scalar value of the right size, try to avoid going
1799 // through memory.
1800 auto *ExpectedVal = convertRValueToInt(Expected);
1801 auto *DesiredVal = convertRValueToInt(Desired);
1802 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1803 Failure, IsWeak);
1804 return std::make_pair(
1805 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1806 SourceLocation(), /*AsValue=*/false),
1807 Res.second);
1808}
1809
1810static void
1811EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1812 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1813 Address DesiredAddr) {
1814 RValue UpRVal;
1815 LValue AtomicLVal = Atomics.getAtomicLValue();
1816 LValue DesiredLVal;
1817 if (AtomicLVal.isSimple()) {
1818 UpRVal = OldRVal;
1819 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1820 } else {
1821 // Build new lvalue for temp address.
1822 Address Ptr = Atomics.materializeRValue(OldRVal);
1823 LValue UpdateLVal;
1824 if (AtomicLVal.isBitField()) {
1825 UpdateLVal =
1826 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1827 AtomicLVal.getType(),
1828 AtomicLVal.getBaseInfo(),
1829 AtomicLVal.getTBAAInfo());
1830 DesiredLVal =
1831 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1832 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1833 AtomicLVal.getTBAAInfo());
1834 } else if (AtomicLVal.isVectorElt()) {
1835 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1836 AtomicLVal.getType(),
1837 AtomicLVal.getBaseInfo(),
1838 AtomicLVal.getTBAAInfo());
1839 DesiredLVal = LValue::MakeVectorElt(
1840 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1841 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1842 } else {
1843 assert(AtomicLVal.isExtVectorElt());
1844 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1845 AtomicLVal.getType(),
1846 AtomicLVal.getBaseInfo(),
1847 AtomicLVal.getTBAAInfo());
1848 DesiredLVal = LValue::MakeExtVectorElt(
1849 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1850 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1851 }
1852 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1853 }
1854 // Store new value in the corresponding memory area.
1855 RValue NewRVal = UpdateOp(UpRVal);
1856 if (NewRVal.isScalar()) {
1857 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1858 } else {
1859 assert(NewRVal.isComplex());
1860 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1861 /*isInit=*/false);
1862 }
1863}
1864
1865void AtomicInfo::EmitAtomicUpdateLibcall(
1866 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1867 bool IsVolatile) {
1868 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1869
1870 Address ExpectedAddr = CreateTempAlloca();
1871
1872 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1873 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1874 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1875 CGF.EmitBlock(ContBB);
1876 Address DesiredAddr = CreateTempAlloca();
1877 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1878 requiresMemSetZero(getAtomicAddress().getElementType())) {
1879 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1880 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1881 }
1882 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1883 AggValueSlot::ignored(),
1884 SourceLocation(), /*AsValue=*/false);
1885 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1886 auto *Res =
1887 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1888 DesiredAddr.getPointer(),
1889 AO, Failure);
1890 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1891 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1892}
1893
1894void AtomicInfo::EmitAtomicUpdateOp(
1895 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1896 bool IsVolatile) {
1897 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1898
1899 // Do the atomic load.
1900 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1901 // For non-simple lvalues perform compare-and-swap procedure.
1902 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1903 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1904 auto *CurBB = CGF.Builder.GetInsertBlock();
1905 CGF.EmitBlock(ContBB);
1906 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1907 /*NumReservedValues=*/2);
1908 PHI->addIncoming(OldVal, CurBB);
1909 Address NewAtomicAddr = CreateTempAlloca();
1910 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1911 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1912 requiresMemSetZero(getAtomicAddress().getElementType())) {
1913 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1914 }
1915 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1916 SourceLocation(), /*AsValue=*/false);
1917 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1918 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1919 // Try to write new value using cmpxchg operation.
1920 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1921 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1922 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1923 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1924}
1925
1926static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1927 RValue UpdateRVal, Address DesiredAddr) {
1928 LValue AtomicLVal = Atomics.getAtomicLValue();
1929 LValue DesiredLVal;
1930 // Build new lvalue for temp address.
1931 if (AtomicLVal.isBitField()) {
1932 DesiredLVal =
1933 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1934 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1935 AtomicLVal.getTBAAInfo());
1936 } else if (AtomicLVal.isVectorElt()) {
1937 DesiredLVal =
1938 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1939 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1940 AtomicLVal.getTBAAInfo());
1941 } else {
1942 assert(AtomicLVal.isExtVectorElt());
1943 DesiredLVal = LValue::MakeExtVectorElt(
1944 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1945 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1946 }
1947 // Store new value in the corresponding memory area.
1948 assert(UpdateRVal.isScalar());
1949 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1950}
1951
1952void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1953 RValue UpdateRVal, bool IsVolatile) {
1954 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1955
1956 Address ExpectedAddr = CreateTempAlloca();
1957
1958 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1959 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1960 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1961 CGF.EmitBlock(ContBB);
1962 Address DesiredAddr = CreateTempAlloca();
1963 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1964 requiresMemSetZero(getAtomicAddress().getElementType())) {
1965 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1966 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1967 }
1968 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1969 auto *Res =
1970 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1971 DesiredAddr.getPointer(),
1972 AO, Failure);
1973 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1974 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1975}
1976
1977void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1978 bool IsVolatile) {
1979 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1980
1981 // Do the atomic load.
1982 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1983 // For non-simple lvalues perform compare-and-swap procedure.
1984 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1985 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1986 auto *CurBB = CGF.Builder.GetInsertBlock();
1987 CGF.EmitBlock(ContBB);
1988 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1989 /*NumReservedValues=*/2);
1990 PHI->addIncoming(OldVal, CurBB);
1991 Address NewAtomicAddr = CreateTempAlloca();
1992 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1993 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1994 requiresMemSetZero(getAtomicAddress().getElementType())) {
1995 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1996 }
1997 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1998 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1999 // Try to write new value using cmpxchg operation.
2000 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2001 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2002 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2003 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2004}
2005
2006void AtomicInfo::EmitAtomicUpdate(
2007 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2008 bool IsVolatile) {
2009 if (shouldUseLibcall()) {
2010 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2011 } else {
2012 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2013 }
2014}
2015
2016void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2017 bool IsVolatile) {
2018 if (shouldUseLibcall()) {
2019 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2020 } else {
2021 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2022 }
2023}
2024
2025void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2026 bool isInit) {
2027 bool IsVolatile = lvalue.isVolatileQualified();
2028 llvm::AtomicOrdering AO;
2029 if (lvalue.getType()->isAtomicType()) {
2030 AO = llvm::AtomicOrdering::SequentiallyConsistent;
2031 } else {
2032 AO = llvm::AtomicOrdering::Release;
2033 IsVolatile = true;
2034 }
2035 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2036}
2037
2038/// Emit a store to an l-value of atomic type.
2039///
2040/// Note that the r-value is expected to be an r-value *of the atomic
2041/// type*; this means that for aggregate r-values, it should include
2042/// storage for any padding that was necessary.
2043void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2044 llvm::AtomicOrdering AO, bool IsVolatile,
2045 bool isInit) {
2046 // If this is an aggregate r-value, it should agree in type except
2047 // maybe for address-space qualification.
2048 assert(!rvalue.isAggregate() ||
2049 rvalue.getAggregateAddress().getElementType() ==
2050 dest.getAddress(*this).getElementType());
2051
2052 AtomicInfo atomics(*this, dest);
2053 LValue LVal = atomics.getAtomicLValue();
2054
2055 // If this is an initialization, just put the value there normally.
2056 if (LVal.isSimple()) {
2057 if (isInit) {
2058 atomics.emitCopyIntoMemory(rvalue);
2059 return;
2060 }
2061
2062 // Check whether we should use a library call.
2063 if (atomics.shouldUseLibcall()) {
2064 // Produce a source address.
2065 Address srcAddr = atomics.materializeRValue(rvalue);
2066
2067 // void __atomic_store(size_t size, void *mem, void *val, int order)
2068 CallArgList args;
2069 args.add(RValue::get(atomics.getAtomicSizeValue()),
2070 getContext().getSizeType());
2071 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
2072 args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy);
2073 args.add(
2074 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2075 getContext().IntTy);
2076 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2077 return;
2078 }
2079
2080 // Okay, we're doing this natively.
2081 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2082
2083 // Do the atomic store.
2084 Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress());
2085 intValue = Builder.CreateIntCast(
2086 intValue, addr.getElementType(), /*isSigned=*/false);
2087 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2088
2089 if (AO == llvm::AtomicOrdering::Acquire)
2090 AO = llvm::AtomicOrdering::Monotonic;
2091 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2092 AO = llvm::AtomicOrdering::Release;
2093 // Initializations don't need to be atomic.
2094 if (!isInit)
2095 store->setAtomic(AO);
2096
2097 // Other decoration.
2098 if (IsVolatile)
2099 store->setVolatile(true);
2100 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2101 return;
2102 }
2103
2104 // Emit simple atomic update operation.
2105 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2106}
2107
2108/// Emit a compare-and-exchange op for atomic type.
2109///
2110std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2111 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2112 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2113 AggValueSlot Slot) {
2114 // If this is an aggregate r-value, it should agree in type except
2115 // maybe for address-space qualification.
2116 assert(!Expected.isAggregate() ||
2117 Expected.getAggregateAddress().getElementType() ==
2118 Obj.getAddress(*this).getElementType());
2119 assert(!Desired.isAggregate() ||
2120 Desired.getAggregateAddress().getElementType() ==
2121 Obj.getAddress(*this).getElementType());
2122 AtomicInfo Atomics(*this, Obj);
2123
2124 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2125 IsWeak);
2126}
2127
2128void CodeGenFunction::EmitAtomicUpdate(
2129 LValue LVal, llvm::AtomicOrdering AO,
2130 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2131 AtomicInfo Atomics(*this, LVal);
2132 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2133}
2134
2135void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2136 AtomicInfo atomics(*this, dest);
2137
2138 switch (atomics.getEvaluationKind()) {
2139 case TEK_Scalar: {
2140 llvm::Value *value = EmitScalarExpr(init);
2141 atomics.emitCopyIntoMemory(RValue::get(value));
2142 return;
2143 }
2144
2145 case TEK_Complex: {
2146 ComplexPairTy value = EmitComplexExpr(init);
2147 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2148 return;
2149 }
2150
2151 case TEK_Aggregate: {
2152 // Fix up the destination if the initializer isn't an expression
2153 // of atomic type.
2154 bool Zeroed = false;
2155 if (!init->getType()->isAtomicType()) {
2156 Zeroed = atomics.emitMemSetZeroIfNecessary();
2157 dest = atomics.projectValue();
2158 }
2159
2160 // Evaluate the expression directly into the destination.
2161 AggValueSlot slot = AggValueSlot::forLValue(
2162 dest, *this, AggValueSlot::IsNotDestructed,
2163 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2164 AggValueSlot::DoesNotOverlap,
2165 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2166
2167 EmitAggExpr(init, slot);
2168 return;
2169 }
2170 }
2171 llvm_unreachable("bad evaluation kind");
2172}
2173