1// Copyright (c) 2018 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "upgrade_memory_model.h"
16
17#include <utility>
18
19#include "source/opt/ir_builder.h"
20#include "source/opt/ir_context.h"
21#include "source/spirv_constant.h"
22#include "source/util/make_unique.h"
23
24namespace spvtools {
25namespace opt {
26
27Pass::Status UpgradeMemoryModel::Process() {
28 // TODO: This pass needs changes to support cooperative matrices.
29 if (context()->get_feature_mgr()->HasCapability(
30 SpvCapabilityCooperativeMatrixNV)) {
31 return Pass::Status::SuccessWithoutChange;
32 }
33
34 // Only update Logical GLSL450 to Logical VulkanKHR.
35 Instruction* memory_model = get_module()->GetMemoryModel();
36 if (memory_model->GetSingleWordInOperand(0u) != SpvAddressingModelLogical ||
37 memory_model->GetSingleWordInOperand(1u) != SpvMemoryModelGLSL450) {
38 return Pass::Status::SuccessWithoutChange;
39 }
40
41 UpgradeMemoryModelInstruction();
42 UpgradeInstructions();
43 CleanupDecorations();
44 UpgradeBarriers();
45 UpgradeMemoryScope();
46
47 return Pass::Status::SuccessWithChange;
48}
49
50void UpgradeMemoryModel::UpgradeMemoryModelInstruction() {
51 // Overall changes necessary:
52 // 1. Add the OpExtension.
53 // 2. Add the OpCapability.
54 // 3. Modify the memory model.
55 Instruction* memory_model = get_module()->GetMemoryModel();
56 context()->AddCapability(MakeUnique<Instruction>(
57 context(), SpvOpCapability, 0, 0,
58 std::initializer_list<Operand>{
59 {SPV_OPERAND_TYPE_CAPABILITY, {SpvCapabilityVulkanMemoryModelKHR}}}));
60 const std::string extension = "SPV_KHR_vulkan_memory_model";
61 std::vector<uint32_t> words(extension.size() / 4 + 1, 0);
62 char* dst = reinterpret_cast<char*>(words.data());
63 strncpy(dst, extension.c_str(), extension.size());
64 context()->AddExtension(
65 MakeUnique<Instruction>(context(), SpvOpExtension, 0, 0,
66 std::initializer_list<Operand>{
67 {SPV_OPERAND_TYPE_LITERAL_STRING, words}}));
68 memory_model->SetInOperand(1u, {SpvMemoryModelVulkanKHR});
69}
70
71void UpgradeMemoryModel::UpgradeInstructions() {
72 // Coherent and Volatile decorations are deprecated. Remove them and replace
73 // with flags on the memory/image operations. The decorations can occur on
74 // OpVariable, OpFunctionParameter (of pointer type) and OpStructType (member
75 // decoration). Trace from the decoration target(s) to the final memory/image
76 // instructions. Additionally, Workgroup storage class variables and function
77 // parameters are implicitly coherent in GLSL450.
78
79 // Upgrade modf and frexp first since they generate new stores.
80 // In SPIR-V 1.4 or later, normalize OpCopyMemory* access operands.
81 for (auto& func : *get_module()) {
82 func.ForEachInst([this](Instruction* inst) {
83 if (inst->opcode() == SpvOpExtInst) {
84 auto ext_inst = inst->GetSingleWordInOperand(1u);
85 if (ext_inst == GLSLstd450Modf || ext_inst == GLSLstd450Frexp) {
86 auto import =
87 get_def_use_mgr()->GetDef(inst->GetSingleWordInOperand(0u));
88 if (reinterpret_cast<char*>(import->GetInOperand(0u).words.data()) ==
89 std::string("GLSL.std.450")) {
90 UpgradeExtInst(inst);
91 }
92 }
93 } else if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
94 if (inst->opcode() == SpvOpCopyMemory ||
95 inst->opcode() == SpvOpCopyMemorySized) {
96 uint32_t start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
97 if (inst->NumInOperands() > start_operand) {
98 auto num_access_words = MemoryAccessNumWords(
99 inst->GetSingleWordInOperand(start_operand));
100 if ((num_access_words + start_operand) == inst->NumInOperands()) {
101 // There is a single memory access operand. Duplicate it to have a
102 // separate operand for both source and target.
103 for (uint32_t i = 0; i < num_access_words; ++i) {
104 auto operand = inst->GetInOperand(start_operand + i);
105 inst->AddOperand(std::move(operand));
106 }
107 }
108 } else {
109 // Add two memory access operands.
110 inst->AddOperand(
111 {SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}});
112 inst->AddOperand(
113 {SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}});
114 }
115 }
116 }
117 });
118 }
119
120 UpgradeMemoryAndImages();
121 UpgradeAtomics();
122}
123
124void UpgradeMemoryModel::UpgradeMemoryAndImages() {
125 for (auto& func : *get_module()) {
126 func.ForEachInst([this](Instruction* inst) {
127 bool is_coherent = false;
128 bool is_volatile = false;
129 bool src_coherent = false;
130 bool src_volatile = false;
131 bool dst_coherent = false;
132 bool dst_volatile = false;
133 uint32_t start_operand = 0u;
134 SpvScope scope = SpvScopeQueueFamilyKHR;
135 SpvScope src_scope = SpvScopeQueueFamilyKHR;
136 SpvScope dst_scope = SpvScopeQueueFamilyKHR;
137 switch (inst->opcode()) {
138 case SpvOpLoad:
139 case SpvOpStore:
140 std::tie(is_coherent, is_volatile, scope) =
141 GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
142 break;
143 case SpvOpImageRead:
144 case SpvOpImageSparseRead:
145 case SpvOpImageWrite:
146 std::tie(is_coherent, is_volatile, scope) =
147 GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
148 break;
149 case SpvOpCopyMemory:
150 case SpvOpCopyMemorySized:
151 std::tie(dst_coherent, dst_volatile, dst_scope) =
152 GetInstructionAttributes(inst->GetSingleWordInOperand(0u));
153 std::tie(src_coherent, src_volatile, src_scope) =
154 GetInstructionAttributes(inst->GetSingleWordInOperand(1u));
155 break;
156 default:
157 break;
158 }
159
160 switch (inst->opcode()) {
161 case SpvOpLoad:
162 UpgradeFlags(inst, 1u, is_coherent, is_volatile, kVisibility,
163 kMemory);
164 break;
165 case SpvOpStore:
166 UpgradeFlags(inst, 2u, is_coherent, is_volatile, kAvailability,
167 kMemory);
168 break;
169 case SpvOpCopyMemory:
170 case SpvOpCopyMemorySized:
171 start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
172 if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
173 // There are guaranteed to be two memory access operands at this
174 // point so treat source and target separately.
175 uint32_t num_access_words = MemoryAccessNumWords(
176 inst->GetSingleWordInOperand(start_operand));
177 UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
178 kAvailability, kMemory);
179 UpgradeFlags(inst, start_operand + num_access_words, src_coherent,
180 src_volatile, kVisibility, kMemory);
181 } else {
182 UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
183 kAvailability, kMemory);
184 UpgradeFlags(inst, start_operand, src_coherent, src_volatile,
185 kVisibility, kMemory);
186 }
187 break;
188 case SpvOpImageRead:
189 case SpvOpImageSparseRead:
190 UpgradeFlags(inst, 2u, is_coherent, is_volatile, kVisibility, kImage);
191 break;
192 case SpvOpImageWrite:
193 UpgradeFlags(inst, 3u, is_coherent, is_volatile, kAvailability,
194 kImage);
195 break;
196 default:
197 break;
198 }
199
200 // |is_coherent| is never used for the same instructions as
201 // |src_coherent| and |dst_coherent|.
202 if (is_coherent) {
203 inst->AddOperand(
204 {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(scope)}});
205 }
206 if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
207 // There are two memory access operands. The first is for the target and
208 // the second is for the source.
209 if (dst_coherent || src_coherent) {
210 start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
211 std::vector<Operand> new_operands;
212 uint32_t num_access_words =
213 MemoryAccessNumWords(inst->GetSingleWordInOperand(start_operand));
214 // The flags were already updated so subtract if we're adding a
215 // scope.
216 if (dst_coherent) --num_access_words;
217 for (uint32_t i = 0; i < start_operand + num_access_words; ++i) {
218 new_operands.push_back(inst->GetInOperand(i));
219 }
220 // Add the target scope if necessary.
221 if (dst_coherent) {
222 new_operands.push_back(
223 {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
224 }
225 // Copy the remaining current operands.
226 for (uint32_t i = start_operand + num_access_words;
227 i < inst->NumInOperands(); ++i) {
228 new_operands.push_back(inst->GetInOperand(i));
229 }
230 // Add the source scope if necessary.
231 if (src_coherent) {
232 new_operands.push_back(
233 {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
234 }
235 inst->SetInOperands(std::move(new_operands));
236 }
237 } else {
238 // According to SPV_KHR_vulkan_memory_model, if both available and
239 // visible flags are used the first scope operand is for availability
240 // (writes) and the second is for visibility (reads).
241 if (dst_coherent) {
242 inst->AddOperand(
243 {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
244 }
245 if (src_coherent) {
246 inst->AddOperand(
247 {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
248 }
249 }
250 });
251 }
252}
253
254void UpgradeMemoryModel::UpgradeAtomics() {
255 for (auto& func : *get_module()) {
256 func.ForEachInst([this](Instruction* inst) {
257 if (spvOpcodeIsAtomicOp(inst->opcode())) {
258 bool unused_coherent = false;
259 bool is_volatile = false;
260 SpvScope unused_scope = SpvScopeQueueFamilyKHR;
261 std::tie(unused_coherent, is_volatile, unused_scope) =
262 GetInstructionAttributes(inst->GetSingleWordInOperand(0));
263
264 UpgradeSemantics(inst, 2u, is_volatile);
265 if (inst->opcode() == SpvOpAtomicCompareExchange ||
266 inst->opcode() == SpvOpAtomicCompareExchangeWeak) {
267 UpgradeSemantics(inst, 3u, is_volatile);
268 }
269 }
270 });
271 }
272}
273
274void UpgradeMemoryModel::UpgradeSemantics(Instruction* inst,
275 uint32_t in_operand,
276 bool is_volatile) {
277 if (!is_volatile) return;
278
279 uint32_t semantics_id = inst->GetSingleWordInOperand(in_operand);
280 const analysis::Constant* constant =
281 context()->get_constant_mgr()->FindDeclaredConstant(semantics_id);
282 const analysis::Integer* type = constant->type()->AsInteger();
283 assert(type && type->width() == 32);
284 uint32_t value = 0;
285 if (type->IsSigned()) {
286 value = static_cast<uint32_t>(constant->GetS32());
287 } else {
288 value = constant->GetU32();
289 }
290
291 value |= SpvMemorySemanticsVolatileMask;
292 auto new_constant = context()->get_constant_mgr()->GetConstant(type, {value});
293 auto new_semantics =
294 context()->get_constant_mgr()->GetDefiningInstruction(new_constant);
295 inst->SetInOperand(in_operand, {new_semantics->result_id()});
296}
297
298std::tuple<bool, bool, SpvScope> UpgradeMemoryModel::GetInstructionAttributes(
299 uint32_t id) {
300 // |id| is a pointer used in a memory/image instruction. Need to determine if
301 // that pointer points to volatile or coherent memory. Workgroup storage
302 // class is implicitly coherent and cannot be decorated with volatile, so
303 // short circuit that case.
304 Instruction* inst = context()->get_def_use_mgr()->GetDef(id);
305 analysis::Type* type = context()->get_type_mgr()->GetType(inst->type_id());
306 if (type->AsPointer() &&
307 type->AsPointer()->storage_class() == SpvStorageClassWorkgroup) {
308 return std::make_tuple(true, false, SpvScopeWorkgroup);
309 }
310
311 bool is_coherent = false;
312 bool is_volatile = false;
313 std::unordered_set<uint32_t> visited;
314 std::tie(is_coherent, is_volatile) =
315 TraceInstruction(context()->get_def_use_mgr()->GetDef(id),
316 std::vector<uint32_t>(), &visited);
317
318 return std::make_tuple(is_coherent, is_volatile, SpvScopeQueueFamilyKHR);
319}
320
321std::pair<bool, bool> UpgradeMemoryModel::TraceInstruction(
322 Instruction* inst, std::vector<uint32_t> indices,
323 std::unordered_set<uint32_t>* visited) {
324 auto iter = cache_.find(std::make_pair(inst->result_id(), indices));
325 if (iter != cache_.end()) {
326 return iter->second;
327 }
328
329 if (!visited->insert(inst->result_id()).second) {
330 return std::make_pair(false, false);
331 }
332
333 // Initialize the cache before |indices| is (potentially) modified.
334 auto& cached_result = cache_[std::make_pair(inst->result_id(), indices)];
335 cached_result.first = false;
336 cached_result.second = false;
337
338 bool is_coherent = false;
339 bool is_volatile = false;
340 switch (inst->opcode()) {
341 case SpvOpVariable:
342 case SpvOpFunctionParameter:
343 is_coherent |= HasDecoration(inst, 0, SpvDecorationCoherent);
344 is_volatile |= HasDecoration(inst, 0, SpvDecorationVolatile);
345 if (!is_coherent || !is_volatile) {
346 bool type_coherent = false;
347 bool type_volatile = false;
348 std::tie(type_coherent, type_volatile) =
349 CheckType(inst->type_id(), indices);
350 is_coherent |= type_coherent;
351 is_volatile |= type_volatile;
352 }
353 break;
354 case SpvOpAccessChain:
355 case SpvOpInBoundsAccessChain:
356 // Store indices in reverse order.
357 for (uint32_t i = inst->NumInOperands() - 1; i > 0; --i) {
358 indices.push_back(inst->GetSingleWordInOperand(i));
359 }
360 break;
361 case SpvOpPtrAccessChain:
362 // Store indices in reverse order. Skip the |Element| operand.
363 for (uint32_t i = inst->NumInOperands() - 1; i > 1; --i) {
364 indices.push_back(inst->GetSingleWordInOperand(i));
365 }
366 break;
367 default:
368 break;
369 }
370
371 // No point searching further.
372 if (is_coherent && is_volatile) {
373 cached_result.first = true;
374 cached_result.second = true;
375 return std::make_pair(true, true);
376 }
377
378 // Variables and function parameters are sources. Continue searching until we
379 // reach them.
380 if (inst->opcode() != SpvOpVariable &&
381 inst->opcode() != SpvOpFunctionParameter) {
382 inst->ForEachInId([this, &is_coherent, &is_volatile, &indices,
383 &visited](const uint32_t* id_ptr) {
384 Instruction* op_inst = context()->get_def_use_mgr()->GetDef(*id_ptr);
385 const analysis::Type* type =
386 context()->get_type_mgr()->GetType(op_inst->type_id());
387 if (type &&
388 (type->AsPointer() || type->AsImage() || type->AsSampledImage())) {
389 bool operand_coherent = false;
390 bool operand_volatile = false;
391 std::tie(operand_coherent, operand_volatile) =
392 TraceInstruction(op_inst, indices, visited);
393 is_coherent |= operand_coherent;
394 is_volatile |= operand_volatile;
395 }
396 });
397 }
398
399 cached_result.first = is_coherent;
400 cached_result.second = is_volatile;
401 return std::make_pair(is_coherent, is_volatile);
402}
403
404std::pair<bool, bool> UpgradeMemoryModel::CheckType(
405 uint32_t type_id, const std::vector<uint32_t>& indices) {
406 bool is_coherent = false;
407 bool is_volatile = false;
408 Instruction* type_inst = context()->get_def_use_mgr()->GetDef(type_id);
409 assert(type_inst->opcode() == SpvOpTypePointer);
410 Instruction* element_inst = context()->get_def_use_mgr()->GetDef(
411 type_inst->GetSingleWordInOperand(1u));
412 for (int i = (int)indices.size() - 1; i >= 0; --i) {
413 if (is_coherent && is_volatile) break;
414
415 if (element_inst->opcode() == SpvOpTypePointer) {
416 element_inst = context()->get_def_use_mgr()->GetDef(
417 element_inst->GetSingleWordInOperand(1u));
418 } else if (element_inst->opcode() == SpvOpTypeStruct) {
419 uint32_t index = indices.at(i);
420 Instruction* index_inst = context()->get_def_use_mgr()->GetDef(index);
421 assert(index_inst->opcode() == SpvOpConstant);
422 uint64_t value = GetIndexValue(index_inst);
423 is_coherent |= HasDecoration(element_inst, static_cast<uint32_t>(value),
424 SpvDecorationCoherent);
425 is_volatile |= HasDecoration(element_inst, static_cast<uint32_t>(value),
426 SpvDecorationVolatile);
427 element_inst = context()->get_def_use_mgr()->GetDef(
428 element_inst->GetSingleWordInOperand(static_cast<uint32_t>(value)));
429 } else {
430 assert(spvOpcodeIsComposite(element_inst->opcode()));
431 element_inst = context()->get_def_use_mgr()->GetDef(
432 element_inst->GetSingleWordInOperand(0u));
433 }
434 }
435
436 if (!is_coherent || !is_volatile) {
437 bool remaining_coherent = false;
438 bool remaining_volatile = false;
439 std::tie(remaining_coherent, remaining_volatile) =
440 CheckAllTypes(element_inst);
441 is_coherent |= remaining_coherent;
442 is_volatile |= remaining_volatile;
443 }
444
445 return std::make_pair(is_coherent, is_volatile);
446}
447
448std::pair<bool, bool> UpgradeMemoryModel::CheckAllTypes(
449 const Instruction* inst) {
450 std::unordered_set<const Instruction*> visited;
451 std::vector<const Instruction*> stack;
452 stack.push_back(inst);
453
454 bool is_coherent = false;
455 bool is_volatile = false;
456 while (!stack.empty()) {
457 const Instruction* def = stack.back();
458 stack.pop_back();
459
460 if (!visited.insert(def).second) continue;
461
462 if (def->opcode() == SpvOpTypeStruct) {
463 // Any member decorated with coherent and/or volatile is enough to have
464 // the related operation be flagged as coherent and/or volatile.
465 is_coherent |= HasDecoration(def, std::numeric_limits<uint32_t>::max(),
466 SpvDecorationCoherent);
467 is_volatile |= HasDecoration(def, std::numeric_limits<uint32_t>::max(),
468 SpvDecorationVolatile);
469 if (is_coherent && is_volatile)
470 return std::make_pair(is_coherent, is_volatile);
471
472 // Check the subtypes.
473 for (uint32_t i = 0; i < def->NumInOperands(); ++i) {
474 stack.push_back(context()->get_def_use_mgr()->GetDef(
475 def->GetSingleWordInOperand(i)));
476 }
477 } else if (spvOpcodeIsComposite(def->opcode())) {
478 stack.push_back(context()->get_def_use_mgr()->GetDef(
479 def->GetSingleWordInOperand(0u)));
480 } else if (def->opcode() == SpvOpTypePointer) {
481 stack.push_back(context()->get_def_use_mgr()->GetDef(
482 def->GetSingleWordInOperand(1u)));
483 }
484 }
485
486 return std::make_pair(is_coherent, is_volatile);
487}
488
489uint64_t UpgradeMemoryModel::GetIndexValue(Instruction* index_inst) {
490 const analysis::Constant* index_constant =
491 context()->get_constant_mgr()->GetConstantFromInst(index_inst);
492 assert(index_constant->AsIntConstant());
493 if (index_constant->type()->AsInteger()->IsSigned()) {
494 if (index_constant->type()->AsInteger()->width() == 32) {
495 return index_constant->GetS32();
496 } else {
497 return index_constant->GetS64();
498 }
499 } else {
500 if (index_constant->type()->AsInteger()->width() == 32) {
501 return index_constant->GetU32();
502 } else {
503 return index_constant->GetU64();
504 }
505 }
506}
507
508bool UpgradeMemoryModel::HasDecoration(const Instruction* inst, uint32_t value,
509 SpvDecoration decoration) {
510 // If the iteration was terminated early then an appropriate decoration was
511 // found.
512 return !context()->get_decoration_mgr()->WhileEachDecoration(
513 inst->result_id(), decoration, [value](const Instruction& i) {
514 if (i.opcode() == SpvOpDecorate || i.opcode() == SpvOpDecorateId) {
515 return false;
516 } else if (i.opcode() == SpvOpMemberDecorate) {
517 if (value == i.GetSingleWordInOperand(1u) ||
518 value == std::numeric_limits<uint32_t>::max())
519 return false;
520 }
521
522 return true;
523 });
524}
525
526void UpgradeMemoryModel::UpgradeFlags(Instruction* inst, uint32_t in_operand,
527 bool is_coherent, bool is_volatile,
528 OperationType operation_type,
529 InstructionType inst_type) {
530 if (!is_coherent && !is_volatile) return;
531
532 uint32_t flags = 0;
533 if (inst->NumInOperands() > in_operand) {
534 flags |= inst->GetSingleWordInOperand(in_operand);
535 }
536 if (is_coherent) {
537 if (inst_type == kMemory) {
538 flags |= SpvMemoryAccessNonPrivatePointerKHRMask;
539 if (operation_type == kVisibility) {
540 flags |= SpvMemoryAccessMakePointerVisibleKHRMask;
541 } else {
542 flags |= SpvMemoryAccessMakePointerAvailableKHRMask;
543 }
544 } else {
545 flags |= SpvImageOperandsNonPrivateTexelKHRMask;
546 if (operation_type == kVisibility) {
547 flags |= SpvImageOperandsMakeTexelVisibleKHRMask;
548 } else {
549 flags |= SpvImageOperandsMakeTexelAvailableKHRMask;
550 }
551 }
552 }
553
554 if (is_volatile) {
555 if (inst_type == kMemory) {
556 flags |= SpvMemoryAccessVolatileMask;
557 } else {
558 flags |= SpvImageOperandsVolatileTexelKHRMask;
559 }
560 }
561
562 if (inst->NumInOperands() > in_operand) {
563 inst->SetInOperand(in_operand, {flags});
564 } else if (inst_type == kMemory) {
565 inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS, {flags}});
566 } else {
567 inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_IMAGE, {flags}});
568 }
569}
570
571uint32_t UpgradeMemoryModel::GetScopeConstant(SpvScope scope) {
572 analysis::Integer int_ty(32, false);
573 uint32_t int_id = context()->get_type_mgr()->GetTypeInstruction(&int_ty);
574 const analysis::Constant* constant =
575 context()->get_constant_mgr()->GetConstant(
576 context()->get_type_mgr()->GetType(int_id),
577 {static_cast<uint32_t>(scope)});
578 return context()
579 ->get_constant_mgr()
580 ->GetDefiningInstruction(constant)
581 ->result_id();
582}
583
584void UpgradeMemoryModel::CleanupDecorations() {
585 // All of the volatile and coherent decorations have been dealt with, so now
586 // we can just remove them.
587 get_module()->ForEachInst([this](Instruction* inst) {
588 if (inst->result_id() != 0) {
589 context()->get_decoration_mgr()->RemoveDecorationsFrom(
590 inst->result_id(), [](const Instruction& dec) {
591 switch (dec.opcode()) {
592 case SpvOpDecorate:
593 case SpvOpDecorateId:
594 if (dec.GetSingleWordInOperand(1u) == SpvDecorationCoherent ||
595 dec.GetSingleWordInOperand(1u) == SpvDecorationVolatile)
596 return true;
597 break;
598 case SpvOpMemberDecorate:
599 if (dec.GetSingleWordInOperand(2u) == SpvDecorationCoherent ||
600 dec.GetSingleWordInOperand(2u) == SpvDecorationVolatile)
601 return true;
602 break;
603 default:
604 break;
605 }
606 return false;
607 });
608 }
609 });
610}
611
612void UpgradeMemoryModel::UpgradeBarriers() {
613 std::vector<Instruction*> barriers;
614 // Collects all the control barriers in |function|. Returns true if the
615 // function operates on the Output storage class.
616 ProcessFunction CollectBarriers = [this, &barriers](Function* function) {
617 bool operates_on_output = false;
618 for (auto& block : *function) {
619 block.ForEachInst([this, &barriers,
620 &operates_on_output](Instruction* inst) {
621 if (inst->opcode() == SpvOpControlBarrier) {
622 barriers.push_back(inst);
623 } else if (!operates_on_output) {
624 // This instruction operates on output storage class if it is a
625 // pointer to output type or any input operand is a pointer to output
626 // type.
627 analysis::Type* type =
628 context()->get_type_mgr()->GetType(inst->type_id());
629 if (type && type->AsPointer() &&
630 type->AsPointer()->storage_class() == SpvStorageClassOutput) {
631 operates_on_output = true;
632 return;
633 }
634 inst->ForEachInId([this, &operates_on_output](uint32_t* id_ptr) {
635 Instruction* op_inst =
636 context()->get_def_use_mgr()->GetDef(*id_ptr);
637 analysis::Type* op_type =
638 context()->get_type_mgr()->GetType(op_inst->type_id());
639 if (op_type && op_type->AsPointer() &&
640 op_type->AsPointer()->storage_class() == SpvStorageClassOutput)
641 operates_on_output = true;
642 });
643 }
644 });
645 }
646 return operates_on_output;
647 };
648
649 std::queue<uint32_t> roots;
650 for (auto& e : get_module()->entry_points())
651 if (e.GetSingleWordInOperand(0u) == SpvExecutionModelTessellationControl) {
652 roots.push(e.GetSingleWordInOperand(1u));
653 if (context()->ProcessCallTreeFromRoots(CollectBarriers, &roots)) {
654 for (auto barrier : barriers) {
655 // Add OutputMemoryKHR to the semantics of the barriers.
656 uint32_t semantics_id = barrier->GetSingleWordInOperand(2u);
657 Instruction* semantics_inst =
658 context()->get_def_use_mgr()->GetDef(semantics_id);
659 analysis::Type* semantics_type =
660 context()->get_type_mgr()->GetType(semantics_inst->type_id());
661 uint64_t semantics_value = GetIndexValue(semantics_inst);
662 const analysis::Constant* constant =
663 context()->get_constant_mgr()->GetConstant(
664 semantics_type, {static_cast<uint32_t>(semantics_value) |
665 SpvMemorySemanticsOutputMemoryKHRMask});
666 barrier->SetInOperand(2u, {context()
667 ->get_constant_mgr()
668 ->GetDefiningInstruction(constant)
669 ->result_id()});
670 }
671 }
672 barriers.clear();
673 }
674}
675
676void UpgradeMemoryModel::UpgradeMemoryScope() {
677 get_module()->ForEachInst([this](Instruction* inst) {
678 // Don't need to handle all the operations that take a scope.
679 // * Group operations can only be subgroup
680 // * Non-uniform can only be workgroup or subgroup
681 // * Named barriers are not supported by Vulkan
682 // * Workgroup ops (e.g. async_copy) have at most workgroup scope.
683 if (spvOpcodeIsAtomicOp(inst->opcode())) {
684 if (IsDeviceScope(inst->GetSingleWordInOperand(1))) {
685 inst->SetInOperand(1, {GetScopeConstant(SpvScopeQueueFamilyKHR)});
686 }
687 } else if (inst->opcode() == SpvOpControlBarrier) {
688 if (IsDeviceScope(inst->GetSingleWordInOperand(1))) {
689 inst->SetInOperand(1, {GetScopeConstant(SpvScopeQueueFamilyKHR)});
690 }
691 } else if (inst->opcode() == SpvOpMemoryBarrier) {
692 if (IsDeviceScope(inst->GetSingleWordInOperand(0))) {
693 inst->SetInOperand(0, {GetScopeConstant(SpvScopeQueueFamilyKHR)});
694 }
695 }
696 });
697}
698
699bool UpgradeMemoryModel::IsDeviceScope(uint32_t scope_id) {
700 const analysis::Constant* constant =
701 context()->get_constant_mgr()->FindDeclaredConstant(scope_id);
702 assert(constant && "Memory scope must be a constant");
703
704 const analysis::Integer* type = constant->type()->AsInteger();
705 assert(type);
706 assert(type->width() == 32 || type->width() == 64);
707 if (type->width() == 32) {
708 if (type->IsSigned())
709 return static_cast<uint32_t>(constant->GetS32()) == SpvScopeDevice;
710 else
711 return static_cast<uint32_t>(constant->GetU32()) == SpvScopeDevice;
712 } else {
713 if (type->IsSigned())
714 return static_cast<uint32_t>(constant->GetS64()) == SpvScopeDevice;
715 else
716 return static_cast<uint32_t>(constant->GetU64()) == SpvScopeDevice;
717 }
718
719 assert(false);
720 return false;
721}
722
723void UpgradeMemoryModel::UpgradeExtInst(Instruction* ext_inst) {
724 const bool is_modf = ext_inst->GetSingleWordInOperand(1u) == GLSLstd450Modf;
725 auto ptr_id = ext_inst->GetSingleWordInOperand(3u);
726 auto ptr_type_id = get_def_use_mgr()->GetDef(ptr_id)->type_id();
727 auto pointee_type_id =
728 get_def_use_mgr()->GetDef(ptr_type_id)->GetSingleWordInOperand(1u);
729 auto element_type_id = ext_inst->type_id();
730 std::vector<const analysis::Type*> element_types(2);
731 element_types[0] = context()->get_type_mgr()->GetType(element_type_id);
732 element_types[1] = context()->get_type_mgr()->GetType(pointee_type_id);
733 analysis::Struct struct_type(element_types);
734 uint32_t struct_id =
735 context()->get_type_mgr()->GetTypeInstruction(&struct_type);
736 // Change the operation
737 GLSLstd450 new_op = is_modf ? GLSLstd450ModfStruct : GLSLstd450FrexpStruct;
738 ext_inst->SetOperand(3u, {static_cast<uint32_t>(new_op)});
739 // Remove the pointer argument
740 ext_inst->RemoveOperand(5u);
741 // Set the type id to the new struct.
742 ext_inst->SetResultType(struct_id);
743
744 // The result is now a struct of the original result. The zero'th element is
745 // old result and should replace the old result. The one'th element needs to
746 // be stored via a new instruction.
747 auto where = ext_inst->NextNode();
748 InstructionBuilder builder(
749 context(), where,
750 IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping);
751 auto extract_0 =
752 builder.AddCompositeExtract(element_type_id, ext_inst->result_id(), {0});
753 context()->ReplaceAllUsesWith(ext_inst->result_id(), extract_0->result_id());
754 // The extract's input was just changed to itself, so fix that.
755 extract_0->SetInOperand(0u, {ext_inst->result_id()});
756 auto extract_1 =
757 builder.AddCompositeExtract(pointee_type_id, ext_inst->result_id(), {1});
758 builder.AddStore(ptr_id, extract_1->result_id());
759}
760
761uint32_t UpgradeMemoryModel::MemoryAccessNumWords(uint32_t mask) {
762 uint32_t result = 1;
763 if (mask & SpvMemoryAccessAlignedMask) ++result;
764 if (mask & SpvMemoryAccessMakePointerAvailableKHRMask) ++result;
765 if (mask & SpvMemoryAccessMakePointerVisibleKHRMask) ++result;
766 return result;
767}
768
769} // namespace opt
770} // namespace spvtools
771