1/*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "gc/shared/c2/barrierSetC2.hpp"
27#include "opto/arraycopynode.hpp"
28#include "opto/convertnode.hpp"
29#include "opto/graphKit.hpp"
30#include "opto/idealKit.hpp"
31#include "opto/macro.hpp"
32#include "opto/narrowptrnode.hpp"
33#include "utilities/macros.hpp"
34
35// By default this is a no-op.
36void BarrierSetC2::resolve_address(C2Access& access) const { }
37
38void* C2ParseAccess::barrier_set_state() const {
39 return _kit->barrier_set_state();
40}
41
42PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
43
44bool C2Access::needs_cpu_membar() const {
45 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
46 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
47 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
48 bool in_heap = (_decorators & IN_HEAP) != 0;
49
50 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
51 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
52 bool is_atomic = is_read && is_write;
53
54 if (is_atomic) {
55 // Atomics always need to be wrapped in CPU membars
56 return true;
57 }
58
59 if (anonymous) {
60 // We will need memory barriers unless we can determine a unique
61 // alias category for this reference. (Note: If for some reason
62 // the barriers get omitted and the unsafe reference begins to "pollute"
63 // the alias analysis of the rest of the graph, either Compile::can_alias
64 // or Compile::must_alias will throw a diagnostic assert.)
65 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
66 return true;
67 }
68 }
69
70 return false;
71}
72
73Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
74 DecoratorSet decorators = access.decorators();
75
76 bool mismatched = (decorators & C2_MISMATCHED) != 0;
77 bool unaligned = (decorators & C2_UNALIGNED) != 0;
78 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
79 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
80
81 bool in_native = (decorators & IN_NATIVE) != 0;
82 assert(!in_native, "not supported yet");
83
84 MemNode::MemOrd mo = access.mem_node_mo();
85
86 Node* store;
87 if (access.is_parse_access()) {
88 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
89
90 GraphKit* kit = parse_access.kit();
91 if (access.type() == T_DOUBLE) {
92 Node* new_val = kit->dstore_rounding(val.node());
93 val.set_node(new_val);
94 }
95
96 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
97 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe);
98 access.set_raw_access(store);
99 } else {
100 assert(!requires_atomic_access, "not yet supported");
101 assert(access.is_opt_access(), "either parse or opt access");
102 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
103 Node* ctl = opt_access.ctl();
104 MergeMemNode* mm = opt_access.mem();
105 PhaseGVN& gvn = opt_access.gvn();
106 const TypePtr* adr_type = access.addr().type();
107 int alias = gvn.C->get_alias_index(adr_type);
108 Node* mem = mm->memory_at(alias);
109
110 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo);
111 if (unaligned) {
112 st->set_unaligned_access();
113 }
114 if (mismatched) {
115 st->set_mismatched_access();
116 }
117 store = gvn.transform(st);
118 if (store == st) {
119 mm->set_memory_at(alias, st);
120 }
121 }
122 return store;
123}
124
125Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
126 DecoratorSet decorators = access.decorators();
127
128 Node* adr = access.addr().node();
129 const TypePtr* adr_type = access.addr().type();
130
131 bool mismatched = (decorators & C2_MISMATCHED) != 0;
132 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
133 bool unaligned = (decorators & C2_UNALIGNED) != 0;
134 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
135 bool pinned = (decorators & C2_PINNED_LOAD) != 0;
136 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
137
138 bool in_native = (decorators & IN_NATIVE) != 0;
139
140 MemNode::MemOrd mo = access.mem_node_mo();
141 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
142
143 Node* load;
144 if (access.is_parse_access()) {
145 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
146 GraphKit* kit = parse_access.kit();
147 Node* control = control_dependent ? kit->control() : NULL;
148
149 if (in_native) {
150 load = kit->make_load(control, adr, val_type, access.type(), mo);
151 } else {
152 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
153 dep, requires_atomic_access, unaligned, mismatched, unsafe);
154 }
155 access.set_raw_access(load);
156 } else {
157 assert(!requires_atomic_access, "not yet supported");
158 assert(access.is_opt_access(), "either parse or opt access");
159 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
160 Node* control = control_dependent ? opt_access.ctl() : NULL;
161 MergeMemNode* mm = opt_access.mem();
162 PhaseGVN& gvn = opt_access.gvn();
163 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
164 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
165 load = gvn.transform(load);
166 }
167
168 return load;
169}
170
171class C2AccessFence: public StackObj {
172 C2Access& _access;
173 Node* _leading_membar;
174
175public:
176 C2AccessFence(C2Access& access) :
177 _access(access), _leading_membar(NULL) {
178 GraphKit* kit = NULL;
179 if (access.is_parse_access()) {
180 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
181 kit = parse_access.kit();
182 }
183 DecoratorSet decorators = access.decorators();
184
185 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
186 bool is_read = (decorators & C2_READ_ACCESS) != 0;
187 bool is_atomic = is_read && is_write;
188
189 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
190 bool is_release = (decorators & MO_RELEASE) != 0;
191
192 if (is_atomic) {
193 assert(kit != NULL, "unsupported at optimization time");
194 // Memory-model-wise, a LoadStore acts like a little synchronized
195 // block, so needs barriers on each side. These don't translate
196 // into actual barriers on most machines, but we still need rest of
197 // compiler to respect ordering.
198 if (is_release) {
199 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
200 } else if (is_volatile) {
201 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
202 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
203 } else {
204 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
205 }
206 }
207 } else if (is_write) {
208 // If reference is volatile, prevent following memory ops from
209 // floating down past the volatile write. Also prevents commoning
210 // another volatile read.
211 if (is_volatile || is_release) {
212 assert(kit != NULL, "unsupported at optimization time");
213 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
214 }
215 } else {
216 // Memory barrier to prevent normal and 'unsafe' accesses from
217 // bypassing each other. Happens after null checks, so the
218 // exception paths do not take memory state from the memory barrier,
219 // so there's no problems making a strong assert about mixing users
220 // of safe & unsafe memory.
221 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
222 assert(kit != NULL, "unsupported at optimization time");
223 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
224 }
225 }
226
227 if (access.needs_cpu_membar()) {
228 assert(kit != NULL, "unsupported at optimization time");
229 kit->insert_mem_bar(Op_MemBarCPUOrder);
230 }
231
232 if (is_atomic) {
233 // 4984716: MemBars must be inserted before this
234 // memory node in order to avoid a false
235 // dependency which will confuse the scheduler.
236 access.set_memory();
237 }
238 }
239
240 ~C2AccessFence() {
241 GraphKit* kit = NULL;
242 if (_access.is_parse_access()) {
243 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
244 kit = parse_access.kit();
245 }
246 DecoratorSet decorators = _access.decorators();
247
248 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
249 bool is_read = (decorators & C2_READ_ACCESS) != 0;
250 bool is_atomic = is_read && is_write;
251
252 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
253 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
254
255 // If reference is volatile, prevent following volatiles ops from
256 // floating up before the volatile access.
257 if (_access.needs_cpu_membar()) {
258 kit->insert_mem_bar(Op_MemBarCPUOrder);
259 }
260
261 if (is_atomic) {
262 assert(kit != NULL, "unsupported at optimization time");
263 if (is_acquire || is_volatile) {
264 Node* n = _access.raw_access();
265 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
266 if (_leading_membar != NULL) {
267 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
268 }
269 }
270 } else if (is_write) {
271 // If not multiple copy atomic, we do the MemBarVolatile before the load.
272 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
273 assert(kit != NULL, "unsupported at optimization time");
274 Node* n = _access.raw_access();
275 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
276 if (_leading_membar != NULL) {
277 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
278 }
279 }
280 } else {
281 if (is_volatile || is_acquire) {
282 assert(kit != NULL, "unsupported at optimization time");
283 Node* n = _access.raw_access();
284 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
285 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
286 mb->as_MemBar()->set_trailing_load();
287 }
288 }
289 }
290};
291
292Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
293 C2AccessFence fence(access);
294 resolve_address(access);
295 return store_at_resolved(access, val);
296}
297
298Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
299 C2AccessFence fence(access);
300 resolve_address(access);
301 return load_at_resolved(access, val_type);
302}
303
304MemNode::MemOrd C2Access::mem_node_mo() const {
305 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
306 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
307 if ((_decorators & MO_SEQ_CST) != 0) {
308 if (is_write && is_read) {
309 // For atomic operations
310 return MemNode::seqcst;
311 } else if (is_write) {
312 return MemNode::release;
313 } else {
314 assert(is_read, "what else?");
315 return MemNode::acquire;
316 }
317 } else if ((_decorators & MO_RELEASE) != 0) {
318 return MemNode::release;
319 } else if ((_decorators & MO_ACQUIRE) != 0) {
320 return MemNode::acquire;
321 } else if (is_write) {
322 // Volatile fields need releasing stores.
323 // Non-volatile fields also need releasing stores if they hold an
324 // object reference, because the object reference might point to
325 // a freshly created object.
326 // Conservatively release stores of object references.
327 return StoreNode::release_if_reference(_type);
328 } else {
329 return MemNode::unordered;
330 }
331}
332
333void C2Access::fixup_decorators() {
334 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
335 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
336 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
337
338 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
339 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
340
341 if (AlwaysAtomicAccesses && is_unordered) {
342 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
343 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
344 }
345
346 _decorators = AccessInternal::decorator_fixup(_decorators);
347
348 if (is_read && !is_write && anonymous) {
349 // To be valid, unsafe loads may depend on other conditions than
350 // the one that guards them: pin the Load node
351 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
352 _decorators |= C2_PINNED_LOAD;
353 const TypePtr* adr_type = _addr.type();
354 Node* adr = _addr.node();
355 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
356 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
357 intptr_t offset = Type::OffsetBot;
358 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
359 if (offset >= 0) {
360 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
361 if (offset < s) {
362 // Guaranteed to be a valid access, no need to pin it
363 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
364 _decorators ^= C2_PINNED_LOAD;
365 }
366 }
367 }
368 }
369}
370
371//--------------------------- atomic operations---------------------------------
372
373void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
374 if (!access.needs_pinning()) {
375 return;
376 }
377 // SCMemProjNodes represent the memory state of a LoadStore. Their
378 // main role is to prevent LoadStore nodes from being optimized away
379 // when their results aren't used.
380 assert(access.is_parse_access(), "entry not supported at optimization time");
381 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
382 GraphKit* kit = parse_access.kit();
383 Node* load_store = access.raw_access();
384 assert(load_store != NULL, "must pin atomic op");
385 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
386 kit->set_memory(proj, access.alias_idx());
387}
388
389void C2AtomicParseAccess::set_memory() {
390 Node *mem = _kit->memory(_alias_idx);
391 _memory = mem;
392}
393
394Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
395 Node* new_val, const Type* value_type) const {
396 GraphKit* kit = access.kit();
397 MemNode::MemOrd mo = access.mem_node_mo();
398 Node* mem = access.memory();
399
400 Node* adr = access.addr().node();
401 const TypePtr* adr_type = access.addr().type();
402
403 Node* load_store = NULL;
404
405 if (access.is_oop()) {
406#ifdef _LP64
407 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
408 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
409 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
410 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
411 } else
412#endif
413 {
414 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
415 }
416 } else {
417 switch (access.type()) {
418 case T_BYTE: {
419 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
420 break;
421 }
422 case T_SHORT: {
423 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
424 break;
425 }
426 case T_INT: {
427 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
428 break;
429 }
430 case T_LONG: {
431 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
432 break;
433 }
434 default:
435 ShouldNotReachHere();
436 }
437 }
438
439 access.set_raw_access(load_store);
440 pin_atomic_op(access);
441
442#ifdef _LP64
443 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
444 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
445 }
446#endif
447
448 return load_store;
449}
450
451Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
452 Node* new_val, const Type* value_type) const {
453 GraphKit* kit = access.kit();
454 DecoratorSet decorators = access.decorators();
455 MemNode::MemOrd mo = access.mem_node_mo();
456 Node* mem = access.memory();
457 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
458 Node* load_store = NULL;
459 Node* adr = access.addr().node();
460
461 if (access.is_oop()) {
462#ifdef _LP64
463 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
464 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
465 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
466 if (is_weak_cas) {
467 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
468 } else {
469 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
470 }
471 } else
472#endif
473 {
474 if (is_weak_cas) {
475 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
476 } else {
477 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
478 }
479 }
480 } else {
481 switch(access.type()) {
482 case T_BYTE: {
483 if (is_weak_cas) {
484 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
485 } else {
486 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
487 }
488 break;
489 }
490 case T_SHORT: {
491 if (is_weak_cas) {
492 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
493 } else {
494 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
495 }
496 break;
497 }
498 case T_INT: {
499 if (is_weak_cas) {
500 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
501 } else {
502 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
503 }
504 break;
505 }
506 case T_LONG: {
507 if (is_weak_cas) {
508 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
509 } else {
510 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
511 }
512 break;
513 }
514 default:
515 ShouldNotReachHere();
516 }
517 }
518
519 access.set_raw_access(load_store);
520 pin_atomic_op(access);
521
522 return load_store;
523}
524
525Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
526 GraphKit* kit = access.kit();
527 Node* mem = access.memory();
528 Node* adr = access.addr().node();
529 const TypePtr* adr_type = access.addr().type();
530 Node* load_store = NULL;
531
532 if (access.is_oop()) {
533#ifdef _LP64
534 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
535 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
536 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
537 } else
538#endif
539 {
540 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
541 }
542 } else {
543 switch (access.type()) {
544 case T_BYTE:
545 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
546 break;
547 case T_SHORT:
548 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type));
549 break;
550 case T_INT:
551 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type));
552 break;
553 case T_LONG:
554 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
555 break;
556 default:
557 ShouldNotReachHere();
558 }
559 }
560
561 access.set_raw_access(load_store);
562 pin_atomic_op(access);
563
564#ifdef _LP64
565 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
566 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
567 }
568#endif
569
570 return load_store;
571}
572
573Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
574 Node* load_store = NULL;
575 GraphKit* kit = access.kit();
576 Node* adr = access.addr().node();
577 const TypePtr* adr_type = access.addr().type();
578 Node* mem = access.memory();
579
580 switch(access.type()) {
581 case T_BYTE:
582 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
583 break;
584 case T_SHORT:
585 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
586 break;
587 case T_INT:
588 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
589 break;
590 case T_LONG:
591 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
592 break;
593 default:
594 ShouldNotReachHere();
595 }
596
597 access.set_raw_access(load_store);
598 pin_atomic_op(access);
599
600 return load_store;
601}
602
603Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
604 Node* new_val, const Type* value_type) const {
605 C2AccessFence fence(access);
606 resolve_address(access);
607 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
608}
609
610Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
611 Node* new_val, const Type* value_type) const {
612 C2AccessFence fence(access);
613 resolve_address(access);
614 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
615}
616
617Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
618 C2AccessFence fence(access);
619 resolve_address(access);
620 return atomic_xchg_at_resolved(access, new_val, value_type);
621}
622
623Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
624 C2AccessFence fence(access);
625 resolve_address(access);
626 return atomic_add_at_resolved(access, new_val, value_type);
627}
628
629void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
630 // Exclude the header but include array length to copy by 8 bytes words.
631 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
632 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
633 instanceOopDesc::base_offset_in_bytes();
634 // base_off:
635 // 8 - 32-bit VM
636 // 12 - 64-bit VM, compressed klass
637 // 16 - 64-bit VM, normal klass
638 if (base_off % BytesPerLong != 0) {
639 assert(UseCompressedClassPointers, "");
640 if (is_array) {
641 // Exclude length to copy by 8 bytes words.
642 base_off += sizeof(int);
643 } else {
644 // Include klass to copy by 8 bytes words.
645 base_off = instanceOopDesc::klass_offset_in_bytes();
646 }
647 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
648 }
649 Node* src_base = kit->basic_plus_adr(src, base_off);
650 Node* dst_base = kit->basic_plus_adr(dst, base_off);
651
652 // Compute the length also, if needed:
653 Node* countx = size;
654 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
655 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
656
657 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
658
659 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false);
660 ac->set_clonebasic();
661 Node* n = kit->gvn().transform(ac);
662 if (n == ac) {
663 ac->_adr_type = TypeRawPtr::BOTTOM;
664 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
665 } else {
666 kit->set_all_memory(n);
667 }
668}
669
670Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
671 Node*& i_o, Node*& needgc_ctrl,
672 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
673 intx prefetch_lines) const {
674
675 Node* eden_top_adr;
676 Node* eden_end_adr;
677
678 macro->set_eden_pointers(eden_top_adr, eden_end_adr);
679
680 // Load Eden::end. Loop invariant and hoisted.
681 //
682 // Note: We set the control input on "eden_end" and "old_eden_top" when using
683 // a TLAB to work around a bug where these values were being moved across
684 // a safepoint. These are not oops, so they cannot be include in the oop
685 // map, but they can be changed by a GC. The proper way to fix this would
686 // be to set the raw memory state when generating a SafepointNode. However
687 // this will require extensive changes to the loop optimization in order to
688 // prevent a degradation of the optimization.
689 // See comment in memnode.hpp, around line 227 in class LoadPNode.
690 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
691
692 // We need a Region for the loop-back contended case.
693 enum { fall_in_path = 1, contended_loopback_path = 2 };
694 Node *contended_region;
695 Node *contended_phi_rawmem;
696 if (UseTLAB) {
697 contended_region = toobig_false;
698 contended_phi_rawmem = mem;
699 } else {
700 contended_region = new RegionNode(3);
701 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
702 // Now handle the passing-too-big test. We fall into the contended
703 // loop-back merge point.
704 contended_region ->init_req(fall_in_path, toobig_false);
705 contended_phi_rawmem->init_req(fall_in_path, mem);
706 macro->transform_later(contended_region);
707 macro->transform_later(contended_phi_rawmem);
708 }
709
710 // Load(-locked) the heap top.
711 // See note above concerning the control input when using a TLAB
712 Node *old_eden_top = UseTLAB
713 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
714 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
715
716 macro->transform_later(old_eden_top);
717 // Add to heap top to get a new heap top
718 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes);
719 macro->transform_later(new_eden_top);
720 // Check for needing a GC; compare against heap end
721 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
722 macro->transform_later(needgc_cmp);
723 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
724 macro->transform_later(needgc_bol);
725 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
726 macro->transform_later(needgc_iff);
727
728 // Plug the failing-heap-space-need-gc test into the slow-path region
729 Node *needgc_true = new IfTrueNode(needgc_iff);
730 macro->transform_later(needgc_true);
731 needgc_ctrl = needgc_true;
732
733 // No need for a GC. Setup for the Store-Conditional
734 Node *needgc_false = new IfFalseNode(needgc_iff);
735 macro->transform_later(needgc_false);
736
737 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
738 old_eden_top, new_eden_top, prefetch_lines);
739
740 Node* fast_oop = old_eden_top;
741
742 // Store (-conditional) the modified eden top back down.
743 // StorePConditional produces flags for a test PLUS a modified raw
744 // memory state.
745 if (UseTLAB) {
746 Node* store_eden_top =
747 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
748 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
749 macro->transform_later(store_eden_top);
750 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
751 fast_oop_rawmem = store_eden_top;
752 } else {
753 Node* store_eden_top =
754 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
755 new_eden_top, fast_oop/*old_eden_top*/);
756 macro->transform_later(store_eden_top);
757 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
758 macro->transform_later(contention_check);
759 store_eden_top = new SCMemProjNode(store_eden_top);
760 macro->transform_later(store_eden_top);
761
762 // If not using TLABs, check to see if there was contention.
763 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
764 macro->transform_later(contention_iff);
765 Node *contention_true = new IfTrueNode(contention_iff);
766 macro->transform_later(contention_true);
767 // If contention, loopback and try again.
768 contended_region->init_req(contended_loopback_path, contention_true);
769 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
770
771 // Fast-path succeeded with no contention!
772 Node *contention_false = new IfFalseNode(contention_iff);
773 macro->transform_later(contention_false);
774 fast_oop_ctrl = contention_false;
775
776 // Bump total allocated bytes for this thread
777 Node* thread = new ThreadLocalNode();
778 macro->transform_later(thread);
779 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
780 in_bytes(JavaThread::allocated_bytes_offset()));
781 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
782 0, TypeLong::LONG, T_LONG);
783#ifdef _LP64
784 Node* alloc_size = size_in_bytes;
785#else
786 Node* alloc_size = new ConvI2LNode(size_in_bytes);
787 macro->transform_later(alloc_size);
788#endif
789 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
790 macro->transform_later(new_alloc_bytes);
791 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
792 0, new_alloc_bytes, T_LONG);
793 }
794 return fast_oop;
795}
796
797void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
798 // no barrier
799 igvn.replace_node(ac, call);
800}
801