1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "textflag.h"
8
9 // For more details about how various memory models are
10 // enforced on POWER, the following paper provides more
11 // details about how they enforce C/C++ like models. This
12 // gives context about why the strange looking code
13 // sequences below work.
14 //
15 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
16
17 // uint32 ·Load(uint32 volatile* ptr)
18 TEXT ·Load(SB),NOSPLIT|NOFRAME,$-8-12
19 MOVD ptr+0(FP), R3
20 SYNC
21 MOVWZ 0(R3), R3
22 CMPW R3, R3, CR7
23 BC 4, 30, 1(PC) // bne- cr7,0x4
24 ISYNC
25 MOVW R3, ret+8(FP)
26 RET
27
28 // uint8 ·Load8(uint8 volatile* ptr)
29 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$-8-9
30 MOVD ptr+0(FP), R3
31 SYNC
32 MOVBZ 0(R3), R3
33 CMP R3, R3, CR7
34 BC 4, 30, 1(PC) // bne- cr7,0x4
35 ISYNC
36 MOVB R3, ret+8(FP)
37 RET
38
39 // uint64 ·Load64(uint64 volatile* ptr)
40 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$-8-16
41 MOVD ptr+0(FP), R3
42 SYNC
43 MOVD 0(R3), R3
44 CMP R3, R3, CR7
45 BC 4, 30, 1(PC) // bne- cr7,0x4
46 ISYNC
47 MOVD R3, ret+8(FP)
48 RET
49
50 // void *·Loadp(void *volatile *ptr)
51 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$-8-16
52 MOVD ptr+0(FP), R3
53 SYNC
54 MOVD 0(R3), R3
55 CMP R3, R3, CR7
56 BC 4, 30, 1(PC) // bne- cr7,0x4
57 ISYNC
58 MOVD R3, ret+8(FP)
59 RET
60
61 // uint32 ·LoadAcq(uint32 volatile* ptr)
62 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$-8-12
63 MOVD ptr+0(FP), R3
64 MOVWZ 0(R3), R3
65 CMPW R3, R3, CR7
66 BC 4, 30, 1(PC) // bne- cr7, 0x4
67 ISYNC
68 MOVW R3, ret+8(FP)
69 RET
70
71 // uint64 ·LoadAcq64(uint64 volatile* ptr)
72 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$-8-16
73 MOVD ptr+0(FP), R3
74 MOVD 0(R3), R3
75 CMP R3, R3, CR7
76 BC 4, 30, 1(PC) // bne- cr7, 0x4
77 ISYNC
78 MOVD R3, ret+8(FP)
79 RET
80
81 // func Cas(ptr *int32, old, new int32) bool
82 // Atomically:
83 // if *ptr == old {
84 // *ptr = new
85 // return true
86 // } else {
87 // return false
88 // }
89 TEXT ·Cas(SB), NOSPLIT, $0-17
90 MOVD ptr+0(FP), R3
91 MOVWZ old+8(FP), R4
92 MOVWZ new+12(FP), R5
93 LWSYNC
94 cas_again:
95 LWAR (R3), R6
96 CMPW R6, R4
97 BNE cas_fail
98 STWCCC R5, (R3)
99 BNE cas_again
100 MOVD $1, R3
101 LWSYNC
102 MOVB R3, ret+16(FP)
103 RET
104 cas_fail:
105 LWSYNC
106 MOVB R0, ret+16(FP)
107 RET
108
109 // func Cas64(ptr *uint64, old, new uint64) bool
110 // Atomically:
111 // if *ptr == old {
112 // *ptr = new
113 // return true
114 // } else {
115 // return false
116 // }
117 TEXT ·Cas64(SB), NOSPLIT, $0-25
118 MOVD ptr+0(FP), R3
119 MOVD old+8(FP), R4
120 MOVD new+16(FP), R5
121 LWSYNC
122 cas64_again:
123 LDAR (R3), R6
124 CMP R6, R4
125 BNE cas64_fail
126 STDCCC R5, (R3)
127 BNE cas64_again
128 MOVD $1, R3
129 LWSYNC
130 MOVB R3, ret+24(FP)
131 RET
132 cas64_fail:
133 LWSYNC
134 MOVB R0, ret+24(FP)
135 RET
136
137 TEXT ·CasRel(SB), NOSPLIT, $0-17
138 MOVD ptr+0(FP), R3
139 MOVWZ old+8(FP), R4
140 MOVWZ new+12(FP), R5
141 LWSYNC
142 cas_again:
143 LWAR (R3), $0, R6 // 0 = Mutex release hint
144 CMPW R6, R4
145 BNE cas_fail
146 STWCCC R5, (R3)
147 BNE cas_again
148 MOVD $1, R3
149 MOVB R3, ret+16(FP)
150 RET
151 cas_fail:
152 MOVB R0, ret+16(FP)
153 RET
154
155 TEXT ·Casint32(SB), NOSPLIT, $0-17
156 BR ·Cas(SB)
157
158 TEXT ·Casint64(SB), NOSPLIT, $0-25
159 BR ·Cas64(SB)
160
161 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
162 BR ·Cas64(SB)
163
164 TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
165 BR ·Load64(SB)
166
167 TEXT ·LoadAcquintptr(SB), NOSPLIT|NOFRAME, $0-16
168 BR ·LoadAcq64(SB)
169
170 TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
171 BR ·Load64(SB)
172
173 TEXT ·Storeint32(SB), NOSPLIT, $0-12
174 BR ·Store(SB)
175
176 TEXT ·Storeint64(SB), NOSPLIT, $0-16
177 BR ·Store64(SB)
178
179 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
180 BR ·Store64(SB)
181
182 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
183 BR ·StoreRel64(SB)
184
185 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
186 BR ·Xadd64(SB)
187
188 TEXT ·Loadint32(SB), NOSPLIT, $0-12
189 BR ·Load(SB)
190
191 TEXT ·Loadint64(SB), NOSPLIT, $0-16
192 BR ·Load64(SB)
193
194 TEXT ·Xaddint32(SB), NOSPLIT, $0-20
195 BR ·Xadd(SB)
196
197 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
198 BR ·Xadd64(SB)
199
200 // func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
201 // Atomically:
202 // if *ptr == old {
203 // *ptr = new
204 // return true
205 // } else {
206 // return false
207 // }
208 TEXT ·Casp1(SB), NOSPLIT, $0-25
209 BR ·Cas64(SB)
210
211 // uint32 xadd(uint32 volatile *ptr, int32 delta)
212 // Atomically:
213 // *val += delta;
214 // return *val;
215 TEXT ·Xadd(SB), NOSPLIT, $0-20
216 MOVD ptr+0(FP), R4
217 MOVW delta+8(FP), R5
218 LWSYNC
219 LWAR (R4), R3
220 ADD R5, R3
221 STWCCC R3, (R4)
222 BNE -3(PC)
223 LWSYNC
224 MOVW R3, ret+16(FP)
225 RET
226
227 // uint64 Xadd64(uint64 volatile *val, int64 delta)
228 // Atomically:
229 // *val += delta;
230 // return *val;
231 TEXT ·Xadd64(SB), NOSPLIT, $0-24
232 MOVD ptr+0(FP), R4
233 MOVD delta+8(FP), R5
234 LWSYNC
235 LDAR (R4), R3
236 ADD R5, R3
237 STDCCC R3, (R4)
238 BNE -3(PC)
239 LWSYNC
240 MOVD R3, ret+16(FP)
241 RET
242
243 // uint8 Xchg(ptr *uint8, new uint8)
244 // Atomically:
245 // old := *ptr;
246 // *ptr = new;
247 // return old;
248 TEXT ·Xchg8(SB), NOSPLIT, $0-17
249 MOVD ptr+0(FP), R4
250 MOVB new+8(FP), R5
251 LWSYNC
252 LBAR (R4), R3
253 STBCCC R5, (R4)
254 BNE -2(PC)
255 ISYNC
256 MOVB R3, ret+16(FP)
257 RET
258
259 // uint32 Xchg(ptr *uint32, new uint32)
260 // Atomically:
261 // old := *ptr;
262 // *ptr = new;
263 // return old;
264 TEXT ·Xchg(SB), NOSPLIT, $0-20
265 MOVD ptr+0(FP), R4
266 MOVW new+8(FP), R5
267 LWSYNC
268 LWAR (R4), R3
269 STWCCC R5, (R4)
270 BNE -2(PC)
271 ISYNC
272 MOVW R3, ret+16(FP)
273 RET
274
275 // uint64 Xchg64(ptr *uint64, new uint64)
276 // Atomically:
277 // old := *ptr;
278 // *ptr = new;
279 // return old;
280 TEXT ·Xchg64(SB), NOSPLIT, $0-24
281 MOVD ptr+0(FP), R4
282 MOVD new+8(FP), R5
283 LWSYNC
284 LDAR (R4), R3
285 STDCCC R5, (R4)
286 BNE -2(PC)
287 ISYNC
288 MOVD R3, ret+16(FP)
289 RET
290
291 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
292 BR ·Xchg(SB)
293
294 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
295 BR ·Xchg64(SB)
296
297 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
298 BR ·Xchg64(SB)
299
300 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
301 BR ·Store64(SB)
302
303 TEXT ·Store(SB), NOSPLIT, $0-12
304 MOVD ptr+0(FP), R3
305 MOVW val+8(FP), R4
306 SYNC
307 MOVW R4, 0(R3)
308 RET
309
310 TEXT ·Store8(SB), NOSPLIT, $0-9
311 MOVD ptr+0(FP), R3
312 MOVB val+8(FP), R4
313 SYNC
314 MOVB R4, 0(R3)
315 RET
316
317 TEXT ·Store64(SB), NOSPLIT, $0-16
318 MOVD ptr+0(FP), R3
319 MOVD val+8(FP), R4
320 SYNC
321 MOVD R4, 0(R3)
322 RET
323
324 TEXT ·StoreRel(SB), NOSPLIT, $0-12
325 MOVD ptr+0(FP), R3
326 MOVW val+8(FP), R4
327 LWSYNC
328 MOVW R4, 0(R3)
329 RET
330
331 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
332 MOVD ptr+0(FP), R3
333 MOVD val+8(FP), R4
334 LWSYNC
335 MOVD R4, 0(R3)
336 RET
337
338 // void ·Or8(byte volatile*, byte);
339 TEXT ·Or8(SB), NOSPLIT, $0-9
340 MOVD ptr+0(FP), R3
341 MOVBZ val+8(FP), R4
342 LWSYNC
343 again:
344 LBAR (R3), R6
345 OR R4, R6
346 STBCCC R6, (R3)
347 BNE again
348 LWSYNC
349 RET
350
351 // void ·And8(byte volatile*, byte);
352 TEXT ·And8(SB), NOSPLIT, $0-9
353 MOVD ptr+0(FP), R3
354 MOVBZ val+8(FP), R4
355 LWSYNC
356 again:
357 LBAR (R3), R6
358 AND R4, R6
359 STBCCC R6, (R3)
360 BNE again
361 LWSYNC
362 RET
363
364 // func Or(addr *uint32, v uint32)
365 TEXT ·Or(SB), NOSPLIT, $0-12
366 MOVD ptr+0(FP), R3
367 MOVW val+8(FP), R4
368 LWSYNC
369 again:
370 LWAR (R3), R6
371 OR R4, R6
372 STWCCC R6, (R3)
373 BNE again
374 LWSYNC
375 RET
376
377 // func And(addr *uint32, v uint32)
378 TEXT ·And(SB), NOSPLIT, $0-12
379 MOVD ptr+0(FP), R3
380 MOVW val+8(FP), R4
381 LWSYNC
382 again:
383 LWAR (R3),R6
384 AND R4, R6
385 STWCCC R6, (R3)
386 BNE again
387 LWSYNC
388 RET
389
390 // func Or32(addr *uint32, v uint32) old uint32
391 TEXT ·Or32(SB), NOSPLIT, $0-20
392 MOVD ptr+0(FP), R3
393 MOVW val+8(FP), R4
394 LWSYNC
395 again:
396 LWAR (R3), R6
397 OR R4, R6, R7
398 STWCCC R7, (R3)
399 BNE again
400 LWSYNC
401 MOVW R6, ret+16(FP)
402 RET
403
404 // func And32(addr *uint32, v uint32) old uint32
405 TEXT ·And32(SB), NOSPLIT, $0-20
406 MOVD ptr+0(FP), R3
407 MOVW val+8(FP), R4
408 LWSYNC
409 again:
410 LWAR (R3),R6
411 AND R4, R6, R7
412 STWCCC R7, (R3)
413 BNE again
414 LWSYNC
415 MOVW R6, ret+16(FP)
416 RET
417
418 // func Or64(addr *uint64, v uint64) old uint64
419 TEXT ·Or64(SB), NOSPLIT, $0-24
420 MOVD ptr+0(FP), R3
421 MOVD val+8(FP), R4
422 LWSYNC
423 again:
424 LDAR (R3), R6
425 OR R4, R6, R7
426 STDCCC R7, (R3)
427 BNE again
428 LWSYNC
429 MOVD R6, ret+16(FP)
430 RET
431
432 // func And64(addr *uint64, v uint64) old uint64
433 TEXT ·And64(SB), NOSPLIT, $0-24
434 MOVD ptr+0(FP), R3
435 MOVD val+8(FP), R4
436 LWSYNC
437 again:
438 LDAR (R3),R6
439 AND R4, R6, R7
440 STDCCC R7, (R3)
441 BNE again
442 LWSYNC
443 MOVD R6, ret+16(FP)
444 RET
445
446 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
447 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
448 JMP ·And64(SB)
449
450 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
451 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
452 JMP ·Or64(SB)
453
View as plain text