Older binutils don't understand qword ptr
[reactos.git] / reactos / ntoskrnl / ex / i386 / fastinterlck_asm.S
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/ex/i386/fastinterlck_asm.S
5 * PURPOSE: FASTCALL Interlocked Functions
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
7 */
8
9 /* INCLUDES ******************************************************************/
10 #include <ndk/asm.h>
11 #include <internal/i386/asmmacro.S>
12 .intel_syntax noprefix
13
14 /* FUNCTIONS ****************************************************************/
15
16 /*
17 * NOTE: These functions must obey the following rules:
18 * - Acquire locks only on MP systems.
19 * - Be safe at HIGH_LEVEL (no paged access).
20 * - Preserve flags.
21 * - Disable interrups.
22 */
23
24 /*VOID
25 *FASTCALL
26 *ExInterlockedAddLargeStatistic(IN PLARGE_INTEGER Addend,
27 * IN ULONG Increment)
28 */
29 .global @ExInterlockedAddLargeStatistic@8
30 @ExInterlockedAddLargeStatistic@8:
31
32 #ifdef CONFIG_SMP
33 /* Do the addition */
34 lock add [ecx], edx
35
36 /* Check for carry bit and return */
37 jb 1f
38 ret
39
40 1:
41 /* Add carry */
42 lock adc dword ptr [ecx+4], 0
43 #else
44 /* Do the addition and add the carry */
45 add dword ptr [ecx], edx
46 adc dword ptr [ecx+4], 0
47 #endif
48 /* Return */
49 ret
50
51 /*ULONG
52 *FASTCALL
53 *ExfInterlockedAddUlong(IN PULONG Addend,
54 * IN ULONG Increment,
55 * IN PKSPIN_LOCK Lock)
56 */
57 .global @ExfInterlockedAddUlong@12
58 @ExfInterlockedAddUlong@12:
59
60 /* Save flags */
61 pushfd
62
63 #ifdef CONFIG_SMP
64 /* Get lock address */
65 mov eax, [esp+8]
66 .start1:
67 #endif
68 /* Disable interrupts */
69 cli
70
71 /* Acquire lock */
72 ACQUIRE_SPINLOCK(eax, .spin1)
73
74 /* Do the add */
75 mov eax, [ecx]
76 add [ecx], edx
77
78 #ifdef CONFIG_SMP
79 /* Get spinlock address and release it */
80 mov edx, [esp+8]
81 RELEASE_SPINLOCK(edx)
82 #endif
83
84 /* Restore flags and return */
85 popfd
86 ret 4
87
88 #ifdef CONFIG_SMP
89 .spin1:
90 /* Restore flags and spin */
91 popfd
92 pushfd
93 SPIN_ON_LOCK(eax, .start1)
94 #endif
95
96 /*PLIST_ENTRY
97 *FASTCALL
98 *ExfInterlockedInsertHeadList(IN PLIST_ENTRY ListHead,
99 * IN PLIST_ENTRY ListEntry,
100 * IN PKSPIN_LOCK Lock)
101 */
102 .global @ExfInterlockedInsertHeadList@12
103 @ExfInterlockedInsertHeadList@12:
104
105 #ifdef CONFIG_SMP
106 /* Save lock address */
107 push esi
108 mov esi, [esp+8]
109 #endif
110
111 /* Save flags and disable interrupts */
112 pushfd
113 .start2:
114 cli
115
116 /* Acquire lock */
117 ACQUIRE_SPINLOCK(esi, .spin2)
118
119 /* Get list pointer */
120 mov eax, [ecx]
121
122 /* Do the insert */
123 mov [edx], eax
124 mov [edx+4], ecx
125 mov [ecx], edx
126 mov [eax+4], edx
127
128 /* Release lock and restore flags */
129 RELEASE_SPINLOCK(esi)
130 popfd
131
132 #ifdef CONFIG_SMP
133 pop esi
134 #endif
135
136 /* Check if list was empty */
137 xor eax, ecx
138 jz 2f
139
140 /* Return list pointer */
141 xor eax, ecx
142 2:
143 ret 4
144
145 #ifdef CONFIG_SMP
146 .spin2:
147 popfd
148 pushfd
149 SPIN_ON_LOCK(esi, .start2)
150 #endif
151
152 /*PLIST_ENTRY
153 *NTAPI
154 *ExfInterlockedInsertTailList(IN PLIST_ENTRY ListHead,
155 * IN PLIST_ENTRY ListEntry,
156 * IN PKSPIN_LOCK Lock)
157 */
158 .global @ExfInterlockedInsertTailList@12
159 @ExfInterlockedInsertTailList@12:
160
161 #ifdef CONFIG_SMP
162 /* Save lock address */
163 push esi
164 mov esi, [esp+8]
165 #endif
166
167 /* Save flags and disable interrupts */
168 pushfd
169 .start3:
170 cli
171
172 /* Acquire lock */
173 ACQUIRE_SPINLOCK(esi, .spin3)
174
175 /* Get list pointer */
176 mov eax, [ecx+4]
177
178 /* Do the insert */
179 mov [edx], ecx
180 mov [edx+4], eax
181 mov [ecx+4], edx
182 mov [eax], edx
183
184 /* Release lock and restore flags */
185 RELEASE_SPINLOCK(esi)
186 popfd
187
188 #ifdef CONFIG_SMP
189 pop esi
190 #endif
191
192 /* Check if list was empty */
193 xor eax, ecx
194 jz 2f
195
196 /* Return list pointer */
197 xor eax, ecx
198 2:
199 ret 4
200
201 #ifdef CONFIG_SMP
202 .spin3:
203 popfd
204 pushfd
205 SPIN_ON_LOCK(esi, .start3)
206 #endif
207
208 /*PLIST_ENTRY
209 *FASTCALL
210 *ExfInterlockedRemoveHeadList(IN PLIST_ENTRY ListHead,
211 * IN PKSPIN_LOCK Lock)
212 */
213 .global @ExfInterlockedRemoveHeadList@8
214 @ExfInterlockedRemoveHeadList@8:
215
216 /* Save flags and disable interrupts */
217 .start4:
218 pushfd
219 cli
220 ACQUIRE_SPINLOCK(edx, .spin4)
221
222 /* Get list pointer */
223 mov eax, [ecx]
224
225 /* Check if it's empty */
226 cmp eax, ecx
227 je 2f
228
229 /* Get the next entry and do the deletion*/
230 #ifdef CONFIG_SMP
231 push ebx
232 mov ebx, [eax]
233 mov [ecx], ebx
234 mov [ebx+4], ecx
235 #else
236 mov edx, [eax]
237 mov [ecx], edx
238 mov [edx+4], ecx
239 #endif
240
241 /* Release lock */
242 #ifdef CONFIG_SMP
243 RELEASE_SPINLOCK(edx)
244 pop ebx
245 #endif
246
247 /* Restore flags */
248 popfd
249
250 /* Return */
251 ret
252
253 2:
254 /* Release lock */
255 RELEASE_SPINLOCK(edx)
256
257 /* Restore flags */
258 popfd
259
260 /* Return empty list */
261 xor eax, eax
262 ret
263
264 #ifdef CONFIG_SMP
265 .spin4:
266 popfd
267 SPIN_ON_LOCK(edx, .start4)
268 #endif
269
270 /*PSINGLE_LIST_ENTRY
271 *FASTCALL
272 *ExfInterlockedPopEntryList(IN PSINGLE_LIST_ENTRY ListHead,
273 * IN PKSPIN_LOCK Lock)
274 */
275 .global @ExfInterlockedPopEntryList@8
276 @ExfInterlockedPopEntryList@8:
277
278 /* Save flags and disable interrupts */
279 .start5:
280 pushfd
281 cli
282 ACQUIRE_SPINLOCK(edx, .spin5)
283
284 /* Get list pointer */
285 mov eax, [ecx]
286
287 /* Check if it's empty */
288 or eax, eax
289 je 3f
290
291 /* Get next entry and do deletion */
292 #ifdef CONFIG_SMP
293 push edx
294 #endif
295 mov edx, [eax]
296 mov [ecx], edx
297 #ifdef CONFIG_SMP
298 pop edx
299 #endif
300
301 2:
302 /* Release lock */
303 RELEASE_SPINLOCK(edx)
304
305 /* Restore flags */
306 popfd
307
308 /* Return */
309 ret
310
311 3:
312 /* Return empty list */
313 xor eax, eax
314 jmp 2b
315
316 #ifdef CONFIG_SMP
317 .spin5:
318 popfd
319 SPIN_ON_LOCK(edx, .start5)
320 #endif
321
322 /*PSINGLE_LIST_ENTRY
323 *NTAPI
324 *ExfInterlockedPushEntryList(IN PSINGLE_LIST_ENTRY ListHead,
325 * IN PSINGLE_LIST_ENTRY ListEntry,
326 * IN PKSPIN_LOCK Lock)
327 */
328 .global @ExfInterlockedPushEntryList@12
329 @ExfInterlockedPushEntryList@12:
330
331 /* Save flags */
332 pushfd
333
334 /* Save lock pointer */
335 #ifdef CONFIG_SMP
336 push edx
337 mov edx, [esp+12]
338 #endif
339
340 /* Disable interrupts */
341 .start6:
342 cli
343 #ifdef CONFIG_SMP
344 ACQUIRE_SPINLOCK(edx, .spin6)
345 pop edx
346 #endif
347
348 /* Get list pointer */
349 mov eax, [ecx]
350
351 /* Do push */
352 mov [edx], eax
353 mov [ecx], edx
354
355 /* Release lock */
356 #ifdef CONFIG_SMP
357 mov edx, [esp+8]
358 RELEASE_SPINLOCK(edx)
359 #endif
360
361 /* Restore flags */
362 popfd
363
364 /* Return */
365 ret 4
366
367 #ifdef CONFIG_SMP
368 .spin6:
369 pop edx
370 popfd
371 pushfd
372 push edx
373 mov edx, [esp+12]
374 SPIN_ON_LOCK(edx, .start6)
375 #endif
376
377 /*PSINGLE_LIST_ENTRY
378 *NTAPI
379 *ExInterlockedPopEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
380 * IN PKSPIN_LOCK Lock)
381 */
382 .global @ExInterlockedPopEntrySList@8
383 .global @InterlockedPopEntrySList@4
384 @ExInterlockedPopEntrySList@8:
385 @InterlockedPopEntrySList@4:
386
387 /* Save registers */
388 push ebx
389 push ebp
390
391 /* Pointer to list */
392 mov ebp, ecx
393
394 /* Get sequence number and link pointer */
395 mov edx, [ebp+4]
396 mov eax, [ebp]
397
398 1:
399 /* Check if the list is empty */
400 or eax, eax
401 jz 2f
402
403 /* Copy sequence number and adjust it */
404 lea ecx, [edx-1]
405
406 /* Get next pointer and do the exchange */
407 mov ebx, [eax]
408 LOCK cmpxchg8b [ebp]
409 jnz 1b
410
411 /* Restore registers and return */
412 2:
413 pop ebp
414 pop ebx
415 ret
416
417 /*PSINGLE_LIST_ENTRY
418 *NTAPI
419 *ExInterlockedPushEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
420 * IN PSINGLE_LIST_ENTRY ListEntry,
421 * IN PKSPIN_LOCK Lock)
422 */
423 .global @ExInterlockedPushEntrySList@12
424 @ExInterlockedPushEntrySList@12:
425
426 /* So we can fall through below */
427 pop [esp]
428
429 .global @InterlockedPushEntrySList@8
430 @InterlockedPushEntrySList@8:
431
432 /* Save registers */
433 push ebx
434 push ebp
435
436 /* Pointer to list */
437 mov ebp, ecx
438 mov ebx, edx
439
440 /* Get sequence number and link pointer */
441 mov edx, [ebp+4]
442 mov eax, [ebp]
443
444 1:
445 /* Set link pointer */
446 mov [ebx], eax
447
448 /* Copy sequence number and adjust it */
449 lea ecx, [edx+0x10001]
450
451 /* Do the exchange */
452 LOCK cmpxchg8b [ebp]
453 jnz 1b
454
455 /* Restore registers and return */
456 2:
457 pop ebp
458 pop ebx
459 ret
460
461 /*PSINGLE_LIST_ENTRY
462 *NTAPI
463 *ExInterlockedFlushSList(IN PSINGLE_LIST_ENTRY ListHead)
464 */
465 .global @ExInterlockedFlushSList@4
466 @ExInterlockedFlushSList@4:
467
468 /* Save registers */
469 push ebx
470 push ebp
471
472 /* Clear ebx */
473 xor ebx, ebx
474
475 /* Pointer to list */
476 mov ebp, ecx
477
478 /* Get sequence number and link pointer */
479 mov edx, [ebp+4]
480 mov eax, [ebp]
481
482 1:
483 /* Check if the list is empty */
484 or eax, eax
485 jz 2f
486
487 /* Clear sequence and pointer */
488 mov ecx, edx
489 mov cx, bx
490
491 /* Do the exchange */
492 LOCK cmpxchg8b [ebp]
493 jnz 1b
494
495 /* Restore registers and return */
496 2:
497 pop ebp
498 pop ebx
499 ret
500
501 /*INTERLOCKED_RESULT
502 *FASTCALL
503 *Exfi386InterlockedIncrementLong(IN PLONG Addend)
504 */
505 .global @Exfi386InterlockedIncrementLong@4
506 @Exfi386InterlockedIncrementLong@4:
507
508 /* Do the op */
509 LOCK add dword ptr [ecx], 1
510
511 /* Return */
512 lahf
513 and eax, EFLAG_SELECT
514 ret
515
516 /*INTERLOCKED_RESULT
517 *FASTCALL
518 *Exfi386InterlockedDecrementLong(IN PLONG Addend)
519 */
520 .global @Exfi386InterlockedDecrementLong@4
521 @Exfi386InterlockedDecrementLong@4:
522
523 /* Do the op */
524 LOCK sub dword ptr [ecx], 1
525
526 /* Return */
527 lahf
528 and eax, EFLAG_SELECT
529 ret
530
531 /*ULONG
532 *FASTCALL
533 *Exfi386InterlockedExchangeUlong(IN PULONG Taget,
534 * IN ULONG Value)
535 */
536 .global @Exfi386InterlockedExchangeUlong@8
537 .global @InterlockedExchange@8
538 @InterlockedExchange@8:
539 @Exfi386InterlockedExchangeUlong@8:
540
541 #ifdef CONFIG_SMP
542 /* On MP, do the exchange */
543 xchg [ecx], edx
544 mov eax, edx
545 #else
546 /* On UP, use cmpxchg */
547 mov eax, [ecx]
548 1:
549 cmpxchg [ecx], edx
550 jnz 1b
551 #endif
552
553 /* Return */
554 ret
555
556 /*ULONG
557 *FASTCALL
558 *InterlockedIncrement(IN PLONG Addend)
559 */
560 .global @InterlockedIncrement@4
561 @InterlockedIncrement@4:
562
563 /* Do the op */
564 mov eax, 1
565 LOCK xadd dword ptr [ecx], eax
566
567 /* Return */
568 inc eax
569 ret
570
571 /*ULONG
572 *FASTCALL
573 *InterlockedDecrement(IN PLONG Addend)
574 */
575 .global @InterlockedDecrement@4
576 @InterlockedDecrement@4:
577
578 /* Do the op */
579 mov eax, -1
580 LOCK xadd dword ptr [ecx], eax
581
582 /* Return */
583 dec eax
584 ret
585
586 /*PVOID
587 *FASTCALL
588 *InterlockedCompareExchange(IN OUT PVOID *Destination,
589 * IN PVOID Exchange,
590 * IN PVOID Comperand)
591 */
592 .global @InterlockedCompareExchange@12
593 @InterlockedCompareExchange@12:
594
595 /* Get comperand */
596 mov eax, [esp+4]
597
598 /* Do the op */
599 LOCK cmpxchg dword ptr [ecx], edx
600
601 /* Return */
602 ret
603
604 /*PVOID
605 *FASTCALL
606 *ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
607 * IN PLONGLONG Exchange,
608 * IN PLONGLONG Comperand)
609 */
610 .global @ExfInterlockedCompareExchange64@12
611 @ExfInterlockedCompareExchange64@12:
612
613 /* Save registers */
614 push ebx
615 push ebp
616
617 /* Get desination pointer, exchange value and comperand value/address */
618 mov ebp, ecx
619 mov ebx, [edx]
620 mov ecx, [edx+4]
621 mov edx, [esp+12]
622 mov eax, [edx]
623 mov edx, [edx+4]
624
625 /* Do the op */
626 LOCK cmpxchg8b [ebp]
627
628 /* Restore volatiles */
629 pop ebp
630 pop ebx
631
632 /* Return */
633 ret 4
634
635 /*PVOID
636 *FASTCALL
637 *ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
638 * IN PLONGLONG Exchange,
639 * IN PLONGLONG Comperand,
640 * IN PKSPIN_LOCK Lock)
641 */
642 .global @ExInterlockedCompareExchange64@16
643 @ExInterlockedCompareExchange64@16:
644
645 /* Save registers */
646 push ebp
647 push ebp
648
649 /* Get desination pointer, exchange value and comperand value/address */
650 mov ebp, ecx
651 mov ebx, [edx]
652 mov ecx, [edx+4]
653 mov edx, [esp+12]
654 mov eax, [edx]
655 mov edx, [edx+4]
656
657 /* Do the op */
658 LOCK cmpxchg8b [ebp]
659
660 /* Restore volatiles */
661 pop ebp
662 pop ebx
663
664 /* Return */
665 ret 8
666
667 /*PVOID
668 *FASTCALL
669 *InterlockedExchangeAdd(IN OUT PLONG Addend,
670 * IN LONG Increment)
671 */
672 .global @InterlockedExchangeAdd@8
673 @InterlockedExchangeAdd@8:
674
675 /* Do the op */
676 LOCK xadd dword ptr [ecx], edx
677
678 /* Return */
679 mov eax, edx
680 ret
681
682 /* EOF */