2 Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3 Microsoft Visual C++ functions. Originally developed for the ReactOS
4 (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
7 Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
9 Permission is hereby granted, free of charge, to any person obtaining a
10 copy of this software and associated documentation files (the "Software"),
11 to deal in the Software without restriction, including without limitation
12 the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 and/or sell copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following conditions:
16 The above copyright notice and this permission notice shall be included in
17 all copies or substantial portions of the Software.
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 DEALINGS IN THE SOFTWARE.
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
32 FIXME: review all "memory" clobbers, add/remove to match Visual C++
33 behavior: some "obvious" memory barriers are not present in the Visual C++
34 implementation - e.g. __stosX; on the other hand, some memory barriers that
35 *are* present could have been missed
39 NOTE: this is a *compatibility* header. Some functions may look wrong at
40 first, but they're only "as wrong" as they would be on Visual C++. Our
41 priority is compatibility
43 NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44 constraints and the uses of __volatile__ out of my... hat. Do not touch
45 them. I hate cargo cult programming
47 NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48 barriers aren't there in Visual C++ (e.g. __stosX)
50 NOTE: review all intrinsics with a return value, add/remove __volatile__
51 where necessary. If an intrinsic whose value is ignored generates a no-op
52 under Visual C++, __volatile__ must be omitted; if it always generates code
53 (for example, if it has side effects), __volatile__ must be specified. GCC
54 will only optimize out non-volatile asm blocks with outputs, so input-only
55 blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56 and follow Visual C++ behavior
58 NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59 atomic operations. Test the version like this:
61 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
64 Pay attention to the type of barrier. Make it match with what Visual C++
65 would use in the same case
68 /*** Stack frame juggling ***/
69 #define _ReturnAddress() (__builtin_return_address(0))
70 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
71 /* TODO: __getcallerseflags but how??? */
74 /*** Atomic operations ***/
76 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
77 #define _ReadWriteBarrier() __sync_synchronize()
79 static void __inline__
__attribute__((always_inline
)) _MemoryBarrier(void)
81 __asm__
__volatile__("" : : : "memory");
83 #define _ReadWriteBarrier() _MemoryBarrier()
86 /* BUGBUG: GCC only supports full barriers */
87 #define _ReadBarrier _ReadWriteBarrier
88 #define _WriteBarrier _ReadWriteBarrier
90 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
92 static __inline__
__attribute__((always_inline
)) char _InterlockedCompareExchange8(volatile char * const Destination
, const char Exchange
, const char Comperand
)
94 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
97 static __inline__
__attribute__((always_inline
)) short _InterlockedCompareExchange16(volatile short * const Destination
, const short Exchange
, const short Comperand
)
99 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
102 static __inline__
__attribute__((always_inline
)) long _InterlockedCompareExchange(volatile long * const Destination
, const long Exchange
, const long Comperand
)
104 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
107 static __inline__
__attribute__((always_inline
)) long long _InterlockedCompareExchange64(volatile long long * const Destination
, const long long Exchange
, const long long Comperand
)
109 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
112 static __inline__
__attribute__((always_inline
)) void * _InterlockedCompareExchangePointer(void * volatile * const Destination
, void * const Exchange
, void * const Comperand
)
114 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
117 static __inline__
__attribute__((always_inline
)) long _InterlockedExchange(volatile long * const Target
, const long Value
)
119 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
120 __sync_synchronize();
121 return __sync_lock_test_and_set(Target
, Value
);
124 #if defined(_M_AMD64)
125 static __inline__
__attribute__((always_inline
)) long long _InterlockedExchange64(volatile long long * const Target
, const long long Value
)
127 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
128 __sync_synchronize();
129 return __sync_lock_test_and_set(Target
, Value
);
133 static __inline__
__attribute__((always_inline
)) void * _InterlockedExchangePointer(void * volatile * const Target
, void * const Value
)
136 __sync_synchronize();
137 return __sync_lock_test_and_set(Target
, Value
);
140 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd16(volatile short * const Addend
, const short Value
)
142 return __sync_fetch_and_add(Addend
, Value
);
145 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd(volatile long * const Addend
, const long Value
)
147 return __sync_fetch_and_add(Addend
, Value
);
150 static __inline__
__attribute__((always_inline
)) char _InterlockedAnd8(volatile char * const value
, const char mask
)
152 return __sync_fetch_and_and(value
, mask
);
155 static __inline__
__attribute__((always_inline
)) short _InterlockedAnd16(volatile short * const value
, const short mask
)
157 return __sync_fetch_and_and(value
, mask
);
160 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd(volatile long * const value
, const long mask
)
162 return __sync_fetch_and_and(value
, mask
);
165 #if defined(_M_AMD64)
166 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd64(volatile long long * const value
, const long long mask
)
168 return __sync_fetch_and_and(value
, mask
);
172 static __inline__
__attribute__((always_inline
)) char _InterlockedOr8(volatile char * const value
, const char mask
)
174 return __sync_fetch_and_or(value
, mask
);
177 static __inline__
__attribute__((always_inline
)) short _InterlockedOr16(volatile short * const value
, const short mask
)
179 return __sync_fetch_and_or(value
, mask
);
182 static __inline__
__attribute__((always_inline
)) long _InterlockedOr(volatile long * const value
, const long mask
)
184 return __sync_fetch_and_or(value
, mask
);
187 #if defined(_M_AMD64)
188 static __inline__
__attribute__((always_inline
)) long _InterlockedOr64(volatile long long * const value
, const long long mask
)
190 return __sync_fetch_and_or(value
, mask
);
194 static __inline__
__attribute__((always_inline
)) char _InterlockedXor8(volatile char * const value
, const char mask
)
196 return __sync_fetch_and_xor(value
, mask
);
199 static __inline__
__attribute__((always_inline
)) short _InterlockedXor16(volatile short * const value
, const short mask
)
201 return __sync_fetch_and_xor(value
, mask
);
204 static __inline__
__attribute__((always_inline
)) long _InterlockedXor(volatile long * const value
, const long mask
)
206 return __sync_fetch_and_xor(value
, mask
);
211 static __inline__
__attribute__((always_inline
)) char _InterlockedCompareExchange8(volatile char * const Destination
, const char Exchange
, const char Comperand
)
213 char retval
= Comperand
;
214 __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
218 static __inline__
__attribute__((always_inline
)) short _InterlockedCompareExchange16(volatile short * const Destination
, const short Exchange
, const short Comperand
)
220 short retval
= Comperand
;
221 __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
225 static __inline__
__attribute__((always_inline
)) long _InterlockedCompareExchange(volatile long * const Destination
, const long Exchange
, const long Comperand
)
227 long retval
= Comperand
;
228 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
232 static __inline__
__attribute__((always_inline
)) long long _InterlockedCompareExchange64(volatile long long * const Destination
, const long long Exchange
, const long long Comperand
)
234 long long retval
= Comperand
;
238 "cmpxchg8b %[Destination]" :
239 [retval
] "+A" (retval
) :
240 [Destination
] "m" (*Destination
),
241 "b" ((unsigned long)((Exchange
>> 0) & 0xFFFFFFFF)),
242 "c" ((unsigned long)((Exchange
>> 32) & 0xFFFFFFFF)) :
249 static __inline__
__attribute__((always_inline
)) void * _InterlockedCompareExchangePointer(void * volatile * const Destination
, void * const Exchange
, void * const Comperand
)
251 void * retval
= (void *)Comperand
;
252 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "=a" (retval
) : "[retval]" (retval
), [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
256 static __inline__
__attribute__((always_inline
)) long _InterlockedExchange(volatile long * const Target
, const long Value
)
259 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
263 static __inline__
__attribute__((always_inline
)) void * _InterlockedExchangePointer(void * volatile * const Target
, void * const Value
)
265 void * retval
= Value
;
266 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
270 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd16(volatile short * const Addend
, const short Value
)
273 __asm__("lock; xaddw %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
277 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd(volatile long * const Addend
, const long Value
)
280 __asm__("lock; xaddl %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
284 static __inline__
__attribute__((always_inline
)) char _InterlockedAnd8(volatile char * const value
, const char mask
)
294 y
= _InterlockedCompareExchange8(value
, x
& mask
, x
);
301 static __inline__
__attribute__((always_inline
)) short _InterlockedAnd16(volatile short * const value
, const short mask
)
311 y
= _InterlockedCompareExchange16(value
, x
& mask
, x
);
318 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd(volatile long * const value
, const long mask
)
328 y
= _InterlockedCompareExchange(value
, x
& mask
, x
);
335 static __inline__
__attribute__((always_inline
)) char _InterlockedOr8(volatile char * const value
, const char mask
)
345 y
= _InterlockedCompareExchange8(value
, x
| mask
, x
);
352 static __inline__
__attribute__((always_inline
)) short _InterlockedOr16(volatile short * const value
, const short mask
)
362 y
= _InterlockedCompareExchange16(value
, x
| mask
, x
);
369 static __inline__
__attribute__((always_inline
)) long _InterlockedOr(volatile long * const value
, const long mask
)
379 y
= _InterlockedCompareExchange(value
, x
| mask
, x
);
386 static __inline__
__attribute__((always_inline
)) char _InterlockedXor8(volatile char * const value
, const char mask
)
396 y
= _InterlockedCompareExchange8(value
, x
^ mask
, x
);
403 static __inline__
__attribute__((always_inline
)) short _InterlockedXor16(volatile short * const value
, const short mask
)
413 y
= _InterlockedCompareExchange16(value
, x
^ mask
, x
);
420 static __inline__
__attribute__((always_inline
)) long _InterlockedXor(volatile long * const value
, const long mask
)
430 y
= _InterlockedCompareExchange(value
, x
^ mask
, x
);
439 static __inline__
__attribute__((always_inline
)) long _InterlockedAddLargeStatistic(volatile long long * const Addend
, const long Value
)
443 "lock; add %[Value], %[Lo32];"
445 "lock; adc $0, %[Hi32];"
447 [Lo32
] "=m" (*((volatile long *)(Addend
) + 0)), [Hi32
] "=m" (*((volatile long *)(Addend
) + 1)) :
454 static __inline__
__attribute__((always_inline
)) long _InterlockedDecrement(volatile long * const lpAddend
)
456 return _InterlockedExchangeAdd(lpAddend
, -1) - 1;
459 static __inline__
__attribute__((always_inline
)) long _InterlockedIncrement(volatile long * const lpAddend
)
461 return _InterlockedExchangeAdd(lpAddend
, 1) + 1;
464 static __inline__
__attribute__((always_inline
)) long _InterlockedDecrement16(volatile short * const lpAddend
)
466 return _InterlockedExchangeAdd16(lpAddend
, -1) - 1;
469 static __inline__
__attribute__((always_inline
)) long _InterlockedIncrement16(volatile short * const lpAddend
)
471 return _InterlockedExchangeAdd16(lpAddend
, 1) + 1;
474 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandreset(volatile long * a
, const long b
)
476 unsigned char retval
;
477 __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval
] "=r" (retval
), [a
] "=m" (a
) : [b
] "Ir" (b
) : "memory");
481 #if defined(_M_AMD64)
482 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandreset64(volatile long long * a
, const long long b
)
484 unsigned char retval
;
485 __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval
] "=r" (retval
), [a
] "=m" (a
) : [b
] "Ir" (b
) : "memory");
490 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandset(volatile long * a
, const long b
)
492 unsigned char retval
;
493 __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval
] "=r" (retval
), [a
] "=m" (a
) : [b
] "Ir" (b
) : "memory");
497 #if defined(_M_AMD64)
498 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandset64(volatile long long * a
, const long long b
)
500 unsigned char retval
;
501 __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval
] "=r" (retval
), [a
] "=m" (a
) : [b
] "Ir" (b
) : "memory");
506 /*** String operations ***/
507 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
508 static __inline__
__attribute__((always_inline
)) void __stosb(unsigned char * Dest
, const unsigned char Data
, size_t Count
)
513 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
514 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
518 static __inline__
__attribute__((always_inline
)) void __stosw(unsigned short * Dest
, const unsigned short Data
, size_t Count
)
523 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
524 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
528 static __inline__
__attribute__((always_inline
)) void __stosd(unsigned long * Dest
, const unsigned long Data
, size_t Count
)
533 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
534 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
538 static __inline__
__attribute__((always_inline
)) void __movsb(unsigned char * Destination
, const unsigned char * Source
, size_t Count
)
543 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
544 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
548 static __inline__
__attribute__((always_inline
)) void __movsw(unsigned short * Destination
, const unsigned short * Source
, size_t Count
)
553 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
554 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
558 static __inline__
__attribute__((always_inline
)) void __movsd(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
563 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
564 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
568 #if defined(_M_AMD64)
569 /*** GS segment addressing ***/
571 static __inline__
__attribute__((always_inline
)) void __writegsbyte(const unsigned long Offset
, const unsigned char Data
)
573 __asm__("movb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
576 static __inline__
__attribute__((always_inline
)) void __writegsword(const unsigned long Offset
, const unsigned short Data
)
578 __asm__("movw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
581 static __inline__
__attribute__((always_inline
)) void __writegsdword(const unsigned long Offset
, const unsigned long Data
)
583 __asm__("movl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
586 static __inline__
__attribute__((always_inline
)) void __writegsqword(const unsigned long Offset
, const unsigned __int64 Data
)
588 __asm__("movq %q[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
591 static __inline__
__attribute__((always_inline
)) unsigned char __readgsbyte(const unsigned long Offset
)
594 __asm__("movb %%gs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
598 static __inline__
__attribute__((always_inline
)) unsigned short __readgsword(const unsigned long Offset
)
600 unsigned short value
;
601 __asm__("movw %%gs:%a[Offset], %w[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
605 static __inline__
__attribute__((always_inline
)) unsigned long __readgsdword(const unsigned long Offset
)
608 __asm__("movl %%gs:%a[Offset], %k[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
612 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readgsqword(const unsigned long Offset
)
615 __asm__("movq %%gs:%a[Offset], %q[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
619 static __inline__
__attribute__((always_inline
)) void __incgsbyte(const unsigned long Offset
)
621 __asm__("incb %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
));
624 static __inline__
__attribute__((always_inline
)) void __incgsword(const unsigned long Offset
)
626 __asm__("incw %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
));
629 static __inline__
__attribute__((always_inline
)) void __incgsdword(const unsigned long Offset
)
631 __asm__("incl %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
));
634 /* NOTE: the bizarre implementation of __addgsxxx mimics the broken Visual C++ behavior */
635 static __inline__
__attribute__((always_inline
)) void __addgsbyte(const unsigned long Offset
, const unsigned char Data
)
637 if(!__builtin_constant_p(Offset
))
638 __asm__("addb %k[Offset], %%gs:%a[Offset]" : : [Offset
] "r" (Offset
));
640 __asm__("addb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
643 static __inline__
__attribute__((always_inline
)) void __addgsword(const unsigned long Offset
, const unsigned short Data
)
645 if(!__builtin_constant_p(Offset
))
646 __asm__("addw %k[Offset], %%gs:%a[Offset]" : : [Offset
] "r" (Offset
));
648 __asm__("addw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
651 static __inline__
__attribute__((always_inline
)) void __addgsdword(const unsigned long Offset
, const unsigned int Data
)
653 if(!__builtin_constant_p(Offset
))
654 __asm__("addl %k[Offset], %%gs:%a[Offset]" : : [Offset
] "r" (Offset
));
656 __asm__("addl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
660 /*** FS segment addressing ***/
661 static __inline__
__attribute__((always_inline
)) void __writefsbyte(const unsigned long Offset
, const unsigned char Data
)
663 __asm__("movb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
666 static __inline__
__attribute__((always_inline
)) void __writefsword(const unsigned long Offset
, const unsigned short Data
)
668 __asm__("movw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
671 static __inline__
__attribute__((always_inline
)) void __writefsdword(const unsigned long Offset
, const unsigned long Data
)
673 __asm__("movl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
676 static __inline__
__attribute__((always_inline
)) unsigned char __readfsbyte(const unsigned long Offset
)
679 __asm__("movb %%fs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
683 static __inline__
__attribute__((always_inline
)) unsigned short __readfsword(const unsigned long Offset
)
685 unsigned short value
;
686 __asm__("movw %%fs:%a[Offset], %w[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
690 static __inline__
__attribute__((always_inline
)) unsigned long __readfsdword(const unsigned long Offset
)
693 __asm__("movl %%fs:%a[Offset], %k[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
697 static __inline__
__attribute__((always_inline
)) void __incfsbyte(const unsigned long Offset
)
699 __asm__("incb %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
));
702 static __inline__
__attribute__((always_inline
)) void __incfsword(const unsigned long Offset
)
704 __asm__("incw %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
));
707 static __inline__
__attribute__((always_inline
)) void __incfsdword(const unsigned long Offset
)
709 __asm__("incl %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
));
712 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
713 static __inline__
__attribute__((always_inline
)) void __addfsbyte(const unsigned long Offset
, const unsigned char Data
)
715 if(!__builtin_constant_p(Offset
))
716 __asm__("addb %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
));
718 __asm__("addb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
721 static __inline__
__attribute__((always_inline
)) void __addfsword(const unsigned long Offset
, const unsigned short Data
)
723 if(!__builtin_constant_p(Offset
))
724 __asm__("addw %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
));
726 __asm__("addw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
729 static __inline__
__attribute__((always_inline
)) void __addfsdword(const unsigned long Offset
, const unsigned int Data
)
731 if(!__builtin_constant_p(Offset
))
732 __asm__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
));
734 __asm__("addl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
));
739 /*** Bit manipulation ***/
740 static __inline__
__attribute__((always_inline
)) unsigned char _BitScanForward(unsigned long * const Index
, const unsigned long Mask
)
742 __asm__("bsfl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
746 static __inline__
__attribute__((always_inline
)) unsigned char _BitScanReverse(unsigned long * const Index
, const unsigned long Mask
)
748 __asm__("bsrl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
752 /* NOTE: again, the bizarre implementation follows Visual C++ */
753 static __inline__
__attribute__((always_inline
)) unsigned char _bittest(const long * const a
, const long b
)
755 unsigned char retval
;
757 if(__builtin_constant_p(b
))
758 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
760 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*a
), [b
] "r" (b
));
765 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandcomplement(long * const a
, const long b
)
767 unsigned char retval
;
769 if(__builtin_constant_p(b
))
770 __asm__("btc %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
772 __asm__("btc %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*a
), [b
] "r" (b
));
777 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandreset(long * const a
, const long b
)
779 unsigned char retval
;
781 if(__builtin_constant_p(b
))
782 __asm__("btr %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
784 __asm__("btr %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*a
), [b
] "r" (b
));
789 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandset(long * const a
, const long b
)
791 unsigned char retval
;
793 if(__builtin_constant_p(b
))
794 __asm__("bts %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
796 __asm__("bts %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*a
), [b
] "r" (b
));
801 static __inline__
__attribute__((always_inline
)) unsigned char _rotl8(const unsigned char value
, const unsigned char shift
)
803 unsigned char retval
;
804 __asm__("rolb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
808 static __inline__
__attribute__((always_inline
)) unsigned short _rotl16(const unsigned short value
, const unsigned char shift
)
810 unsigned short retval
;
811 __asm__("rolw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
816 static __inline__
__attribute__((always_inline
)) unsigned int _rotl(const unsigned int value
, const int shift
)
818 unsigned long retval
;
819 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
823 static __inline__
__attribute__((always_inline
)) unsigned long _rotr(const unsigned int value
, const unsigned char shift
)
825 unsigned long retval
;
826 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
831 static __inline__
__attribute__((always_inline
)) unsigned char _rotr8(const unsigned char value
, const unsigned char shift
)
833 unsigned char retval
;
834 __asm__("rorb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
838 static __inline__
__attribute__((always_inline
)) unsigned short _rotr16(const unsigned short value
, const unsigned char shift
)
840 unsigned short retval
;
841 __asm__("rorw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
846 NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
847 constraint (edx:eax) for the Mask argument, because it's the only way GCC
848 can pass 64-bit operands around - passing the two 32 bit parts separately
849 just confuses it. Also we declare Bit as an int and then truncate it to
850 match Visual C++ behavior
852 static __inline__
__attribute__((always_inline
)) unsigned long long __ll_lshift(const unsigned long long Mask
, const int Bit
)
854 unsigned long long retval
= Mask
;
858 "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
860 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
866 static __inline__
__attribute__((always_inline
)) long long __ll_rshift(const long long Mask
, const int Bit
)
868 unsigned long long retval
= Mask
;
872 "shldl %b[Bit], %%eax, %%edx; sarl %b[Bit], %%eax" :
874 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
880 static __inline__
__attribute__((always_inline
)) unsigned long long __ull_rshift(const unsigned long long Mask
, int Bit
)
882 unsigned long long retval
= Mask
;
886 "shrdl %b[Bit], %%eax, %%edx; shrl %b[Bit], %%eax" :
888 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
895 /*** 64-bit math ***/
896 static __inline__
__attribute__((always_inline
)) long long __emul(const int a
, const int b
)
899 __asm__("imull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
903 static __inline__
__attribute__((always_inline
)) unsigned long long __emulu(const unsigned int a
, const unsigned int b
)
905 unsigned long long retval
;
906 __asm__("mull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
912 static __inline__
__attribute__((always_inline
)) unsigned char __inbyte(const unsigned short Port
)
915 __asm__
__volatile__("inb %w[Port], %b[byte]" : [byte
] "=a" (byte
) : [Port
] "Nd" (Port
));
919 static __inline__
__attribute__((always_inline
)) unsigned short __inword(const unsigned short Port
)
922 __asm__
__volatile__("inw %w[Port], %w[word]" : [word
] "=a" (word
) : [Port
] "Nd" (Port
));
926 static __inline__
__attribute__((always_inline
)) unsigned long __indword(const unsigned short Port
)
929 __asm__
__volatile__("inl %w[Port], %k[dword]" : [dword
] "=a" (dword
) : [Port
] "Nd" (Port
));
933 static __inline__
__attribute__((always_inline
)) void __inbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
938 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
939 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
944 static __inline__
__attribute__((always_inline
)) void __inwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
949 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
950 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
955 static __inline__
__attribute__((always_inline
)) void __indwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
960 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
961 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
966 static __inline__
__attribute__((always_inline
)) void __outbyte(unsigned short const Port
, const unsigned char Data
)
968 __asm__
__volatile__("outb %b[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
971 static __inline__
__attribute__((always_inline
)) void __outword(unsigned short const Port
, const unsigned short Data
)
973 __asm__
__volatile__("outw %w[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
976 static __inline__
__attribute__((always_inline
)) void __outdword(unsigned short const Port
, const unsigned long Data
)
978 __asm__
__volatile__("outl %k[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
981 static __inline__
__attribute__((always_inline
)) void __outbytestring(unsigned short const Port
, const unsigned char * const Buffer
, const unsigned long Count
)
983 __asm__
__volatile__("rep; outsb" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
986 static __inline__
__attribute__((always_inline
)) void __outwordstring(unsigned short const Port
, const unsigned short * const Buffer
, const unsigned long Count
)
988 __asm__
__volatile__("rep; outsw" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
991 static __inline__
__attribute__((always_inline
)) void __outdwordstring(unsigned short const Port
, const unsigned long * const Buffer
, const unsigned long Count
)
993 __asm__
__volatile__("rep; outsl" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
997 /*** System information ***/
998 static __inline__
__attribute__((always_inline
)) void __cpuid(int CPUInfo
[], const int InfoType
)
1000 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
));
1003 static __inline__
__attribute__((always_inline
)) unsigned long long __rdtsc(void)
1005 unsigned long long retval
;
1006 __asm__
__volatile__("rdtsc" : "=A"(retval
));
1011 /*** Interrupts ***/
1012 static __inline__
__attribute__((always_inline
)) void __debugbreak(void)
1017 static __inline__
__attribute__((always_inline
)) void __int2c(void)
1019 __asm__("int $0x2c");
1022 static __inline__
__attribute__((always_inline
)) void _disable(void)
1027 static __inline__
__attribute__((always_inline
)) void _enable(void)
1033 /*** Protected memory management ***/
1034 static __inline__
__attribute__((always_inline
)) unsigned long __readcr0(void)
1036 unsigned long value
;
1037 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=q" (value
));
1041 static __inline__
__attribute__((always_inline
)) unsigned long __readcr2(void)
1043 unsigned long value
;
1044 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=q" (value
));
1048 static __inline__
__attribute__((always_inline
)) unsigned long __readcr3(void)
1050 unsigned long value
;
1051 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=q" (value
));
1055 static __inline__
__attribute__((always_inline
)) unsigned long __readcr4(void)
1057 unsigned long value
;
1058 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=q" (value
));
1062 static __inline__
__attribute__((always_inline
)) void __writecr0(const unsigned long long Data
)
1064 __asm__("mov %[Data], %%cr0" : : [Data
] "q" ((const unsigned long)(Data
& 0xFFFFFFFF)) : "memory");
1067 static __inline__
__attribute__((always_inline
)) void __writecr3(const unsigned long long Data
)
1069 __asm__("mov %[Data], %%cr3" : : [Data
] "q" ((const unsigned long)(Data
& 0xFFFFFFFF)) : "memory");
1072 static __inline__
__attribute__((always_inline
)) void __writecr4(const unsigned long long Data
)
1074 __asm__("mov %[Data], %%cr4" : : [Data
] "q" ((const unsigned long)(Data
& 0xFFFFFFFF)) : "memory");
1077 static __inline__
__attribute__((always_inline
)) void __invlpg(void * const Address
)
1079 __asm__("invlpg %[Address]" : : [Address
] "m" (*((unsigned char *)(Address
))));
1083 /*** System operations ***/
1084 static __inline__
__attribute__((always_inline
)) unsigned long long __readmsr(const int reg
)
1086 unsigned long long retval
;
1087 __asm__
__volatile__("rdmsr" : "=A" (retval
) : "c" (reg
));
1091 static __inline__
__attribute__((always_inline
)) void __writemsr(const unsigned long Register
, const unsigned long long Value
)
1093 __asm__
__volatile__("wrmsr" : : "A" (Value
), "c" (Register
));
1096 static __inline__
__attribute__((always_inline
)) unsigned long long __readpmc(const int counter
)
1098 unsigned long long retval
;
1099 __asm__
__volatile__("rdpmc" : "=A" (retval
) : "c" (counter
));
1103 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1104 static __inline__
__attribute__((always_inline
)) unsigned long __segmentlimit(const unsigned long a
)
1106 unsigned long retval
;
1107 __asm__
__volatile__("lsl %[a], %[retval]" : [retval
] "=r" (retval
) : [a
] "rm" (a
));
1111 static __inline__
__attribute__((always_inline
)) void __wbinvd(void)
1113 __asm__
__volatile__("wbinvd");
1116 #endif /* KJK_INTRIN_X86_H_ */