2 Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3 Microsoft Visual C++ functions. Originally developed for the ReactOS
4 (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
7 Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
9 Permission is hereby granted, free of charge, to any person obtaining a
10 copy of this software and associated documentation files (the "Software"),
11 to deal in the Software without restriction, including without limitation
12 the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 and/or sell copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following conditions:
16 The above copyright notice and this permission notice shall be included in
17 all copies or substantial portions of the Software.
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 DEALINGS IN THE SOFTWARE.
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
32 FIXME: review all "memory" clobbers, add/remove to match Visual C++
33 behavior: some "obvious" memory barriers are not present in the Visual C++
34 implementation - e.g. __stosX; on the other hand, some memory barriers that
35 *are* present could have been missed
39 NOTE: this is a *compatibility* header. Some functions may look wrong at
40 first, but they're only "as wrong" as they would be on Visual C++. Our
41 priority is compatibility
43 NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44 constraints and the uses of __volatile__ out of my... hat. Do not touch
45 them. I hate cargo cult programming
47 NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48 barriers aren't there in Visual C++ (e.g. __stosX)
50 NOTE: review all intrinsics with a return value, add/remove __volatile__
51 where necessary. If an intrinsic whose value is ignored generates a no-op
52 under Visual C++, __volatile__ must be omitted; if it always generates code
53 (for example, if it has side effects), __volatile__ must be specified. GCC
54 will only optimize out non-volatile asm blocks with outputs, so input-only
55 blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56 and follow Visual C++ behavior
58 NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59 atomic operations. Test the version like this:
61 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
64 Pay attention to the type of barrier. Make it match with what Visual C++
65 would use in the same case
72 /*** memcopy must be memmove ***/
73 void* memmove(void* dest
, const void* source
, size_t num
);
74 __INTRIN_INLINE
void* memcpy(void* dest
, const void* source
, size_t num
)
76 return memmove(dest
, source
, num
);
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
85 /* Maybe the same for x86? */
87 #define _alloca(s) __builtin_alloca(s)
90 /*** Memory barriers ***/
92 __INTRIN_INLINE
void _ReadWriteBarrier(void)
94 __asm__
__volatile__("" : : : "memory");
97 /* GCC only supports full barriers */
98 #define _ReadBarrier _ReadWriteBarrier
99 #define _WriteBarrier _ReadWriteBarrier
101 __INTRIN_INLINE
void _mm_mfence(void)
103 __asm__
__volatile__("mfence" : : : "memory");
106 __INTRIN_INLINE
void _mm_lfence(void)
109 __asm__
__volatile__("lfence");
113 __INTRIN_INLINE
void _mm_sfence(void)
116 __asm__
__volatile__("sfence");
121 __INTRIN_INLINE
void __faststorefence(void)
124 __asm__
__volatile__("lock; orl $0, %0;" : : "m"(local
));
129 /*** Atomic operations ***/
131 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
133 __INTRIN_INLINE
char _InterlockedCompareExchange8(volatile char * Destination
, char Exchange
, char Comperand
)
135 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
138 __INTRIN_INLINE
short _InterlockedCompareExchange16(volatile short * Destination
, short Exchange
, short Comperand
)
140 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
145 __INTRIN_INLINE
long _InterlockedCompareExchange(volatile long * Destination
, long Exchange
, long Comperand
)
147 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
150 __INTRIN_INLINE
void * _InterlockedCompareExchangePointer(void * volatile * Destination
, void * Exchange
, void * Comperand
)
152 return (void *)__sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
157 __INTRIN_INLINE
char _InterlockedExchange8(volatile char * Target
, char Value
)
159 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
160 __sync_synchronize();
161 return __sync_lock_test_and_set(Target
, Value
);
164 __INTRIN_INLINE
short _InterlockedExchange16(volatile short * Target
, short Value
)
166 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
167 __sync_synchronize();
168 return __sync_lock_test_and_set(Target
, Value
);
173 __INTRIN_INLINE
long _InterlockedExchange(volatile long * Target
, long Value
)
175 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
176 __sync_synchronize();
177 return __sync_lock_test_and_set(Target
, Value
);
180 __INTRIN_INLINE
void * _InterlockedExchangePointer(void * volatile * Target
, void * Value
)
182 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
183 __sync_synchronize();
184 return (void *)__sync_lock_test_and_set(Target
, Value
);
189 #if defined(__x86_64__)
190 __INTRIN_INLINE
long long _InterlockedExchange64(volatile long long * Target
, long long Value
)
192 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
193 __sync_synchronize();
194 return __sync_lock_test_and_set(Target
, Value
);
198 __INTRIN_INLINE
char _InterlockedExchangeAdd8(char volatile * Addend
, char Value
)
200 return __sync_fetch_and_add(Addend
, Value
);
203 __INTRIN_INLINE
short _InterlockedExchangeAdd16(volatile short * Addend
, short Value
)
205 return __sync_fetch_and_add(Addend
, Value
);
209 __INTRIN_INLINE
long _InterlockedExchangeAdd(volatile long * Addend
, long Value
)
211 return __sync_fetch_and_add(Addend
, Value
);
215 #if defined(__x86_64__)
216 __INTRIN_INLINE
long long _InterlockedExchangeAdd64(volatile long long * Addend
, long long Value
)
218 return __sync_fetch_and_add(Addend
, Value
);
222 __INTRIN_INLINE
char _InterlockedAnd8(volatile char * value
, char mask
)
224 return __sync_fetch_and_and(value
, mask
);
227 __INTRIN_INLINE
short _InterlockedAnd16(volatile short * value
, short mask
)
229 return __sync_fetch_and_and(value
, mask
);
232 __INTRIN_INLINE
long _InterlockedAnd(volatile long * value
, long mask
)
234 return __sync_fetch_and_and(value
, mask
);
237 #if defined(__x86_64__)
238 __INTRIN_INLINE
long long _InterlockedAnd64(volatile long long * value
, long long mask
)
240 return __sync_fetch_and_and(value
, mask
);
244 __INTRIN_INLINE
char _InterlockedOr8(volatile char * value
, char mask
)
246 return __sync_fetch_and_or(value
, mask
);
249 __INTRIN_INLINE
short _InterlockedOr16(volatile short * value
, short mask
)
251 return __sync_fetch_and_or(value
, mask
);
254 __INTRIN_INLINE
long _InterlockedOr(volatile long * value
, long mask
)
256 return __sync_fetch_and_or(value
, mask
);
259 #if defined(__x86_64__)
260 __INTRIN_INLINE
long long _InterlockedOr64(volatile long long * value
, long long mask
)
262 return __sync_fetch_and_or(value
, mask
);
266 __INTRIN_INLINE
char _InterlockedXor8(volatile char * value
, char mask
)
268 return __sync_fetch_and_xor(value
, mask
);
271 __INTRIN_INLINE
short _InterlockedXor16(volatile short * value
, short mask
)
273 return __sync_fetch_and_xor(value
, mask
);
276 __INTRIN_INLINE
long _InterlockedXor(volatile long * value
, long mask
)
278 return __sync_fetch_and_xor(value
, mask
);
281 #if defined(__x86_64__)
282 __INTRIN_INLINE
long long _InterlockedXor64(volatile long long * value
, long long mask
)
284 return __sync_fetch_and_xor(value
, mask
);
289 __INTRIN_INLINE
long _InterlockedDecrement(volatile long * lpAddend
)
291 return __sync_sub_and_fetch(lpAddend
, 1);
294 __INTRIN_INLINE
long _InterlockedIncrement(volatile long * lpAddend
)
296 return __sync_add_and_fetch(lpAddend
, 1);
300 __INTRIN_INLINE
short _InterlockedDecrement16(volatile short * lpAddend
)
302 return __sync_sub_and_fetch(lpAddend
, 1);
305 __INTRIN_INLINE
short _InterlockedIncrement16(volatile short * lpAddend
)
307 return __sync_add_and_fetch(lpAddend
, 1);
310 #if defined(__x86_64__)
311 __INTRIN_INLINE
long long _InterlockedDecrement64(volatile long long * lpAddend
)
313 return __sync_sub_and_fetch(lpAddend
, 1);
316 __INTRIN_INLINE
long long _InterlockedIncrement64(volatile long long * lpAddend
)
318 return __sync_add_and_fetch(lpAddend
, 1);
322 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
324 __INTRIN_INLINE
char _InterlockedCompareExchange8(volatile char * Destination
, char Exchange
, char Comperand
)
326 char retval
= Comperand
;
327 __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
331 __INTRIN_INLINE
short _InterlockedCompareExchange16(volatile short * Destination
, short Exchange
, short Comperand
)
333 short retval
= Comperand
;
334 __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
338 __INTRIN_INLINE
long _InterlockedCompareExchange(volatile long * Destination
, long Exchange
, long Comperand
)
340 long retval
= Comperand
;
341 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
345 __INTRIN_INLINE
void * _InterlockedCompareExchangePointer(void * volatile * Destination
, void * Exchange
, void * Comperand
)
347 void * retval
= (void *)Comperand
;
348 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "=a" (retval
) : "[retval]" (retval
), [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
352 __INTRIN_INLINE
char _InterlockedExchange8(volatile char * Target
, char Value
)
355 __asm__("xchgb %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
359 __INTRIN_INLINE
short _InterlockedExchange16(volatile short * Target
, short Value
)
361 short retval
= Value
;
362 __asm__("xchgw %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
366 __INTRIN_INLINE
long _InterlockedExchange(volatile long * Target
, long Value
)
369 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
373 __INTRIN_INLINE
void * _InterlockedExchangePointer(void * volatile * Target
, void * Value
)
375 void * retval
= Value
;
376 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
380 __INTRIN_INLINE
char _InterlockedExchangeAdd8(char volatile * Addend
, char Value
)
383 __asm__("lock; xaddb %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
387 __INTRIN_INLINE
short _InterlockedExchangeAdd16(volatile short * Addend
, short Value
)
389 short retval
= Value
;
390 __asm__("lock; xaddw %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
394 __INTRIN_INLINE
long _InterlockedExchangeAdd(volatile long * Addend
, long Value
)
397 __asm__("lock; xaddl %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
401 __INTRIN_INLINE
char _InterlockedAnd8(volatile char * value
, char mask
)
411 y
= _InterlockedCompareExchange8(value
, x
& mask
, x
);
418 __INTRIN_INLINE
short _InterlockedAnd16(volatile short * value
, short mask
)
428 y
= _InterlockedCompareExchange16(value
, x
& mask
, x
);
435 __INTRIN_INLINE
long _InterlockedAnd(volatile long * value
, long mask
)
445 y
= _InterlockedCompareExchange(value
, x
& mask
, x
);
452 __INTRIN_INLINE
char _InterlockedOr8(volatile char * value
, char mask
)
462 y
= _InterlockedCompareExchange8(value
, x
| mask
, x
);
469 __INTRIN_INLINE
short _InterlockedOr16(volatile short * value
, short mask
)
479 y
= _InterlockedCompareExchange16(value
, x
| mask
, x
);
486 __INTRIN_INLINE
long _InterlockedOr(volatile long * value
, long mask
)
496 y
= _InterlockedCompareExchange(value
, x
| mask
, x
);
503 __INTRIN_INLINE
char _InterlockedXor8(volatile char * value
, char mask
)
513 y
= _InterlockedCompareExchange8(value
, x
^ mask
, x
);
520 __INTRIN_INLINE
short _InterlockedXor16(volatile short * value
, short mask
)
530 y
= _InterlockedCompareExchange16(value
, x
^ mask
, x
);
537 __INTRIN_INLINE
long _InterlockedXor(volatile long * value
, long mask
)
547 y
= _InterlockedCompareExchange(value
, x
^ mask
, x
);
554 __INTRIN_INLINE
long _InterlockedDecrement(volatile long * lpAddend
)
556 return _InterlockedExchangeAdd(lpAddend
, -1) - 1;
559 __INTRIN_INLINE
long _InterlockedIncrement(volatile long * lpAddend
)
561 return _InterlockedExchangeAdd(lpAddend
, 1) + 1;
564 __INTRIN_INLINE
short _InterlockedDecrement16(volatile short * lpAddend
)
566 return _InterlockedExchangeAdd16(lpAddend
, -1) - 1;
569 __INTRIN_INLINE
short _InterlockedIncrement16(volatile short * lpAddend
)
571 return _InterlockedExchangeAdd16(lpAddend
, 1) + 1;
574 #if defined(__x86_64__)
575 __INTRIN_INLINE
long long _InterlockedDecrement64(volatile long long * lpAddend
)
577 return _InterlockedExchangeAdd64(lpAddend
, -1) - 1;
580 __INTRIN_INLINE
long long _InterlockedIncrement64(volatile long long * lpAddend
)
582 return _InterlockedExchangeAdd64(lpAddend
, 1) + 1;
586 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
588 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
590 __INTRIN_INLINE
long long _InterlockedCompareExchange64(volatile long long * Destination
, long long Exchange
, long long Comperand
)
592 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
597 __INTRIN_INLINE
long long _InterlockedCompareExchange64(volatile long long * Destination
, long long Exchange
, long long Comperand
)
599 long long retval
= Comperand
;
603 "lock; cmpxchg8b %[Destination]" :
604 [retval
] "+A" (retval
) :
605 [Destination
] "m" (*Destination
),
606 "b" ((unsigned long)((Exchange
>> 0) & 0xFFFFFFFF)),
607 "c" ((unsigned long)((Exchange
>> 32) & 0xFFFFFFFF)) :
617 __INTRIN_INLINE
long _InterlockedAddLargeStatistic(volatile long long * Addend
, long Value
)
621 "lock; addl %[Value], %[Lo32];"
623 "lock; adcl $0, %[Hi32];"
625 [Lo32
] "+m" (*((volatile long *)(Addend
) + 0)), [Hi32
] "+m" (*((volatile long *)(Addend
) + 1)) :
626 [Value
] "ir" (Value
) :
632 #endif /* __i386__ */
634 __INTRIN_INLINE
unsigned char _interlockedbittestandreset(volatile long * a
, long b
)
636 unsigned char retval
;
637 __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
641 #if defined(__x86_64__)
642 __INTRIN_INLINE
unsigned char _interlockedbittestandreset64(volatile long long * a
, long long b
)
644 unsigned char retval
;
645 __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
650 __INTRIN_INLINE
unsigned char _interlockedbittestandset(volatile long * a
, long b
)
652 unsigned char retval
;
653 __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
657 #if defined(__x86_64__)
658 __INTRIN_INLINE
unsigned char _interlockedbittestandset64(volatile long long * a
, long long b
)
660 unsigned char retval
;
661 __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
666 /*** String operations ***/
668 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
669 __INTRIN_INLINE
void __stosb(unsigned char * Dest
, unsigned char Data
, size_t Count
)
674 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
675 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
679 __INTRIN_INLINE
void __stosw(unsigned short * Dest
, unsigned short Data
, size_t Count
)
684 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
685 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
689 __INTRIN_INLINE
void __stosd(unsigned long * Dest
, unsigned long Data
, size_t Count
)
694 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
695 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
700 __INTRIN_INLINE
void __stosq(unsigned long long * Dest
, unsigned long long Data
, size_t Count
)
705 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
706 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
711 __INTRIN_INLINE
void __movsb(unsigned char * Destination
, const unsigned char * Source
, size_t Count
)
716 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
717 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
721 __INTRIN_INLINE
void __movsw(unsigned short * Destination
, const unsigned short * Source
, size_t Count
)
726 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
727 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
731 __INTRIN_INLINE
void __movsd(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
736 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
737 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
742 __INTRIN_INLINE
void __movsq(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
747 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
748 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
753 #if defined(__x86_64__)
755 /*** GS segment addressing ***/
757 __INTRIN_INLINE
void __writegsbyte(unsigned long Offset
, unsigned char Data
)
759 __asm__
__volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
762 __INTRIN_INLINE
void __writegsword(unsigned long Offset
, unsigned short Data
)
764 __asm__
__volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
767 __INTRIN_INLINE
void __writegsdword(unsigned long Offset
, unsigned long Data
)
769 __asm__
__volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
772 __INTRIN_INLINE
void __writegsqword(unsigned long Offset
, unsigned long long Data
)
774 __asm__
__volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
777 __INTRIN_INLINE
unsigned char __readgsbyte(unsigned long Offset
)
780 __asm__
__volatile__("movb %%gs:%a[Offset], %b[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
784 __INTRIN_INLINE
unsigned short __readgsword(unsigned long Offset
)
786 unsigned short value
;
787 __asm__
__volatile__("movw %%gs:%a[Offset], %w[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
791 __INTRIN_INLINE
unsigned long __readgsdword(unsigned long Offset
)
794 __asm__
__volatile__("movl %%gs:%a[Offset], %k[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
798 __INTRIN_INLINE
unsigned long long __readgsqword(unsigned long Offset
)
800 unsigned long long value
;
801 __asm__
__volatile__("movq %%gs:%a[Offset], %q[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
805 __INTRIN_INLINE
void __incgsbyte(unsigned long Offset
)
807 __asm__
__volatile__("incb %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
810 __INTRIN_INLINE
void __incgsword(unsigned long Offset
)
812 __asm__
__volatile__("incw %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
815 __INTRIN_INLINE
void __incgsdword(unsigned long Offset
)
817 __asm__
__volatile__("incl %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
820 __INTRIN_INLINE
void __incgsqword(unsigned long Offset
)
822 __asm__
__volatile__("incq %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
825 __INTRIN_INLINE
void __addgsbyte(unsigned long Offset
, unsigned char Data
)
827 __asm__
__volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
830 __INTRIN_INLINE
void __addgsword(unsigned long Offset
, unsigned short Data
)
832 __asm__
__volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
835 __INTRIN_INLINE
void __addgsdword(unsigned long Offset
, unsigned int Data
)
837 __asm__
__volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
840 __INTRIN_INLINE
void __addgsqword(unsigned long Offset
, unsigned long long Data
)
842 __asm__
__volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
845 #else /* defined(__x86_64__) */
847 /*** FS segment addressing ***/
849 __INTRIN_INLINE
void __writefsbyte(unsigned long Offset
, unsigned char Data
)
851 __asm__
__volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
854 __INTRIN_INLINE
void __writefsword(unsigned long Offset
, unsigned short Data
)
856 __asm__
__volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
859 __INTRIN_INLINE
void __writefsdword(unsigned long Offset
, unsigned long Data
)
861 __asm__
__volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
864 __INTRIN_INLINE
unsigned char __readfsbyte(unsigned long Offset
)
867 __asm__
__volatile__("movb %%fs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "ir" (Offset
));
871 __INTRIN_INLINE
unsigned short __readfsword(unsigned long Offset
)
873 unsigned short value
;
874 __asm__
__volatile__("movw %%fs:%a[Offset], %w[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
878 __INTRIN_INLINE
unsigned long __readfsdword(unsigned long Offset
)
881 __asm__
__volatile__("movl %%fs:%a[Offset], %k[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
885 __INTRIN_INLINE
void __incfsbyte(unsigned long Offset
)
887 __asm__
__volatile__("incb %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
890 __INTRIN_INLINE
void __incfsword(unsigned long Offset
)
892 __asm__
__volatile__("incw %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
895 __INTRIN_INLINE
void __incfsdword(unsigned long Offset
)
897 __asm__
__volatile__("incl %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
900 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
901 __INTRIN_INLINE
void __addfsbyte(unsigned long Offset
, unsigned char Data
)
903 if(!__builtin_constant_p(Offset
))
904 __asm__
__volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
906 __asm__
__volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
909 __INTRIN_INLINE
void __addfsword(unsigned long Offset
, unsigned short Data
)
911 if(!__builtin_constant_p(Offset
))
912 __asm__
__volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
914 __asm__
__volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
917 __INTRIN_INLINE
void __addfsdword(unsigned long Offset
, unsigned long Data
)
919 if(!__builtin_constant_p(Offset
))
920 __asm__
__volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
922 __asm__
__volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
925 #endif /* defined(__x86_64__) */
928 /*** Bit manipulation ***/
930 __INTRIN_INLINE
unsigned char _BitScanForward(unsigned long * Index
, unsigned long Mask
)
932 __asm__("bsfl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
936 __INTRIN_INLINE
unsigned char _BitScanReverse(unsigned long * Index
, unsigned long Mask
)
938 __asm__("bsrl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
942 /* NOTE: again, the bizarre implementation follows Visual C++ */
943 __INTRIN_INLINE
unsigned char _bittest(const long * a
, long b
)
945 unsigned char retval
;
947 if(__builtin_constant_p(b
))
948 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
950 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "m" (*a
), [b
] "r" (b
));
956 __INTRIN_INLINE
unsigned char _BitScanForward64(unsigned long * Index
, unsigned long long Mask
);
958 __asm__("bsfq %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
962 __INTRIN_INLINE
unsigned char _BitScanReverse64(unsigned long * Index
, unsigned long long Mask
);
964 __asm__("bsrq %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
968 __INTRIN_INLINE
unsigned char _bittest64(const long long * a
, long long b
)
970 unsigned char retval
;
972 if(__builtin_constant_p(b
))
973 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 64))), [b
] "Ir" (b
% 64));
975 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "m" (*a
), [b
] "r" (b
));
981 __INTRIN_INLINE
unsigned char _bittestandcomplement(long * a
, long b
)
983 unsigned char retval
;
985 if(__builtin_constant_p(b
))
986 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
988 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
993 __INTRIN_INLINE
unsigned char _bittestandreset(long * a
, long b
)
995 unsigned char retval
;
997 if(__builtin_constant_p(b
))
998 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
1000 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1005 __INTRIN_INLINE
unsigned char _bittestandset(long * a
, long b
)
1007 unsigned char retval
;
1009 if(__builtin_constant_p(b
))
1010 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
1012 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1019 __INTRIN_INLINE
unsigned char _bittestandset64(long long * a
, long long b
);
1021 unsigned char retval
;
1023 if(__builtin_constant_p(b
))
1024 __asm__("btsq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1026 __asm__("btsq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1031 __INTRIN_INLINE
unsigned char _bittestandreset64(long long * a
, long long b
);
1033 unsigned char retval
;
1035 if(__builtin_constant_p(b
))
1036 __asm__("btrq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1038 __asm__("btrq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1043 __INTRIN_INLINE
unsigned char _bittestandcomplement64(long long * a
, long long b
);
1045 unsigned char retval
;
1047 if(__builtin_constant_p(b
))
1048 __asm__("btcq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1050 __asm__("btcq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1057 __INTRIN_INLINE
unsigned char _rotl8(unsigned char value
, unsigned char shift
)
1059 unsigned char retval
;
1060 __asm__("rolb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1064 __INTRIN_INLINE
unsigned short _rotl16(unsigned short value
, unsigned char shift
)
1066 unsigned short retval
;
1067 __asm__("rolw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1071 __INTRIN_INLINE
unsigned int _rotl(unsigned int value
, int shift
)
1073 unsigned int retval
;
1074 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1079 __INTRIN_INLINE
unsigned long long _rotl64(unsigned long long value
, int shift
)
1081 unsigned long long retval
;
1082 __asm__("rolq %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1086 __INTRIN_INLINE
unsigned long long _rotl64(unsigned long long value
, int shift
)
1088 /* FIXME: this is probably not optimal */
1089 return (value
<< shift
) | (value
>> (64 - shift
));
1093 __INTRIN_INLINE
unsigned int _rotr(unsigned int value
, int shift
)
1095 unsigned int retval
;
1096 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1100 __INTRIN_INLINE
unsigned char _rotr8(unsigned char value
, unsigned char shift
)
1102 unsigned char retval
;
1103 __asm__("rorb %b[shift], %b[retval]" : [retval
] "=qm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1107 __INTRIN_INLINE
unsigned short _rotr16(unsigned short value
, unsigned char shift
)
1109 unsigned short retval
;
1110 __asm__("rorw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1115 __INTRIN_INLINE
unsigned long long _rotr64(unsigned long long value
, int shift
)
1117 unsigned long long retval
;
1118 __asm__("rorq %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1122 __INTRIN_INLINE
unsigned long long _rotr64(unsigned long long value
, int shift
)
1124 /* FIXME: this is probably not optimal */
1125 return (value
>> shift
) | (value
<< (64 - shift
));
1129 __INTRIN_INLINE
unsigned long __cdecl
_lrotl(unsigned long value
, int shift
)
1131 unsigned long retval
;
1132 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1136 __INTRIN_INLINE
unsigned long __cdecl
_lrotr(unsigned long value
, int shift
)
1138 unsigned long retval
;
1139 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1144 NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1145 constraint (edx:eax) for the Mask argument, because it's the only way GCC
1146 can pass 64-bit operands around - passing the two 32 bit parts separately
1147 just confuses it. Also we declare Bit as an int and then truncate it to
1148 match Visual C++ behavior
1150 __INTRIN_INLINE
unsigned long long __ll_lshift(unsigned long long Mask
, int Bit
)
1152 unsigned long long retval
= Mask
;
1156 "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1158 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1164 __INTRIN_INLINE
long long __ll_rshift(long long Mask
, int Bit
)
1166 long long retval
= Mask
;
1170 "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1172 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1178 __INTRIN_INLINE
unsigned long long __ull_rshift(unsigned long long Mask
, int Bit
)
1180 unsigned long long retval
= Mask
;
1184 "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1186 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1192 __INTRIN_INLINE
unsigned short _byteswap_ushort(unsigned short value
)
1194 unsigned short retval
;
1195 __asm__("rorw $8, %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
));
1199 __INTRIN_INLINE
unsigned long _byteswap_ulong(unsigned long value
)
1201 unsigned long retval
;
1202 __asm__("bswapl %[retval]" : [retval
] "=r" (retval
) : "[retval]" (value
));
1207 __INTRIN_INLINE
unsigned long long _byteswap_uint64(unsigned long long value
)
1209 unsigned long long retval
;
1210 __asm__("bswapq %[retval]" : [retval
] "=r" (retval
) : "[retval]" (value
));
1214 __INTRIN_INLINE
unsigned long long _byteswap_uint64(unsigned long long value
)
1217 unsigned long long int64part
;
1219 unsigned long lowpart
;
1220 unsigned long hipart
;
1223 retval
.int64part
= value
;
1224 __asm__("bswapl %[lowpart]\n"
1225 "bswapl %[hipart]\n"
1226 : [lowpart
] "=r" (retval
.hipart
), [hipart
] "=r" (retval
.lowpart
) : "[lowpart]" (retval
.lowpart
), "[hipart]" (retval
.hipart
) );
1227 return retval
.int64part
;
1231 __INTRIN_INLINE
unsigned int __lzcnt(unsigned int value
)
1233 return __builtin_clz(value
);
1236 __INTRIN_INLINE
unsigned short __lzcnt16(unsigned short value
)
1238 return __builtin_clz(value
);
1241 __INTRIN_INLINE
unsigned int __popcnt(unsigned int value
)
1243 return __builtin_popcount(value
);
1246 __INTRIN_INLINE
unsigned short __popcnt16(unsigned short value
)
1248 return __builtin_popcount(value
);
1252 unsigned long long __lzcnt64(unsigned long long value
)
1254 return __builtin_clzll(value
);
1257 unsigned long long __popcnt64(unsigned long long value
)
1259 return __builtin_popcountll(value
);
1263 /*** 64-bit math ***/
1265 __INTRIN_INLINE
long long __emul(int a
, int b
)
1268 __asm__("imull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1272 __INTRIN_INLINE
unsigned long long __emulu(unsigned int a
, unsigned int b
)
1274 unsigned long long retval
;
1275 __asm__("mull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1279 __INTRIN_INLINE
long long __cdecl
_abs64(long long value
)
1281 return (value
>= 0) ? value
: -value
;
1286 __INTRIN_INLINE
long long __mulh(long long a
, long long b
)
1289 __asm__("imulq %[b]" : "=d" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1293 __INTRIN_INLINE
unsigned long long __umulh(unsigned long long a
, unsigned long long b
)
1295 unsigned long long retval
;
1296 __asm__("mulq %[b]" : "=d" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1304 __INTRIN_INLINE
unsigned char __inbyte(unsigned short Port
)
1307 __asm__
__volatile__("inb %w[Port], %b[byte]" : [byte
] "=a" (byte
) : [Port
] "Nd" (Port
));
1311 __INTRIN_INLINE
unsigned short __inword(unsigned short Port
)
1313 unsigned short word
;
1314 __asm__
__volatile__("inw %w[Port], %w[word]" : [word
] "=a" (word
) : [Port
] "Nd" (Port
));
1318 __INTRIN_INLINE
unsigned long __indword(unsigned short Port
)
1320 unsigned long dword
;
1321 __asm__
__volatile__("inl %w[Port], %k[dword]" : [dword
] "=a" (dword
) : [Port
] "Nd" (Port
));
1325 __INTRIN_INLINE
void __inbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
1327 __asm__ __volatile__
1330 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1331 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1336 __INTRIN_INLINE
void __inwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
1338 __asm__ __volatile__
1341 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1342 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1347 __INTRIN_INLINE
void __indwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
1349 __asm__ __volatile__
1352 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1353 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1358 __INTRIN_INLINE
void __outbyte(unsigned short Port
, unsigned char Data
)
1360 __asm__
__volatile__("outb %b[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1363 __INTRIN_INLINE
void __outword(unsigned short Port
, unsigned short Data
)
1365 __asm__
__volatile__("outw %w[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1368 __INTRIN_INLINE
void __outdword(unsigned short Port
, unsigned long Data
)
1370 __asm__
__volatile__("outl %k[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1373 __INTRIN_INLINE
void __outbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
1375 __asm__
__volatile__("rep; outsb" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1378 __INTRIN_INLINE
void __outwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
1380 __asm__
__volatile__("rep; outsw" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1383 __INTRIN_INLINE
void __outdwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
1385 __asm__
__volatile__("rep; outsl" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1388 __INTRIN_INLINE
int _inp(unsigned short Port
)
1390 return __inbyte(Port
);
1393 __INTRIN_INLINE
unsigned short _inpw(unsigned short Port
)
1395 return __inword(Port
);
1398 __INTRIN_INLINE
unsigned long _inpd(unsigned short Port
)
1400 return __indword(Port
);
1403 __INTRIN_INLINE
int _outp(unsigned short Port
, int databyte
)
1405 __outbyte(Port
, (unsigned char)databyte
);
1409 __INTRIN_INLINE
unsigned short _outpw(unsigned short Port
, unsigned short dataword
)
1411 __outword(Port
, dataword
);
1415 __INTRIN_INLINE
unsigned long _outpd(unsigned short Port
, unsigned long dataword
)
1417 __outdword(Port
, dataword
);
1422 /*** System information ***/
1424 __INTRIN_INLINE
void __cpuid(int CPUInfo
[4], int InfoType
)
1426 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
));
1429 __INTRIN_INLINE
void __cpuidex(int CPUInfo
[4], int InfoType
, int ECXValue
)
1431 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
), "c" (ECXValue
));
1434 __INTRIN_INLINE
unsigned long long __rdtsc(void)
1437 unsigned long long low
, high
;
1438 __asm__
__volatile__("rdtsc" : "=a"(low
), "=d"(high
));
1439 return low
| (high
<< 32);
1441 unsigned long long retval
;
1442 __asm__
__volatile__("rdtsc" : "=A"(retval
));
1447 __INTRIN_INLINE
void __writeeflags(uintptr_t Value
)
1449 __asm__
__volatile__("push %0\n popf" : : "rim"(Value
));
1452 __INTRIN_INLINE
uintptr_t __readeflags(void)
1455 __asm__
__volatile__("pushf\n pop %0" : "=rm"(retval
));
1459 /*** Interrupts ***/
1462 #define __debugbreak() __asm__("int $3")
1464 __INTRIN_INLINE
void __debugbreak(void)
1470 __INTRIN_INLINE
void __ud2(void)
1475 __INTRIN_INLINE
void __int2c(void)
1477 __asm__("int $0x2c");
1480 __INTRIN_INLINE
void _disable(void)
1482 __asm__("cli" : : : "memory");
1485 __INTRIN_INLINE
void _enable(void)
1487 __asm__("sti" : : : "memory");
1490 __INTRIN_INLINE
void __halt(void)
1492 __asm__("hlt" : : : "memory");
1495 __declspec(noreturn
)
1496 __INTRIN_INLINE
void __fastfail(unsigned int Code
)
1498 __asm__("int $0x29" : : "c"(Code
) : "memory");
1499 __builtin_unreachable();
1502 /*** Protected memory management ***/
1506 __INTRIN_INLINE
void __writecr0(unsigned long long Data
)
1508 __asm__("mov %[Data], %%cr0" : : [Data
] "r" (Data
) : "memory");
1511 __INTRIN_INLINE
void __writecr3(unsigned long long Data
)
1513 __asm__("mov %[Data], %%cr3" : : [Data
] "r" (Data
) : "memory");
1516 __INTRIN_INLINE
void __writecr4(unsigned long long Data
)
1518 __asm__("mov %[Data], %%cr4" : : [Data
] "r" (Data
) : "memory");
1521 __INTRIN_INLINE
void __writecr8(unsigned long long Data
)
1523 __asm__("mov %[Data], %%cr8" : : [Data
] "r" (Data
) : "memory");
1526 __INTRIN_INLINE
unsigned long long __readcr0(void)
1528 unsigned long long value
;
1529 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=r" (value
));
1533 __INTRIN_INLINE
unsigned long long __readcr2(void)
1535 unsigned long long value
;
1536 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=r" (value
));
1540 __INTRIN_INLINE
unsigned long long __readcr3(void)
1542 unsigned long long value
;
1543 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=r" (value
));
1547 __INTRIN_INLINE
unsigned long long __readcr4(void)
1549 unsigned long long value
;
1550 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=r" (value
));
1554 __INTRIN_INLINE
unsigned long long __readcr8(void)
1556 unsigned long long value
;
1557 __asm__
__volatile__("movq %%cr8, %q[value]" : [value
] "=r" (value
));
1561 #else /* __x86_64__ */
1563 __INTRIN_INLINE
void __writecr0(unsigned int Data
)
1565 __asm__("mov %[Data], %%cr0" : : [Data
] "r" (Data
) : "memory");
1568 __INTRIN_INLINE
void __writecr3(unsigned int Data
)
1570 __asm__("mov %[Data], %%cr3" : : [Data
] "r" (Data
) : "memory");
1573 __INTRIN_INLINE
void __writecr4(unsigned int Data
)
1575 __asm__("mov %[Data], %%cr4" : : [Data
] "r" (Data
) : "memory");
1578 __INTRIN_INLINE
void __writecr8(unsigned int Data
)
1580 __asm__("mov %[Data], %%cr8" : : [Data
] "r" (Data
) : "memory");
1583 __INTRIN_INLINE
unsigned long __readcr0(void)
1585 unsigned long value
;
1586 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=r" (value
));
1590 __INTRIN_INLINE
unsigned long __readcr2(void)
1592 unsigned long value
;
1593 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=r" (value
));
1597 __INTRIN_INLINE
unsigned long __readcr3(void)
1599 unsigned long value
;
1600 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=r" (value
));
1604 __INTRIN_INLINE
unsigned long __readcr4(void)
1606 unsigned long value
;
1607 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=r" (value
));
1611 __INTRIN_INLINE
unsigned long __readcr8(void)
1613 unsigned long value
;
1614 __asm__
__volatile__("mov %%cr8, %[value]" : [value
] "=r" (value
));
1618 #endif /* __x86_64__ */
1622 __INTRIN_INLINE
unsigned long long __readdr(unsigned int reg
)
1624 unsigned long long value
;
1628 __asm__
__volatile__("movq %%dr0, %q[value]" : [value
] "=r" (value
));
1631 __asm__
__volatile__("movq %%dr1, %q[value]" : [value
] "=r" (value
));
1634 __asm__
__volatile__("movq %%dr2, %q[value]" : [value
] "=r" (value
));
1637 __asm__
__volatile__("movq %%dr3, %q[value]" : [value
] "=r" (value
));
1640 __asm__
__volatile__("movq %%dr4, %q[value]" : [value
] "=r" (value
));
1643 __asm__
__volatile__("movq %%dr5, %q[value]" : [value
] "=r" (value
));
1646 __asm__
__volatile__("movq %%dr6, %q[value]" : [value
] "=r" (value
));
1649 __asm__
__volatile__("movq %%dr7, %q[value]" : [value
] "=r" (value
));
1655 __INTRIN_INLINE
void __writedr(unsigned reg
, unsigned long long value
)
1660 __asm__("movq %q[value], %%dr0" : : [value
] "r" (value
) : "memory");
1663 __asm__("movq %q[value], %%dr1" : : [value
] "r" (value
) : "memory");
1666 __asm__("movq %q[value], %%dr2" : : [value
] "r" (value
) : "memory");
1669 __asm__("movq %q[value], %%dr3" : : [value
] "r" (value
) : "memory");
1672 __asm__("movq %q[value], %%dr4" : : [value
] "r" (value
) : "memory");
1675 __asm__("movq %q[value], %%dr5" : : [value
] "r" (value
) : "memory");
1678 __asm__("movq %q[value], %%dr6" : : [value
] "r" (value
) : "memory");
1681 __asm__("movq %q[value], %%dr7" : : [value
] "r" (value
) : "memory");
1686 #else /* __x86_64__ */
1688 __INTRIN_INLINE
unsigned int __readdr(unsigned int reg
)
1694 __asm__
__volatile__("mov %%dr0, %[value]" : [value
] "=r" (value
));
1697 __asm__
__volatile__("mov %%dr1, %[value]" : [value
] "=r" (value
));
1700 __asm__
__volatile__("mov %%dr2, %[value]" : [value
] "=r" (value
));
1703 __asm__
__volatile__("mov %%dr3, %[value]" : [value
] "=r" (value
));
1706 __asm__
__volatile__("mov %%dr4, %[value]" : [value
] "=r" (value
));
1709 __asm__
__volatile__("mov %%dr5, %[value]" : [value
] "=r" (value
));
1712 __asm__
__volatile__("mov %%dr6, %[value]" : [value
] "=r" (value
));
1715 __asm__
__volatile__("mov %%dr7, %[value]" : [value
] "=r" (value
));
1721 __INTRIN_INLINE
void __writedr(unsigned reg
, unsigned int value
)
1726 __asm__("mov %[value], %%dr0" : : [value
] "r" (value
) : "memory");
1729 __asm__("mov %[value], %%dr1" : : [value
] "r" (value
) : "memory");
1732 __asm__("mov %[value], %%dr2" : : [value
] "r" (value
) : "memory");
1735 __asm__("mov %[value], %%dr3" : : [value
] "r" (value
) : "memory");
1738 __asm__("mov %[value], %%dr4" : : [value
] "r" (value
) : "memory");
1741 __asm__("mov %[value], %%dr5" : : [value
] "r" (value
) : "memory");
1744 __asm__("mov %[value], %%dr6" : : [value
] "r" (value
) : "memory");
1747 __asm__("mov %[value], %%dr7" : : [value
] "r" (value
) : "memory");
1752 #endif /* __x86_64__ */
1754 __INTRIN_INLINE
void __invlpg(void *Address
)
1756 __asm__("invlpg %[Address]" : : [Address
] "m" (*((unsigned char *)(Address
))) : "memory");
1760 /*** System operations ***/
1762 __INTRIN_INLINE
unsigned long long __readmsr(unsigned long reg
)
1765 unsigned long low
, high
;
1766 __asm__
__volatile__("rdmsr" : "=a" (low
), "=d" (high
) : "c" (reg
));
1767 return ((unsigned long long)high
<< 32) | low
;
1769 unsigned long long retval
;
1770 __asm__
__volatile__("rdmsr" : "=A" (retval
) : "c" (reg
));
1775 __INTRIN_INLINE
void __writemsr(unsigned long Register
, unsigned long long Value
)
1778 __asm__
__volatile__("wrmsr" : : "a" (Value
), "d" (Value
>> 32), "c" (Register
));
1780 __asm__
__volatile__("wrmsr" : : "A" (Value
), "c" (Register
));
1784 __INTRIN_INLINE
unsigned long long __readpmc(unsigned long counter
)
1786 unsigned long long retval
;
1787 __asm__
__volatile__("rdpmc" : "=A" (retval
) : "c" (counter
));
1791 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1792 __INTRIN_INLINE
unsigned long __segmentlimit(unsigned long a
)
1794 unsigned long retval
;
1795 __asm__
__volatile__("lsl %[a], %[retval]" : [retval
] "=r" (retval
) : [a
] "rm" (a
));
1799 __INTRIN_INLINE
void __wbinvd(void)
1801 __asm__
__volatile__("wbinvd" : : : "memory");
1804 __INTRIN_INLINE
void __lidt(void *Source
)
1806 __asm__
__volatile__("lidt %0" : : "m"(*(short*)Source
));
1809 __INTRIN_INLINE
void __sidt(void *Destination
)
1811 __asm__
__volatile__("sidt %0" : : "m"(*(short*)Destination
) : "memory");
1814 /*** Misc operations ***/
1816 __INTRIN_INLINE
void _mm_pause(void)
1818 __asm__
__volatile__("pause" : : : "memory");
1821 __INTRIN_INLINE
void __nop(void)
1823 __asm__
__volatile__("nop");
1830 #endif /* KJK_INTRIN_X86_H_ */